[Flaky CI]: _queue.Empty #5768
Description
Flakiness Type
Timeout
Name of Test
tests/integration/test_normalization.py::test_auto_infer_setting_objective_c
Link to Test Run
Details
=================================== FAILURES ===================================
__ test_auto_infer_setting_objective_c[123.123.123.123-auto-123.123.123.123] ___
[gw5] linux -- Python 3.13.5 /home/runner/work/relay/relay/.venv/bin/python3
Traceback (most recent call last):
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 341, in from_call
result: TResult | None = func()
~~~~^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_hooks.py", line 513, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_manager.py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 182, in _multicall
return outcome.get_result()
~~~~~~~~~~~~~~~~~~^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_result.py", line 100, in get_result
raise exc.with_traceback(exc.__traceback__)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/threadexception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/threadexception.py", line 68, in thread_exception_runtest_hook
yield
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/unraisableexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/unraisableexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/logging.py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/logging.py", line 829, in _runtest_for
yield
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/capture.py", line 880, in pytest_runtest_call
return (yield)
^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/skipping.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 103, in _multicall
res = hook_impl.function(*args)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 174, in pytest_runtest_call
item.runtest()
~~~~~~~~~~~~^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/python.py", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_hooks.py", line 513, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_manager.py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 139, in _multicall
raise exception.with_traceback(exception.__traceback__)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 103, in _multicall
res = hook_impl.function(*args)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/python.py", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
File "/home/runner/work/relay/relay/tests/integration/test_normalization.py", line 603, in test_auto_infer_setting_objective_c
envelope = mini_sentry.get_captured_envelope()
File "/home/runner/work/relay/relay/tests/integration/fixtures/mini_sentry.py", line 210, in get_captured_envelope
return self.captured_envelopes.get(timeout=timeout or self.timeout)
~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/hostedtoolcache/Python/3.13.5/x64/lib/python3.13/queue.py", line 212, in get
raise Empty
_queue.Empty
----------------------------- Captured stderr call -----------------------------
mimalloc: warning: unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x20000 bytes, address: 0x7FC660721000, alignment: 0x10000, commit: 1)
2026-03-26T06:51:49.316326Z INFO relay_log::setup: Initialized Sentry client options release="relay@26.3.1+d6ac8a89aba35c2e64c74c236917111b97207933" traces_sample_rate=1.0
2026-03-26T06:51:49.326412Z INFO relay_config::config: generating new relay credentials
mimalloc: warning: unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x20000 bytes, address: 0x7FD3B56ED000, alignment: 0x10000, commit: 1)
2026-03-26T06:51:49.344274Z INFO relay_log::setup: Initialized Sentry client options release="relay@26.3.1+d6ac8a89aba35c2e64c74c236917111b97207933" traces_sample_rate=1.0
2026-03-26T06:51:49.354414Z INFO relay::setup: launching relay from config folder /tmp/pytest-of-runner/pytest-0/popen-gw5/test_auto_infer_setting_object4/relay-1
2026-03-26T06:51:49.354484Z INFO relay::setup: relay mode: managed
2026-03-26T06:51:49.354508Z INFO relay::setup: relay id: 785c37d4-e736-4950-b42e-2c4d337a48b7
2026-03-26T06:51:49.354524Z INFO relay::setup: public key: 0XgfzOgtrpIYwQqD4gOCTnbyUHyX3feUByBorafH6ts
2026-03-26T06:51:49.354553Z INFO relay::setup: log level: trace
2026-03-26T06:51:49.355861Z INFO relay_server: relay server starting
2026-03-26T06:51:49.357905Z INFO relay_server::services::outcome: Configured to emit outcomes as client reports
2026-03-26T06:51:49.360035Z INFO relay_server::services::global_config: global config service starting
2026-03-26T06:51:49.360928Z INFO relay_server::services::outcome: OutcomeProducer started.
2026-03-26T06:51:49.361004Z INFO relay_server::services::global_config: requesting global config from upstream
2026-03-26T06:51:49.361262Z INFO relay_server::services::outcome_aggregator: outcome aggregator started
2026-03-26T06:51:49.361294Z INFO relay_server::services::projects::source::upstream: project upstream cache started
2026-03-26T06:51:49.379117Z INFO relay_server::services::upstream: registering with upstream descriptor=http://127.0.0.1:43855/
2026-03-26T06:51:49.383987Z DEBUG relay_server::services::upstream: got register challenge token="123"
2026-03-26T06:51:49.384007Z DEBUG relay_server::services::upstream: sending register challenge response
2026-03-26T06:51:49.388829Z INFO relay_server::services::upstream: relay successfully registered with upstream
2026-03-26T06:51:49.396068Z INFO relay_server::services::global_config: received global config from upstream
2026-03-26T06:51:49.455749Z INFO relay_server::service: starting 1 store workers
2026-03-26T06:51:49.455945Z INFO relay_server::service: starting 3 envelope processing workers
2026-03-26T06:51:49.457014Z TRACE relay_server::services::buffer::envelope_buffer: PolymorphicEnvelopeBuffer: initializing memory envelope buffer
2026-03-26T06:51:49.457207Z INFO relay_server::services::buffer: EnvelopeBufferService 0: starting
2026-03-26T06:51:49.458106Z INFO relay_server::services::metrics::router: metrics router started
2026-03-26T06:51:49.458626Z INFO relay_server::services::relays: key cache started
2026-03-26T06:51:49.458666Z TRACE relay_system::service::concurrent: Concurrent service loop iteration
2026-03-26T06:51:49.458683Z INFO relay_server::services::server: spawning http server
2026-03-26T06:51:49.458699Z INFO relay_server::services::server: listening on http://127.0.0.1:55559/
2026-03-26T06:51:49.468112Z DEBUG request{method=GET uri=/api/relay/healthcheck/live/ version=HTTP/1.1}: tower_http::trace::on_request: started processing request
2026-03-26T06:51:49.468379Z DEBUG request{method=GET uri=/api/relay/healthcheck/live/ version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=200
2026-03-26T06:51:49.470572Z DEBUG request{method=GET uri=/api/relay/healthcheck/ready/ version=HTTP/1.1}: tower_http::trace::on_request: started processing request
2026-03-26T06:51:49.470868Z DEBUG request{method=GET uri=/api/relay/healthcheck/ready/ version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=200
2026-03-26T06:51:49.472611Z DEBUG request{method=POST uri=/api/42/envelope/ version=HTTP/1.1}: tower_http::trace::on_request: started processing request
2026-03-26T06:51:49.473426Z DEBUG request{method=POST uri=/api/42/envelope/ version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=200
2026-03-26T06:51:49.473635Z TRACE relay_server::services::buffer: EnvelopeBufferService: received push message
2026-03-26T06:51:49.473790Z TRACE relay_server::services::buffer: EnvelopeBufferService: project(s) of envelope ready
2026-03-26T06:51:49.473819Z TRACE relay_server::services::buffer::envelope_buffer: buffer marked not ready project_key="89215aab4e024b0b8351b3cafee55b4c"
2026-03-26T06:51:49.473864Z TRACE relay_server::services::buffer: EnvelopeBufferService: project(s) of envelope not ready
2026-03-26T06:51:49.473874Z TRACE relay_server::services::buffer: EnvelopeBufferService: requesting project(s) update
2026-03-26T06:51:49.473929Z TRACE relay_server::services::projects::cache::state: project state fetch scheduled in 0ns tags.project_key="89215aab4e024b0b8351b3cafee55b4c" attempts=1
2026-03-26T06:51:49.473985Z TRACE relay_server::services::projects::cache::state: project fetch skipped, fetch in progress tags.project_key="89215aab4e024b0b8351b3cafee55b4c"
2026-03-26T06:51:49.474001Z TRACE relay_server::services::projects::cache::state: project fetch skipped, fetch in progress tags.project_key="89215aab4e024b0b8351b3cafee55b4c"
2026-03-26T06:51:49.474013Z TRACE relay_server::services::projects::cache::state: project fetch skipped, fetch in progress tags.project_key="89215aab4e024b0b8351b3cafee55b4c"
2026-03-26T06:51:49.475749Z DEBUG relay_server::services::projects::source::upstream: updating project states for 1/1 projects (attempt 1)
2026-03-26T06:51:49.475839Z DEBUG relay_server::services::projects::source::upstream: sending request of size 1
2026-03-26T06:51:49.477523Z TRACE relay_server::services::projects::cache::state: project state fetch completed with non-pending config tags.project_key="89215aab4e024b0b8351b3cafee55b4c"
2026-03-26T06:51:49.477617Z TRACE relay_server::services::buffer::envelope_buffer: buffer marked ready project_key="89215aab4e024b0b8351b3cafee55b4c"
2026-03-26T06:51:49.477661Z TRACE relay_server::services::buffer: EnvelopeBufferService: project(s) of envelope ready
2026-03-26T06:51:49.477679Z TRACE relay_server::services::buffer: EnvelopeBufferService: popping envelope
2026-03-26T06:51:49.477735Z DEBUG relay_server::managed::envelope: dropped envelope: invalid data (project_id)
2026-03-26T06:51:49.477739Z TRACE relay_server::services::projects::cache::state: project fetch skipped, already up to date tags.project_key="89215aab4e024b0b8351b3cafee55b4c"
2026-03-26T06:51:49.477930Z DEBUG relay_server::services::outcome: Outcome 'invalid data (project_id)' cannot be converted to client report
2026-03-26T06:51:49.478008Z DEBUG relay_server::services::outcome: Outcome 'invalid data (project_id)' cannot be converted to client report
2026-03-26T06:51:49.478011Z DEBUG relay_server::services::outcome: Outcome 'invalid data (project_id)' cannot be converted to client report
2026-03-26T06:51:49.478015Z DEBUG relay_server::services::outcome: Outcome 'invalid data (project_id)' cannot be converted to client report
------------------------------ Captured log call -------------------------------
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/register/challenge/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/register/response/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/projectconfigs/?version=3 HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/projectconfigs/?version=3 HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:54] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:59] "POST /api/0/relays/projectconfigs/?version=3 HTTP/1.1" 200 -
____________________ test_span_attachment_ds_drop[project] _____________________
[gw0] linux -- Python 3.13.5 /home/runner/work/relay/relay/.venv/bin/python3
Traceback (most recent call last):
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 341, in from_call
result: TResult | None = func()
~~~~^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 242, in <lambda>
lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_hooks.py", line 513, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_manager.py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 182, in _multicall
return outcome.get_result()
~~~~~~~~~~~~~~~~~~^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_result.py", line 100, in get_result
raise exc.with_traceback(exc.__traceback__)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/threadexception.py", line 92, in pytest_runtest_call
yield from thread_exception_runtest_hook()
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/threadexception.py", line 68, in thread_exception_runtest_hook
yield
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/unraisableexception.py", line 95, in pytest_runtest_call
yield from unraisable_exception_runtest_hook()
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/unraisableexception.py", line 70, in unraisable_exception_runtest_hook
yield
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/logging.py", line 846, in pytest_runtest_call
yield from self._runtest_for(item, "call")
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/logging.py", line 829, in _runtest_for
yield
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/capture.py", line 880, in pytest_runtest_call
return (yield)
^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 167, in _multicall
teardown.throw(outcome._exception)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/skipping.py", line 257, in pytest_runtest_call
return (yield)
^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 103, in _multicall
res = hook_impl.function(*args)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/runner.py", line 174, in pytest_runtest_call
item.runtest()
~~~~~~~~~~~~^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/python.py", line 1627, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_hooks.py", line 513, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_manager.py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 139, in _multicall
raise exception.with_traceback(exception.__traceback__)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/pluggy/_callers.py", line 103, in _multicall
res = hook_impl.function(*args)
File "/home/runner/work/relay/relay/.venv/lib/python3.13/site-packages/_pytest/python.py", line 159, in pytest_pyfunc_call
result = testfunction(**testargs)
File "/home/runner/work/relay/relay/tests/integration/test_attachmentsv2.py", line 613, in test_span_attachment_ds_drop
assert mini_sentry.get_outcomes(n=3) == [
...<30 lines>...
]
File "/home/runner/work/relay/relay/tests/integration/fixtures/mini_sentry.py", line 241, in get_outcomes
self.captured_outcomes.get(timeout=timeout or self.timeout).get(
~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/hostedtoolcache/Python/3.13.5/x64/lib/python3.13/queue.py", line 212, in get
raise Empty
_queue.Empty
----------------------------- Captured stderr call -----------------------------
mimalloc: warning: unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x20000 bytes, address: 0x7F3384CC9000, alignment: 0x10000, commit: 1)
2026-03-26T06:51:49.441681Z INFO relay_log::setup: Initialized Sentry client options release="relay@26.3.1+d6ac8a89aba35c2e64c74c236917111b97207933" traces_sample_rate=1.0
2026-03-26T06:51:49.456627Z INFO relay_config::config: generating new relay credentials
2026-03-26T06:51:49.478457Z INFO relay_log::setup: Initialized Sentry client options release="relay@26.3.1+d6ac8a89aba35c2e64c74c236917111b97207933" traces_sample_rate=1.0
2026-03-26T06:51:49.489271Z INFO relay::setup: launching relay from config folder /tmp/pytest-of-runner/pytest-0/popen-gw0/test_span_attachment_ds_drop_p0/relay-1
2026-03-26T06:51:49.489314Z INFO relay::setup: relay mode: managed
2026-03-26T06:51:49.489328Z INFO relay::setup: relay id: e5bf1a15-4390-4bcd-b7b3-2b41886b691f
2026-03-26T06:51:49.489337Z INFO relay::setup: public key: cilf0JG9ILNerY-LeTGfgCV8k_4RtA-DSmXRqE4_Le0
2026-03-26T06:51:49.489354Z INFO relay::setup: log level: trace
2026-03-26T06:51:49.490445Z INFO relay_server: relay server starting
2026-03-26T06:51:49.491870Z INFO relay_server::services::outcome: Configured to emit outcomes via http
2026-03-26T06:51:49.492806Z INFO relay_server::services::outcome_aggregator: outcome aggregator started
2026-03-26T06:51:49.493198Z INFO relay_server::services::outcome: OutcomeProducer started.
2026-03-26T06:51:49.493491Z INFO relay_server::services::global_config: global config service starting
2026-03-26T06:51:49.493630Z INFO relay_server::services::global_config: requesting global config from upstream
2026-03-26T06:51:49.493733Z INFO relay_server::services::projects::source::upstream: project upstream cache started
2026-03-26T06:51:49.503402Z INFO relay_server::services::upstream: registering with upstream descriptor=http://127.0.0.1:46501/
2026-03-26T06:51:49.506212Z DEBUG relay_server::services::upstream: got register challenge token="123"
2026-03-26T06:51:49.506233Z DEBUG relay_server::services::upstream: sending register challenge response
2026-03-26T06:51:49.509274Z INFO relay_server::services::upstream: relay successfully registered with upstream
2026-03-26T06:51:49.513156Z INFO relay_server::services::global_config: received global config from upstream
2026-03-26T06:51:49.562716Z INFO relay_server::service: starting 1 store workers
2026-03-26T06:51:49.562955Z INFO relay_server::service: starting 3 envelope processing workers
2026-03-26T06:51:49.563128Z TRACE relay_server::services::buffer::envelope_buffer: PolymorphicEnvelopeBuffer: initializing memory envelope buffer
2026-03-26T06:51:49.564048Z INFO relay_server::services::buffer: EnvelopeBufferService 0: starting
2026-03-26T06:51:49.564574Z INFO relay_server::services::metrics::router: metrics router started
2026-03-26T06:51:49.564691Z INFO relay_server::services::relays: key cache started
2026-03-26T06:51:49.564729Z TRACE relay_system::service::concurrent: Concurrent service loop iteration
2026-03-26T06:51:49.564747Z INFO relay_server::services::server: spawning http server
2026-03-26T06:51:49.564766Z INFO relay_server::services::server: listening on http://127.0.0.1:55559/
2026-03-26T06:51:49.638161Z DEBUG request{method=GET uri=/api/relay/healthcheck/live/ version=HTTP/1.1}: tower_http::trace::on_request: started processing request
2026-03-26T06:51:49.638528Z DEBUG request{method=GET uri=/api/relay/healthcheck/live/ version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=200
2026-03-26T06:51:49.640289Z DEBUG request{method=GET uri=/api/relay/healthcheck/ready/ version=HTTP/1.1}: tower_http::trace::on_request: started processing request
2026-03-26T06:51:49.640417Z DEBUG request{method=GET uri=/api/relay/healthcheck/ready/ version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=200
2026-03-26T06:51:49.642142Z DEBUG request{method=POST uri=/api/42/store/ version=HTTP/1.1}: tower_http::trace::on_request: started processing request
2026-03-26T06:51:49.642914Z DEBUG request{method=POST uri=/api/42/store/ version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=200
2026-03-26T06:51:49.643079Z TRACE relay_server::services::buffer: EnvelopeBufferService: received push message
2026-03-26T06:51:49.643235Z TRACE relay_server::services::buffer: EnvelopeBufferService: project(s) of envelope ready
2026-03-26T06:51:49.643261Z TRACE relay_server::services::buffer::envelope_buffer: buffer marked not ready project_key="feb43f2f153241fd8fc208dc11865562"
2026-03-26T06:51:49.643288Z TRACE relay_server::services::buffer: EnvelopeBufferService: project(s) of envelope not ready
2026-03-26T06:51:49.643297Z TRACE relay_server::services::buffer: EnvelopeBufferService: requesting project(s) update
2026-03-26T06:51:49.643462Z TRACE relay_server::services::projects::cache::state: project state fetch scheduled in 0ns tags.project_key="feb43f2f153241fd8fc208dc11865562" attempts=1
2026-03-26T06:51:49.643567Z TRACE relay_server::services::projects::cache::state: project fetch skipped, fetch in progress tags.project_key="feb43f2f153241fd8fc208dc11865562"
2026-03-26T06:51:49.643584Z TRACE relay_server::services::projects::cache::state: project fetch skipped, fetch in progress tags.project_key="feb43f2f153241fd8fc208dc11865562"
2026-03-26T06:51:49.643596Z TRACE relay_server::services::projects::cache::state: project fetch skipped, fetch in progress tags.project_key="feb43f2f153241fd8fc208dc11865562"
2026-03-26T06:51:49.644196Z DEBUG relay_server::services::projects::source::upstream: updating project states for 1/1 projects (attempt 1)
2026-03-26T06:51:49.644265Z DEBUG relay_server::services::projects::source::upstream: sending request of size 1
2026-03-26T06:51:49.647702Z TRACE relay_server::services::projects::cache::state: project state fetch completed with non-pending config tags.project_key="feb43f2f153241fd8fc208dc11865562"
2026-03-26T06:51:49.647889Z TRACE relay_server::services::buffer::envelope_buffer: buffer marked ready project_key="feb43f2f153241fd8fc208dc11865562"
2026-03-26T06:51:49.647957Z TRACE relay_server::services::buffer: EnvelopeBufferService: project(s) of envelope ready
2026-03-26T06:51:49.648022Z TRACE relay_server::services::projects::cache::state: project fetch skipped, already up to date tags.project_key="feb43f2f153241fd8fc208dc11865562"
2026-03-26T06:51:49.648179Z TRACE relay_server::services::buffer: EnvelopeBufferService: popping envelope
2026-03-26T06:51:49.648244Z DEBUG relay_server::managed::envelope: dropped envelope: invalid data (project_id)
2026-03-26T06:51:49.648519Z TRACE relay_server::services::outcome: batching outcome
2026-03-26T06:51:49.648537Z TRACE relay_server::services::outcome: sending outcome batch of size 1
2026-03-26T06:51:49.650493Z TRACE relay_server::services::outcome: outcome batch sent
------------------------------ Captured log call -------------------------------
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/register/challenge/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/register/response/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/projectconfigs/?version=3 HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/projectconfigs/?version=3 HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:49] "POST /api/0/relays/outcomes/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:54] "POST /api/666/envelope/ HTTP/1.1" 200 -
INFO werkzeug:_internal.py:97 127.0.0.1 - - [26/Mar/2026 06:51:59] "POST /api/0/relays/projectconfigs/?version=3 HTTP/1.1" 200 -
============================= slowest 5 durations ==============================
33.53s call integration/test_spans.py::test_rate_limit_is_consistent_between_transaction_and_spans[666-transaction]
33.48s call integration/test_spans.py::test_rate_limit_is_consistent_between_transaction_and_spans[None-transaction]
17.59s call integration/test_spans.py::test_rate_limit_is_consistent_between_transaction_and_spans[None-transaction_indexed]
17.58s call integration/test_spans.py::test_rate_limit_is_consistent_between_transaction_and_spans[666-transaction_indexed]
15.49s call integration/test_store.py::test_processing_quota_transaction_indexing[3]
=========================== short test summary info ============================
SKIPPED [1] tests/integration/test_autoscaling.py:29: Fails in github CI for unknown reasons
SKIPPED [14] tests/integration/fixtures/gobetween.py:22: Gobetween not installed
SKIPPED [14] tests/integration/fixtures/haproxy.py:22: HAProxy not installed
SKIPPED [1] tests/integration/test_basic.py:139: Flaky test
FAILED tests/integration/test_normalization.py::test_auto_infer_setting_objective_c[123.123.123.123-auto-123.123.123.123] - _queue.Empty
FAILED tests/integration/test_attachmentsv2.py::test_span_attachment_ds_drop[project] - _queue.Empty
============ 2 failed, 853 passed, 30 skipped in 387.93s (0:06:27) =============