Nemesis to workload tests 2

.github/config/muted_ya.txt

@@ -14,6 +14,7 @@ ydb/apps/ydb/ut YdbWorkloadTransferTopicToTable.Statistics
 ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk
 ydb/core/blobstorage/dsproxy/ut TDSProxyPutTest.TestBlock42PutStatusOkWith_2_0_VdiskErrors
 ydb/core/blobstorage/dsproxy/ut_fat TBlobStorageProxyTest.TestBatchedPutRequestDoesNotContainAHugeBlob
+ydb/core/blobstorage/pdisk/ut TPDiskTest.FailedToFormatDiskInfoUpdate
 ydb/core/blobstorage/pdisk/ut TPDiskTest.PDiskSlotSizeInUnits
 ydb/core/blobstorage/ut_blobstorage NodeDisconnected.BsQueueRetries
 ydb/core/blobstorage/ut_blobstorage/ut_huge HugeBlobOnlineSizeChange.Compaction

build/conf/bison_lex.conf

@@ -139,15 +139,17 @@ macro _SRC("y", SRC, SRCFLAGS...) {
     .PEERDIR=build/induced/by_bison
     .CMD=${tool:"contrib/tools/bison"} $BISON_FLAGS ${env:_SRC_y_env} $_BISON_HEADER ${hide:_BISON_GEN_EXT} -o ${output;suf=$_BISON_GEN_EXT;nopath:SRC} ${input:SRC} ${SRCFLAGS} ${hide;kv:"p YC"} ${hide;kv:"pc light-green"} && $_BISON_PP
     .STRUCT_CMD=yes
-    .SEM=target_macroses-ITEM && target_macroses-macro target_bison_parser && target_macroses-args PRIVATE ${input:SRC} ${hide;output;suf=${OBJ_SUF}.o;nopath;noext:SRC} ${hide;nopath;noext;output;suf=.h:SRC} ${hide;nopath;noext;addincl;output;suf=.h:SRC} && platform_vars-BISON_FLAGS ${quo:BISON_FLAGS} && conan-tool_requires m4/1.4.19 && conan-imports 'bin, m4* -> ./bin/m4/bin' && conan-tool_requires bison/3.8.2 && conan-imports 'bin, bison* -> ./bin/bison/bin' && conan-imports 'res, * -> ./bin/bison/res'
+    .SEM=target_macroses-ITEM && target_macroses-macro target_bison_parser && target_macroses-args PRIVATE ${input:SRC} ${hide;output;suf=${OBJ_SUF}.o;nopath;noext:SRC} ${hide;output;suf=.h;nopath;noext:SRC} ${hide;addincl;output;suf=.h;nopath;noext:SRC} && platform_vars-BISON_FLAGS ${join= :BISON_FLAGS} && conan-tool_requires m4/1.4.19 && conan-imports 'bin, m4* -> ./bin/m4/bin' && conan-tool_requires bison/3.8.2 && conan-imports 'bin, bison* -> ./bin/bison/bin' && conan-imports 'res, * -> ./bin/bison/res'
+    .STRUCT_SEM=yes
 }
 
 macro _SRC("ypp", SRC, SRCFLAGS...) {
     .PEERDIR=build/induced/by_bison
     # TODO overloading support and redirection to `$_SRC(y $SRC $SRCFLAGS)`
     .CMD=${tool:"contrib/tools/bison"} $BISON_FLAGS ${env:_SRC_y_env} $_BISON_HEADER ${hide:_BISON_GEN_EXT} -o ${output;suf=$_BISON_GEN_EXT;nopath:SRC} ${input:SRC} ${SRCFLAGS} ${hide;kv:"p YC"} ${hide;kv:"pc light-green"} && $_BISON_PP
     .STRUCT_CMD=yes
-    .SEM=$_SRC(y $SRC $SRCFLAGS)
+    .SEM=target_macroses-ITEM && target_macroses-macro target_bison_parser && target_macroses-args PRIVATE ${input:SRC} ${hide;output;suf=${OBJ_SUF}.o;nopath;noext:SRC} ${hide;output;suf=.h;nopath;noext:SRC} ${hide;addincl;output;suf=.h;nopath;noext:SRC} && platform_vars-BISON_FLAGS ${join= :BISON_FLAGS} && conan-tool_requires m4/1.4.19 && conan-imports 'bin, m4* -> ./bin/m4/bin' && conan-tool_requires bison/3.8.2 && conan-imports 'bin, bison* -> ./bin/bison/bin' && conan-imports 'res, * -> ./bin/bison/res'
+    .STRUCT_SEM=yes
 }
 
 macro _SRC("l", SRC, SRCFLAGS...) {

build/conf/coverage_full_instrumentation.conf

@@ -6,3 +6,9 @@ macro POPULATE_CPP_COVERAGE_FLAGS() {
         LDFLAGS+=-fprofile-instr-generate -fcoverage-mapping
     }
 }
+
+macro _SETUP_PYTHON_COVERAGE() {
+    when ($PYTHON_COVERAGE == "yes") {
+        _NEED_PYTHON_COVERAGE=yes
+    }
+}

build/conf/coverage_selective_instrumentation.conf

@@ -16,3 +16,13 @@ macro _SETUP_CLANG_COVERAGE() {
 }
 
 
+macro _SETUP_PYTHON_COVERAGE() {
+    _FILTER_PY_COVERAGE()
+}
+
+# tag:internal
+### @usage: _ENABLE_PYTHON_COVERAGE() # internal
+### This macro is called from python plugin, so by some reason triggers won't work inside it and in global scope.
+macro _ENABLE_PYTHON_COVERAGE() {
+    ENABLE(_NEED_PYTHON_COVERAGE)
+}

build/conf/java.conf

@@ -459,7 +459,7 @@ macro _SEM_JAR_SOURCE_SET(SRCDIR=".", PACKAGE_PREFIX="", EXCLUDE[], FILES[], RES
 # Fill semantic source_sets/resource_sets by JAVA_SRCS() Args
 macro _SEM_SOURCE_SETS(SRCDIR=".", PACKAGE_PREFIX="", EXCLUDE[], FILES[], RESOURCES?"resource_sets":"source_sets", SKIP_CHECK_SRCDIR?"SKIP_CHECK_SRCDIR":"", Globs...) {
     # Now ignore SKIP_CHECK_SRCDIR, EXCLUDE, FILES
-    SET_APPEND(_JAR_SOURCE_SET_SEM && ${RESOURCES}-ITEM && ${RESOURCES}-dir ${SRCDIR} && ${RESOURCES}-globs ${Globs} && ${RESOURCES}-prefix ${PACKAGE_PREFIX})
+    SET_APPEND(_JAR_SOURCE_SET_SEM && ${RESOURCES}-ITEM && ${RESOURCES}-dir ${SRCDIR} && ${RESOURCES}-realdir ${SRCDIR} && ${RESOURCES}-globs ${Globs} && ${RESOURCES}-prefix ${PACKAGE_PREFIX})
 }
 
 ### Fill JAVA_SRCS to value for ya ide idea and real apply for late globs

build/conf/proto.conf

@@ -197,8 +197,9 @@ macro JAVA_PROTO_PLUGIN(NAME, TOOL, DEPS[]) {
     SET(JAVA_PROTOBUF_PEERS $JAVA_PROTOBUF_PEERS $DEPS)
 }
 
+_ADD_SEM_PROP_IF_NON_EMPTY__CMDS=target_properties-ITEM && target_properties-name $Prop && target_properties-value $Args
 macro _ADD_SEM_PROP_IF_NON_EMPTY(Prop, Args...) {
-    .SEM=${pre=&& target_properties-ITEM && target_properties-name $Prop && target_properties-value :Args}
+    .SEM=$IF(${not;empty:Args} THEN $_ADD_SEM_PROP_IF_NON_EMPTY__CMDS)
 }
 
 # tag:proto tag:java-specific
@@ -216,7 +217,7 @@ macro WITH_KOTLIN_GRPC() {
 # tag:proto tag:cpp-specific
 macro _ADD_CPP_PROTO_OUT(Suf) {
     SET_APPEND(CPP_PROTO_OUTS \${norel;output;suf=$Suf;nopath;noext:File})
-    SET_APPEND(PROTOC_EXTRA_OUTS_SEM && protoc_extra_outs $Suf \${hide;output;suf=.o:Suf} \$_ADD_SEM_PROP_IF_NON_EMPTY(PROTO_NAMESPACE $PROTO_NAMESPACE))
+    SET_APPEND(PROTOC_EXTRA_OUTS_SEM && protoc_extra_outs $Suf && \$_ADD_SEM_PROP_IF_NON_EMPTY(PROTO_NAMESPACE $PROTO_NAMESPACE))
 
     # XXX fix variable expansion in plugins
     SET(CPP_PROTO_SUFFIXES $CPP_PROTO_SUFFIXES $Suf)
@@ -232,7 +233,8 @@ CPP_PROTOBUF_PEERS=
 ### Define protoc plugin for C++ with given Name that emits code into regular outputs
 ### using Tool. Extra dependencies are passed via DEPS.
 macro CPP_PROTO_PLUGIN0(NAME, TOOL, DEPS[], EXTRA_OUT_FLAG="") {
-    .SEM=target_macroses-ITEM && target_macroses-macro target_proto_plugin && target_macroses-args $NAME ${tool;rootrel:TOOL} ${hide;output;suf=.fake.o:NAME}
+    .SEM=target_macroses-ITEM && target_macroses-macro target_proto_plugin && target_macroses-args $NAME ${tool:TOOL} ${hide;output;suf=.fake.o:NAME}
+    .STRUCT_SEM=yes
     SET_APPEND(CPP_PROTO_OPTS $_PROTO_PLUGIN_ARGS_BASE($NAME $TOOL $EXTRA_OUT_FLAG))
 
     # XXX fix variable expansion in plugins
@@ -419,7 +421,7 @@ macro YT_ORM_PROTO_YSON(OUT_OPTS[], Files...) {
 
 _SEM_CPP_PROTO_CMD=target_options-privates-ITEM && target_options-privates-option target_proto_messages && target_options-privates-args ${input:File} \
                    $CPP_PROTO_OUTS_SEM ${hide;output;suf=${OBJ_SUF}.pb.o:File} $DEFAULT_PROTOC_TOOLS \
-                   $PROTOC_EXTRA_OUTS_SEM \
+                   && $PROTOC_EXTRA_OUTS_SEM \
                    && platform_vars-COMMON_PROTOC_FLAGS "" \
                    && target_macroses-ITEM && target_macroses-macro target_proto_outs && target_macroses-args --cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $PROTOC_STYLEGUIDE_OUT \
                    && target_macroses-ITEM && target_macroses-macro target_proto_addincls && target_macroses-args ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE ${_PROTO__INCLUDE} $ARCADIA_BUILD_ROOT $PROTOBUF_INCLUDE_PATH

build/conf/python.conf

@@ -3,7 +3,7 @@ CYTHON_SCRIPT=${input:"${ARCADIA_ROOT}/contrib/tools/cython/cython.py"} ${hide:C
 RUN_CYTHON_SCRIPT=$YMAKE_PYTHON $CYTHON_SCRIPT
 RUN_CYTHON_SCRIPT_H=$YMAKE_PYTHON $CYTHON_SCRIPT
 PYTHON_TYPE_FOR_CYTHON=PY3
-
+_NEED_PYTHON_COVERAGE=no
 
 CYTHON_EMBEDED_FILES=\
 contrib/tools/cython/Cython/Utility/arrayarray.h \
@@ -711,6 +711,7 @@ module PY3_LIBRARY: _LIBRARY {
 module _BASE_PY_PROGRAM: _BASE_PROGRAM {
     .CMD=$PY_PROGRAM_LINK_EXE
     _ARCADIA_PYTHON_ADDINCL()
+    _SETUP_PYTHON_COVERAGE()
 
     when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") {
         NO_CHECK_IMPORTS_FOR_VALUE=
@@ -727,7 +728,7 @@ module _BASE_PY_PROGRAM: _BASE_PROGRAM {
             PEERDIR += contrib/tools/python/src/Modules/_sqlite
         }
     }
-    when ($PYTHON_COVERAGE == "yes") {
+    when ($_NEED_PYTHON_COVERAGE == "yes") {
         PEERDIR+=library/python/coverage
     }
 
@@ -805,6 +806,7 @@ module _BASE_PY3_PROGRAM: _BASE_PROGRAM {
     }
     ALLOCATOR($_MY_ALLOCATOR)
     STRIP()
+    _SETUP_PYTHON_COVERAGE()
 
     when ($NO_PYTHON_INCLS != "yes") {
         PEERDIR+=contrib/libs/python
@@ -815,7 +817,7 @@ module _BASE_PY3_PROGRAM: _BASE_PROGRAM {
     when ($SANITIZER_TYPE && $SANITIZER_TYPE != "no") {
         NO_STRIP=yes
     }
-    when ($PYTHON_COVERAGE == "yes") {
+    when ($_NEED_PYTHON_COVERAGE == "yes") {
         PEERDIR+=library/python/coverage
     }
     when ($PYTHON_IMPORT_TRACING == "yes") {

build/conf/settings.conf

@@ -5,6 +5,8 @@ RESOLVE_FORCE_LISTDIR=no
 REPORT_CONFIGURE_PROGRESS=yes
 REPORT_ALL_DUPSRC=yes
 DEPS_CACHE_CONTROL_UIDS_CACHE=yes
+CONF_CACHE_ENABLED=yes
+DEPS_CACHE_CONTROL_CONF_CACHE=yes
 USE_GLOBAL_CMD=yes
 PIC_NO_PIE=no
 FAIL_PY2=no

build/export_generators/ide-gradle/build.gradle.kts.any.jinja

@@ -7,12 +7,43 @@
 {%-     if depend -%}
 {#- Replace (export_root) === (BUILD_ROOT in ymake) to $arcadia_root in Gradle, because prebuilt tools in arcadia, not in build_root -#}
 "{{ arg|replace(export_root, "$output_root")|replace(project_root, "$project_root")|replace(arcadia_root, "$arcadia_root") }}{{ tail }}"
-{%-     elif output and arg[0] != '/' -%}
-{#- Relative outputs in buildDir -#}
-"$buildDir/{{ arg }}{{ tail }}"
 {%-     elif output -%}
+{%-         if arg[0] != '/' -%}
+{#-             Relative outputs in buildDir, make relative from baseBuildDir -#}
+{%-             set bindir = curdir + "/" + arg -%}
+{%-         elif arg|replace(export_root + "/", "") != arg -%}
+{%-             set bindir = arg|replace(export_root + "/", "") -%}
+{%-         else -%}
+{%-             set bindir = false -%}
+{%-         endif -%}
+{%-         if bindir and current_target -%}
+{%-             set all_sets = [] -%}
+{%-             if current_target.source_sets|length -%}
+{%-                 set all_sets = all_sets + current_target.source_sets -%}
+{%-             endif -%}
+{%-             if current_target.resource_sets|length -%}
+{%-                 set all_sets = all_sets + current_target.resource_sets -%}
+{%-             endif -%}
+{#-             Output is relative in export_root -#}
+{%-             set out_set = all_sets|selectattr('realdir', 'startsWith', export_root)|rejectattr('dir', 'startsWith', export_root)|selectattr('realdir', 'in', export_root + "/" + bindir) -%}
+{%-             if out_set|length -%}
+{%-                 set out_set = out_set|first -%}
+{%-                 set out_dir = out_set.dir -%}
+{%-                 if current_target.test_reldir -%}
+{%-                     set out_dir = current_target.test_reldir + "/" + out_set.dir -%}
+{%-                 endif -%}
+{%-                 set out_tail = (export_root + "/" + bindir)|replace(out_set.realdir + "/", "")|replace(out_set.realdir, "") -%}
+{%-                 if out_tail -%}
+{%-                     set out_dir = out_dir + "/" + out_tail -%}
+{%-                 endif -%}
+"$arcadia_root/{{ curdir }}/{{ out_dir }}{{ tail }}"
+{%-             else -%}
 {#- Replace (export_root) === (BUILD_ROOT in ymake) to baseBuildDir in Gradle - root of all build folders for modules -#}
 "{{ arg|replace(export_root, "$baseBuildDir")|replace(project_root, "$project_root")|replace(arcadia_root, "$output_root")|replace(output_root, "$output_root") }}{{ tail }}"
+{%-             endif -%}
+{%-         else -%}
+"{{ arg|replace(project_root, "$project_root")|replace(arcadia_root, "$output_root")|replace(output_root, "$output_root") }}{{ tail }}"
+{%-         endif -%}
 {%-     else -%}
 {#- Replace (export_root) === (BUILD_ROOT in ymake) to baseBuildDir in Gradle - root of all build folders for modules -#}
 "{{ arg|replace(export_root, "$baseBuildDir")|replace(project_root, "$project_root")|replace(arcadia_root, "$arcadia_root")|replace(output_root, "$output_root") }}{{ tail }}"

build/export_generators/ide-gradle/generator.toml

@@ -50,11 +50,13 @@ jdk_path="str"
 
 source_sets="list"
 source_sets-dir="str"
+source_sets-realdir="str"
 source_sets-globs="list"
 source_sets-prefix="str"
 
 resource_sets="list"
 resource_sets-dir="str"
+resource_sets-realdir="str"
 resource_sets-globs="list"
 resource_sets-prefix="str"
 

build/plugins/_common.py

@@ -182,7 +182,6 @@ def ugly_conftest_exception(path):
     exceptions = [
         'taxi/uservices/userver-arc-utils/functional_tests/basic/conftest.py',
         'taxi/uservices/userver-arc-utils/functional_tests/basic_chaos/conftest.py',
-        'taxi/uservices/userver-arc-utils/functional_tests/json2yaml/conftest.py',
     ]
 
     if not path.endswith('conftest.py'):

build/plugins/coverage.py

@@ -16,7 +16,7 @@ def should_be_covered(unit, filters):
     return not any(pred(unit_path) for pred in filters)
 
 
-def get_cpp_coverage_filters(unit, filters=[]):
+def get_coverage_filters(unit, filters=[]):
     # don`t calculate filters if it already was calculated
     if filters:
         return filters
@@ -36,8 +36,21 @@ def get_cpp_coverage_filters(unit, filters=[]):
 
 
 def onset_cpp_coverage_flags(unit):
-    if unit.get("CLANG_COVERAGE") == "no":
+    if unit.get("CLANG_COVERAGE") != "yes":
         return
-    filters = get_cpp_coverage_filters(unit)
+    filters = get_coverage_filters(unit)
     if should_be_covered(unit, filters):
         unit.on_setup_clang_coverage()
+
+
+def on_filter_py_coverage(unit):
+    if unit.get("PYTHON_COVERAGE") != "yes":
+        return
+
+    if unit.get("COVERAGE_FILTER_PROGRAMS") != "yes" or unit.get("CYTHON_COVERAGE") == "yes":
+        unit.on_enable_python_coverage()
+        return
+
+    filters = get_coverage_filters(unit)
+    if should_be_covered(unit, filters):
+        unit.on_enable_python_coverage()

build/sysincl/unsorted.yml

@@ -102,7 +102,6 @@
   - note.h
   - opcdef.h
   - paper.h
-  - pire/config.h
   - pth.h
   - rtl.h
   - shlib-compat.h