diff --git a/include/swift/Runtime/EnvironmentVariables.h b/include/swift/Runtime/EnvironmentVariables.h index de4f799beb586..c0c4c1cdcfcfc 100644 --- a/include/swift/Runtime/EnvironmentVariables.h +++ b/include/swift/Runtime/EnvironmentVariables.h @@ -30,14 +30,24 @@ extern swift::once_t initializeToken; using string = const char *; // Declare backing variables. -#define VARIABLE(name, type, defaultValue, help) extern type name ## _variable; +#define VARIABLE(name, type, defaultValue, help) \ + extern type name##_variable; \ + extern bool name##_isSet_variable; #include "../../../stdlib/public/runtime/EnvironmentVariables.def" -// Define getter functions. +// Define getter functions. This creates one function with the same name as the +// variable which returns the value set for that variable, and second function +// ending in _isSet which returns a boolean indicating whether the variable was +// set at all, to allow detecting when the variable was explicitly set to the +// same value as the default. #define VARIABLE(name, type, defaultValue, help) \ inline type name() { \ swift::once(initializeToken, initialize, nullptr); \ return name##_variable; \ + } \ + inline bool name##_isSet() { \ + swift::once(initializeToken, initialize, nullptr); \ + return name##_isSet_variable; \ } #include "../../../stdlib/public/runtime/EnvironmentVariables.def" diff --git a/include/swift/Runtime/LibPrespecialized.h b/include/swift/Runtime/LibPrespecialized.h index b9523abed7237..1a74bbf34cfd2 100644 --- a/include/swift/Runtime/LibPrespecialized.h +++ b/include/swift/Runtime/LibPrespecialized.h @@ -20,6 +20,7 @@ #include "PrebuiltStringMap.h" #include "swift/ABI/Metadata.h" #include "swift/ABI/TargetLayout.h" +#include "swift/Demangling/Demangler.h" #define LIB_PRESPECIALIZED_TOP_LEVEL_SYMBOL_NAME "_swift_prespecializationsData" @@ -36,23 +37,44 @@ struct LibPrespecializedData { typename Runtime::StoredSize optionFlags; + TargetPointer descriptorMap; + // Existing fields are above, add new fields below this point. + // The major/minor version numbers for this version of the struct. static constexpr uint32_t currentMajorVersion = 1; - static constexpr uint32_t currentMinorVersion = 3; + static constexpr uint32_t currentMinorVersion = 4; + // Version numbers where various fields were introduced. static constexpr uint32_t minorVersionWithDisabledProcessesTable = 2; static constexpr uint32_t minorVersionWithPointerKeyedMetadataMap = 3; static constexpr uint32_t minorVersionWithOptionFlags = 3; + static constexpr uint32_t minorVersionWithDescriptorMap = 4; // Option flags values. enum : typename Runtime::StoredSize { // When this flag is set, the runtime should default to using the // pointer-keyed table. When not set, default to using the name-keyed table. OptionFlagDefaultToPointerKeyedMap = 1ULL << 0, + + // When this flag is set, the runtime should default to using the descriptor + // map. When not set, default to turning off the descriptor map. + OptionFlagDescriptorMapDefaultOn = 1ULL << 1, + + // When this flag is set, descriptorMap is not comprehensive, meaning that + // a negative lookup result is not a definitive failure. + OptionFlagDescriptorMapNotComprehensive = 1ULL << 2, }; - // Helpers for retrieving the metadata map in-process. + // Helpers for safely retrieving various fields. Helpers return 0 or NULL if + // the version number indicates that the field is not present. + + typename Runtime::StoredSize getOptionFlags() const { + if (minorVersion < minorVersionWithOptionFlags) + return 0; + return optionFlags; + } + static bool stringIsNull(const char *str) { return str == nullptr; } using MetadataMap = PrebuiltStringMap; @@ -73,18 +95,141 @@ struct LibPrespecializedData { return pointerKeyedMetadataMap; } - typename Runtime::StoredSize getOptionFlags() const { - if (minorVersion < minorVersionWithOptionFlags) - return 0; - return optionFlags; + using DescriptorMap = + PrebuiltAuxDataImplicitStringMap, + uint16_t>; + + const DescriptorMap *getDescriptorMap() const { + if (minorVersion < minorVersionWithDescriptorMap) + return nullptr; + return reinterpret_cast(descriptorMap); } }; +enum class LibPrespecializedLookupResult { + // We found something. + Found, + + // We didn't find anything, and we know it's not in the shared cache. + DefinitiveNotFound, + + // We didn't find anything, but we couldn't rule out the shared cache. Caller + // must do a full search. + NonDefinitiveNotFound, +}; + const LibPrespecializedData *getLibPrespecializedData(); + Metadata *getLibPrespecializedMetadata(const TypeContextDescriptor *description, const void *const *arguments); void libPrespecializedImageLoaded(); +std::pair +getLibPrespecializedTypeDescriptor(Demangle::NodePointer node); + +/// Given the demangling referring to a particular descriptor, build the +/// canonical simplified version of the demangling that's used for the keys in +/// the descriptorMap. We copy across Extension and Module nodes. Type nodes are +/// all normalized to be OtherNominalType to allow for the runtime allowing +/// type kind mismatches on imported C types in certain cases. Other nodes are +/// skipped. +/// +/// The runtime always searches through duplicates in the table, and uses its +/// own matching on all candidates, so the simplified demangling is allowed to +/// be simplified to the point of having different descriptors sometimes produce +/// the same demangling. +static inline Demangle::NodePointer +buildSimplifiedDescriptorDemangling(Demangle::NodePointer node, + Demangle::Demangler &dem) { + // The node that will be returned to the caller. + Demangle::NodePointer result = nullptr; + + // The bottommost node in the result that we've generated. Additional nodes + // are added as children to this one. + Demangle::NodePointer resultBottom = nullptr; + + // The current node that we're iterating over in the input node tree. + Demangle::NodePointer current = node; + + using Kind = Demangle::Node::Kind; + + // Helper to add a new node to the result. This sets `result` to the node if + // it hasn't already been set (indicating this is the topmost node), and adds + // the node as a child to `resultBottom` otherwise. `resultBottom` is updated + // to point to the new node. + auto addNode = [&](Demangle::NodePointer newNode) { + if (!result) { + result = newNode; + } else { + if (resultBottom->getKind() == Kind::Extension) { + resultBottom->addChild(newNode, dem); + } else { + // Shift the Identifier down, insert before it. + resultBottom->addChild(resultBottom->getFirstChild(), dem); + resultBottom->replaceChild(0, newNode); + } + } + resultBottom = newNode; + }; + + // Walk down the input node tree. + while (current) { + switch (current->getKind()) { + case Kind::Extension: { + // Extensions are copied across. The new extension node has the module + // from the original, and the second child will be added as we traverse + // the next node in the tree. + auto copy = dem.createNode(Kind::Extension); + auto module = current->getChild(0); + if (module == nullptr || module->getKind() != Kind::Module) + return nullptr; + copy->addChild(module, dem); + addNode(copy); + current = current->getChild(1); + break; + } + case Kind::Module: { + // Module contents are always in the form we want, so we can incorporate + // this node verbatim and terminate the walk. + addNode(current); + current = nullptr; + break; + } + case Kind::Protocol: { + // Bring Protocol nodes across verbatim, there's no fuzzy matching. + addNode(current); + current = nullptr; + break; + } + case Kind::OpaqueType: + case Kind::Class: + case Kind::Structure: + case Kind::Enum: + case Kind::TypeAlias: + case Kind::OtherNominalType: { + // Type nodes are copied across with the kind always set to + // OtherNominalType. + auto copy = dem.createNode(Kind::OtherNominalType); + auto identifier = current->getChild(1); + if (identifier == nullptr || identifier->getKind() != Kind::Identifier) + return nullptr; + copy->addChild(identifier, dem); + addNode(copy); + current = current->getChild(0); + break; + } + + default: + // If we don't know about this node, continue the walk with its first + // child. + current = current->getFirstChild(); + break; + } + } + + return result; +} + } // namespace swift // Validate the prespecialized metadata map by building each entry dynamically diff --git a/include/swift/Runtime/PrebuiltStringMap.h b/include/swift/Runtime/PrebuiltStringMap.h index 85dc55a0b4ffa..7051924ed6fbe 100644 --- a/include/swift/Runtime/PrebuiltStringMap.h +++ b/include/swift/Runtime/PrebuiltStringMap.h @@ -17,44 +17,18 @@ #include #include #include +#include namespace swift { -/// A map that can be pre-built out of process. Uses a fixed hash function with -/// no per-process seeding to ensure consistent hashes between builder and user. -/// -/// The elements are tail allocated. `byteSize` can be used to calculate the -/// amount of memory needed. The memory must be initialized with all string -/// values set to null. StringTy is opaque for insertion, except for using the -/// provided stringIsNull function to check for null values. -template -struct PrebuiltStringMap { +struct PrebuiltStringMapBase { uint64_t arraySize; - struct ArrayElement { - StringTy key; - ElemTy value; - }; - - ArrayElement *array() { - uintptr_t start = (uintptr_t)(&arraySize + 1); - return (ArrayElement *)start; - } - - const ArrayElement *array() const { - uintptr_t start = (uintptr_t)(&arraySize + 1); - return (ArrayElement *)start; - } - - static size_t byteSize(uint64_t arraySize) { - return sizeof(PrebuiltStringMap) + sizeof(ArrayElement) * arraySize; - } - /// Construct an empty map. Must be constructed in memory at least as large as /// byteSize(arraySize). The map can hold at most arraySize-1 values. /// Attempting to insert more than that will result in fatal errors when /// inserting or retrieving values. - PrebuiltStringMap(uint64_t arraySize) : arraySize(arraySize) {} + PrebuiltStringMapBase(uint64_t arraySize) : arraySize(arraySize) {} // Based on MurmurHash2 uint64_t hash(const void *data, size_t len) const { @@ -116,17 +90,17 @@ struct PrebuiltStringMap { return hash; } - /// Perform the search portion of an insertion operation. Returns a pointer to - /// the element where string is to be inserted. The caller is responsible for - /// initializing the element to contain the string/value. It is assumed that - /// the key does not already exist in the map. If it does exist, this will - /// insert a useless duplicate. - ArrayElement *insert(const void *string, size_t len) { + /// Search for a matching entry in the map. `isMatch` is called with a + /// candidate index and returns true if there is a match at that index. + template + std::optional findIndex(const void *string, size_t len, + const IsMatch &isMatch) const { uint64_t hashValue = hash(string, len); + size_t index = hashValue % arraySize; size_t numSearched = 0; - while (!stringIsNull(array()[index].key)) { + while (!isMatch(index)) { index = index + 1; if (index >= arraySize) index = 0; @@ -134,12 +108,58 @@ struct PrebuiltStringMap { numSearched++; if (numSearched > arraySize) { assert(false && - "Could not find empty element in PrebuiltStringMap::insert"); - return nullptr; + "Could not find match in PrebuiltStringMapBase::findIndex"); + return std::nullopt; } } - return &array()[index]; + return index; + } +}; + +/// A map that can be pre-built out of process. Uses a fixed hash function with +/// no per-process seeding to ensure consistent hashes between builder and user. +/// +/// The elements are tail allocated. `byteSize` can be used to calculate the +/// amount of memory needed. The memory must be initialized with all string +/// values set to null. StringTy is opaque for insertion, except for using the +/// provided stringIsNull function to check for null values. +template +struct PrebuiltStringMap : PrebuiltStringMapBase { + PrebuiltStringMap(uint64_t arraySize) : PrebuiltStringMapBase(arraySize) {} + + struct ArrayElement { + StringTy key; + ElemTy value; + }; + + ArrayElement *array() { + uintptr_t start = (uintptr_t)(&arraySize + 1); + return (ArrayElement *)start; + } + + const ArrayElement *array() const { + uintptr_t start = (uintptr_t)(&arraySize + 1); + return (ArrayElement *)start; + } + + static size_t byteSize(uint64_t arraySize) { + return sizeof(PrebuiltStringMapBase) + sizeof(ArrayElement) * arraySize; + } + + /// Perform the search portion of an insertion operation. Returns a pointer to + /// the element where string is to be inserted. The caller is responsible for + /// initializing the element to contain the string/value. It is assumed that + /// the key does not already exist in the map. If it does exist, this will + /// insert a useless duplicate. + ArrayElement *insert(const void *string, size_t len) { + auto foundIndex = findIndex(string, len, [&](size_t index) { + return stringIsNull(array()[index].key); + }); + + if (foundIndex) + return &array()[*foundIndex]; + return nullptr; } ArrayElement *insert(const char *string) { @@ -154,32 +174,109 @@ struct PrebuiltStringMap { } const ArrayElement *find(const char *toFind, size_t len) const { - uint64_t hashValue = hash(toFind, len); + auto equalOrNull = [&](size_t index) { + auto key = array()[index].key; - size_t index = hashValue % arraySize; + // NULL is considered a "match" as we want to stop the search on NULL too. + if (stringIsNull(key)) + return true; - size_t numSearched = 0; - while (const char *key = array()[index].key) { // key is NUL terminated but toFind may not be. Check that they have equal // contents up to len, and check that key has a terminating NUL at the // right point. if (strncmp(key, toFind, len) == 0 && key[len] == 0) - return &array()[index]; + return true; - index = index + 1; - if (index >= arraySize) - index = 0; + // Not NULL, not equal, keep searching. + return false; + }; + auto foundIndex = findIndex(toFind, len, equalOrNull); + if (!foundIndex) + return nullptr; - numSearched++; - if (numSearched > arraySize) { - assert( - false && - "Could not find match or empty element in PrebuiltStringMap::find"); - return nullptr; - } - } + const auto &elementPtr = &array()[*foundIndex]; - return nullptr; + // If the "matching" element contains a NULL then we didn't find a match. + if (stringIsNull(elementPtr->key)) + return nullptr; + + return elementPtr; + } +}; + +/// A pre-built map with string-based keys that are implicit, i.e. equality can +/// be determined by looking at the values. The map contains auxiliary data +/// stored out of line from the main elements, to avoid padding when the aux +/// data is smaller than the alignment of the main elements. +template +struct PrebuiltAuxDataImplicitStringMap : PrebuiltStringMapBase { + PrebuiltAuxDataImplicitStringMap(uint64_t arraySize) + : PrebuiltStringMapBase(arraySize) {} + + static size_t byteSize(uint64_t arraySize) { + return sizeof(PrebuiltStringMapBase) + sizeof(ElemTy) * arraySize + + sizeof(AuxTy) * arraySize; + } + + using DataPointers = std::pair; + using DataPointersConst = std::pair; + + const ElemTy *elements() const { return (const ElemTy *)(&arraySize + 1); } + + ElemTy *elements() { return (ElemTy *)(&arraySize + 1); } + + const AuxTy *aux() const { return (const AuxTy *)(elements() + arraySize); } + + AuxTy *aux() { return (AuxTy *)(elements() + arraySize); } + + DataPointersConst pointers(size_t index) const { + return {&elements()[index], &aux()[index]}; + } + + DataPointers pointers(size_t index) { + return {&elements()[index], &aux()[index]}; + } + + /// Perform the search portion of an insertion operation. Returns pointers to + /// the element and aux data where the value is to be inserted. The caller is + /// responsible for initializing the element and aux data. It is assumed that + /// the key does not already exist in the map. If it does exist, this will + /// insert a duplicate. + /// + /// isNull is a callable passed a pair of pointers to an element and + /// corresponding auxiliary data, and must return true if the element is + /// considered NULL (empty). + template + DataPointers insert(const char *string, const IsNull &isNull) { + auto foundIndex = findIndex(string, strlen(string), [&](size_t index) { + return isNull(pointers(index)); + }); + if (!foundIndex) + return {nullptr, nullptr}; + return pointers(*foundIndex); + } + + /// Look up the given key in the map. + /// + /// isMatch is a callable passed a pair of pointers to the element and + /// auxiliary data, and must return true if the elements they point to are a + /// match for what's being looked up. + /// + /// isNull must return true if the elements are NULL/empty, as with insert(). + /// + /// The returned pointers point to the matched element and auxiliary data, if + /// a match was found. They point to a NULL entry if no map was found. They + /// will only be NULL if the table data was malformed and no match or NULL + /// exists in it. + template + DataPointersConst find(const char *toFind, size_t len, const IsMatch &isMatch, + const IsNull &isNull) const { + auto foundIndex = findIndex(toFind, len, [&](size_t index) { + return isNull(pointers(index)) || isMatch(pointers(index)); + }); + if (!foundIndex) + return {nullptr, nullptr}; + return pointers(*foundIndex); } }; diff --git a/stdlib/public/runtime/EnvironmentVariables.cpp b/stdlib/public/runtime/EnvironmentVariables.cpp index eeb9b7b0c1153..20491493e018b 100644 --- a/stdlib/public/runtime/EnvironmentVariables.cpp +++ b/stdlib/public/runtime/EnvironmentVariables.cpp @@ -161,8 +161,9 @@ void printHelp(const char *extra) { } // end anonymous namespace // Define backing variables. -#define VARIABLE(name, type, defaultValue, help) \ - type swift::runtime::environment::name ## _variable = defaultValue; +#define VARIABLE(name, type, defaultValue, help) \ + type swift::runtime::environment::name##_variable = defaultValue; \ + bool swift::runtime::environment::name##_isSet_variable = false; #include "EnvironmentVariables.def" // Initialization code. @@ -194,6 +195,11 @@ void swift::runtime::environment::initialize(void *context) { // us to detect some spelling mistakes by warning on unknown SWIFT_ variables. bool SWIFT_DEBUG_HELP_variable = false; + + // Placeholder variable, we never use the result but the macros want to write + // to it. + bool SWIFT_DEBUG_HELP_isSet_variable = false; + (void)SWIFT_DEBUG_HELP_isSet_variable; // Silence warnings about unused vars. for (char **var = ENVIRON; *var; var++) { // Immediately skip anything without a SWIFT_ prefix. if (strncmp(*var, "SWIFT_", 6) != 0) @@ -204,12 +210,13 @@ void swift::runtime::environment::initialize(void *context) { // parsed by functions named parse_ above. An unknown type will // produce an error that parse_ doesn't exist. Add new parsers // above. -#define VARIABLE(name, type, defaultValue, help) \ - if (strncmp(*var, #name "=", strlen(#name "=")) == 0) { \ - name ## _variable = \ - parse_ ## type(#name, *var + strlen(#name "="), defaultValue); \ - foundVariable = true; \ - } +#define VARIABLE(name, type, defaultValue, help) \ + if (strncmp(*var, #name "=", strlen(#name "=")) == 0) { \ + name##_variable = \ + parse_##type(#name, *var + strlen(#name "="), defaultValue); \ + name##_isSet_variable = true; \ + foundVariable = true; \ + } // SWIFT_DEBUG_HELP is not in the variables list. Parse it like the other // variables. VARIABLE(SWIFT_DEBUG_HELP, bool, false, ) @@ -238,8 +245,13 @@ void swift::runtime::environment::initialize(void *context) { void swift::runtime::environment::initialize(void *context) { // Emit a getenv call for each variable. This is less efficient but works // everywhere. -#define VARIABLE(name, type, defaultValue, help) \ - name ## _variable = parse_ ## type(#name, getenv(#name), defaultValue); +#define VARIABLE(name, type, defaultValue, help) \ + do { \ + const char name##_string = getenv(#name); \ + if (name##_string) \ + name##_isSet_variable = true; \ + name##_variable = parse_##type(#name, name##_string, defaultValue); \ + } while (0) #include "EnvironmentVariables.def" // Print help if requested. diff --git a/stdlib/public/runtime/EnvironmentVariables.def b/stdlib/public/runtime/EnvironmentVariables.def index 8b8ea3f78ad02..e384c27557b3b 100644 --- a/stdlib/public/runtime/EnvironmentVariables.def +++ b/stdlib/public/runtime/EnvironmentVariables.def @@ -78,8 +78,15 @@ VARIABLE(SWIFT_BINARY_COMPATIBILITY_VERSION, uint32_t, 0, VARIABLE(SWIFT_DEBUG_FAILED_TYPE_LOOKUP, bool, false, "Enable warnings when we fail to look up a type by name.") -VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED, bool, true, - "Enable use of prespecializations library.") +VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_METADATA, bool, true, + "Enable use of metadata in prespecializations library.") + +VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP, bool, true, + "Enable use of descriptor map in prespecializations library.") + +VARIABLE(SWIFT_DEBUG_VALIDATE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP, bool, false, + "Validate results from the prespecializations map descriptor map by " + "comparing to a full scan.") VARIABLE(SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH, string, "", "A path to a prespecializations library to use at runtime. In order " diff --git a/stdlib/public/runtime/LibPrespecialized.cpp b/stdlib/public/runtime/LibPrespecialized.cpp index 7f9ff22050f4c..82e7e30999122 100644 --- a/stdlib/public/runtime/LibPrespecialized.cpp +++ b/stdlib/public/runtime/LibPrespecialized.cpp @@ -35,8 +35,6 @@ using namespace swift; -static std::atomic disablePrespecializedMetadata = false; - static bool prespecializedLoggingEnabled = false; #define LOG(fmt, ...) \ @@ -45,6 +43,8 @@ static bool prespecializedLoggingEnabled = false; fprintf(stderr, "Prespecializations library: " fmt "\n", __VA_ARGS__); \ } while (0) +#define LOG0(string) LOG("%s", string) + static bool environmentProcessListContainsProcess(const char *list, const char *progname) { auto prognameLen = strlen(progname); @@ -110,68 +110,6 @@ static bool isThisProcessEnabled(const LibPrespecializedData *data) { return true; } -static const LibPrespecializedData *findLibPrespecialized() { - if (!runtime::environment::SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED()) { - LOG("Disabling, SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED = %d", - runtime::environment::SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED()); - return nullptr; - } - - const void *dataPtr = nullptr; -#if USE_DLOPEN - auto path = runtime::environment::SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH(); - if (path && path[0]) { - // Use RTLD_NOLOAD to avoid actually loading the library. We just want to - // find it if it has already been loaded by other means, such as - // DYLD_INSERT_LIBRARIES. - void *handle = dlopen(path, RTLD_LAZY | RTLD_NOLOAD); - if (!handle) { - swift::warning(0, "Failed to load prespecializations library: %s\n", - dlerror()); - return nullptr; - } - - dataPtr = dlsym(handle, LIB_PRESPECIALIZED_TOP_LEVEL_SYMBOL_NAME); - LOG("Loaded custom library from %s, found dataPtr %p", path, dataPtr); - } -#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED - else if (SWIFT_RUNTIME_WEAK_CHECK(_dyld_get_swift_prespecialized_data)) { - // Disable the prespecializations library if anything in the shared cache is - // overridden. Eventually we want to be cleverer and only disable the - // prespecializations that have been invalidated, but we'll start with the - // simplest approach. - if (!dyld_shared_cache_some_image_overridden()) { - dataPtr = SWIFT_RUNTIME_WEAK_USE(_dyld_get_swift_prespecialized_data()); - LOG("Got dataPtr %p from _dyld_get_swift_prespecialized_data", dataPtr); - } else { - LOG("Not calling _dyld_get_swift_prespecialized_data " - "dyld_shared_cache_some_image_overridden = %d", - dyld_shared_cache_some_image_overridden()); - } - } -#endif -#endif - - if (!dataPtr) - return nullptr; - - auto *data = - reinterpret_cast *>(dataPtr); - if (data->majorVersion != - LibPrespecializedData::currentMajorVersion) { - LOG("Unknown major version %" PRIu32 ", disabling", data->majorVersion); - return nullptr; - } - - if (!isThisProcessEnabled(data)) - return nullptr; - - LOG("Returning data %p, major version %" PRIu32 " minor %" PRIu32, data, - data->majorVersion, data->minorVersion); - - return data; -} - struct LibPrespecializedState { struct AddressRange { uintptr_t start, end; @@ -182,6 +120,7 @@ struct LibPrespecializedState { }; enum class MapConfiguration { + Unset, UseNameKeyedMap, UsePointerKeyedMap, UsePointerKeyedMapDebugMode, @@ -189,9 +128,10 @@ struct LibPrespecializedState { }; const LibPrespecializedData *data; - MapConfiguration mapConfiguration; + std::atomic mapConfiguration = MapConfiguration::Unset; AddressRange sharedCacheRange{0, 0}; AddressRange metadataAllocatorInitialPoolRange{0, 0}; + bool descriptorMapEnabled; LibPrespecializedState() { prespecializedLoggingEnabled = @@ -211,8 +151,41 @@ struct LibPrespecializedState { metadataAllocatorInitialPoolRange.start + initialPoolLength; #endif - // Must do this after the shared cache range has been retrieved. - mapConfiguration = computeMapConfiguration(data); + // Compute our map configuration if it hasn't already been set. We must do + // this after the shared cache range has been retrieved, because the map + // configuration can be different depending on whether the map is in the + // shared cache. + if (mapConfiguration.load(std::memory_order_relaxed) == + MapConfiguration::Unset) + mapConfiguration.store(computeMapConfiguration(data), + std::memory_order_relaxed); + + if (data) { + descriptorMapEnabled = + data->getOptionFlags() & + LibPrespecializedData::OptionFlagDescriptorMapDefaultOn; + LOG("Setting descriptorMapEnabled=%s from the option flags.", + descriptorMapEnabled ? "true" : "false"); + } + + if (runtime::environment:: + SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP_isSet()) { + descriptorMapEnabled = runtime::environment:: + SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP(); + LOG("Setting descriptorMapEnabled=%s from " + "SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP.", + descriptorMapEnabled ? "true" : "false"); + } else { +#if HAS_OS_FEATURE + if (os_feature_enabled_simple(Swift, togglePrespecializationDescriptorMap, + false)) { + descriptorMapEnabled = !descriptorMapEnabled; + LOG("Toggling descriptorMapEnabled to %s " + "togglePrespecializationDescriptorMap is set.", + descriptorMapEnabled ? "true" : "false"); + } +#endif + } } MapConfiguration @@ -221,6 +194,13 @@ struct LibPrespecializedState { if (!data) return MapConfiguration::Disabled; + if (!runtime::environment:: + SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_METADATA()) { + LOG0("Disabling metadata, SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_METADATA " + "is false."); + return MapConfiguration::Disabled; + } + auto nameKeyedMap = data->getMetadataMap(); auto pointerKeyedMap = data->getPointerKeyedMetadataMap(); @@ -265,14 +245,75 @@ struct LibPrespecializedState { } return MapConfiguration::UseNameKeyedMap; } + + const LibPrespecializedData *findLibPrespecialized() { + const void *dataPtr = nullptr; +#if USE_DLOPEN + auto path = runtime::environment::SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH(); + if (path && path[0]) { + // Use RTLD_NOLOAD to avoid actually loading the library. We just want to + // find it if it has already been loaded by other means, such as + // DYLD_INSERT_LIBRARIES. + void *handle = dlopen(path, RTLD_LAZY | RTLD_NOLOAD); + if (!handle) { + swift::warning(0, "Failed to load prespecializations library: %s\n", + dlerror()); + return nullptr; + } + + dataPtr = dlsym(handle, LIB_PRESPECIALIZED_TOP_LEVEL_SYMBOL_NAME); + LOG("Loaded custom library from %s, found dataPtr %p", path, dataPtr); + } +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + else if (SWIFT_RUNTIME_WEAK_CHECK(_dyld_get_swift_prespecialized_data)) { + dataPtr = SWIFT_RUNTIME_WEAK_USE(_dyld_get_swift_prespecialized_data()); + LOG("Got dataPtr %p from _dyld_get_swift_prespecialized_data", dataPtr); + + // Disable the prespecialized metadata if anything in the shared cache is + // overridden. Eventually we want to be cleverer and only disable the + // prespecializations that have been invalidated, but we'll start with the + // simplest approach. + if (dyld_shared_cache_some_image_overridden()) { + mapConfiguration.store(MapConfiguration::Disabled, + std::memory_order_release); + LOG("Disabling prespecialized metadata, " + "dyld_shared_cache_some_image_overridden = %d", + dyld_shared_cache_some_image_overridden()); + } + } +#endif +#endif + + LOG("Returning data pointer %p", dataPtr); + + if (!dataPtr) + return nullptr; + + auto *data = + reinterpret_cast *>(dataPtr); + if (data->majorVersion != + LibPrespecializedData::currentMajorVersion) { + LOG("Unknown major version %" PRIu32 ", disabling", data->majorVersion); + return nullptr; + } + + if (!isThisProcessEnabled(data)) + return nullptr; + + LOG("Returning data %p, major version %" PRIu32 " minor %" PRIu32, data, + data->majorVersion, data->minorVersion); + LOG(" optionFlags=%#zx", data->getOptionFlags()); + LOG(" metadataMap=%p", data->getMetadataMap()); + LOG(" disabledProcessTable=%p", data->getDisabledProcessesTable()); + LOG(" pointerKeyedMetadataMap=%p", data->getPointerKeyedMetadataMap()); + LOG(" descriptorMap=%p", data->getDescriptorMap()); + + return data; + } }; static Lazy LibPrespecialized; -const LibPrespecializedData *swift::getLibPrespecializedData() { - return SWIFT_LAZY_CONSTANT(findLibPrespecialized()); -} - // Returns true if the type has any arguments that aren't plain types (packs or // unknown kinds). static bool @@ -310,17 +351,27 @@ isPotentialPrespecializedPointer(const LibPrespecializedState &state, return true; } -static bool disableForValidation = false; +static bool isDescriptorLoaded(const void *descriptor, uint16_t imageIndex) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + return _dyld_is_preoptimized_objc_image_loaded(imageIndex); +#else + // If we're not using the dyld SPI, then we're working with a test dylib, and + // a test dylib can't have pointers to unloaded dylibs. + return true; +#endif +} void swift::libPrespecializedImageLoaded() { - #if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED // A newly loaded image might have caused us to load images that are // overriding images in the shared cache. If we do that, turn off // prespecialized metadata. if (dyld_shared_cache_some_image_overridden()) - disablePrespecializedMetadata.store(true, std::memory_order_release); - #endif + LibPrespecialized.get().mapConfiguration.store( + LibPrespecializedState::MapConfiguration::Disabled, + std::memory_order_release); +#endif } static Metadata * @@ -476,13 +527,13 @@ static Metadata *getMetadataFromPointerKeyedMapDebugMode( Metadata * swift::getLibPrespecializedMetadata(const TypeContextDescriptor *description, const void *const *arguments) { - if (SWIFT_UNLIKELY(disableForValidation || disablePrespecializedMetadata.load( - std::memory_order_acquire))) - return nullptr; - auto &state = LibPrespecialized.get(); switch (state.mapConfiguration) { + case LibPrespecializedState::MapConfiguration::Unset: + assert(false && + "Map configuration should never be unset after initialization."); + return nullptr; case LibPrespecializedState::MapConfiguration::Disabled: return nullptr; case LibPrespecializedState::MapConfiguration::UseNameKeyedMap: @@ -495,13 +546,125 @@ swift::getLibPrespecializedMetadata(const TypeContextDescriptor *description, } } +std::pair +swift::getLibPrespecializedTypeDescriptor(Demangle::NodePointer node) { + auto &state = LibPrespecialized.get(); + + // Retrieve the map and return immediately if we don't have it. + auto *data = state.data; + if (!data) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + + if (!state.descriptorMapEnabled) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + + auto *descriptorMap = data->getDescriptorMap(); + if (!descriptorMap) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + + // Demangler and resolver for subsequent mangling operations. + StackAllocatedDemangler<4096> dem; + ExpandResolvedSymbolicReferences resolver{dem}; + + if (SWIFT_UNLIKELY(prespecializedLoggingEnabled)) { + auto mangling = Demangle::mangleNode(node, resolver, dem); + if (!mangling.isSuccess()) { + LOG("Failed to build demangling for node %p.", node); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + auto mangled = mangling.result(); + LOG("Looking up descriptor named '%.*s'.", (int)mangled.size(), + mangled.data()); + } + + // Get the simplified mangling that we use as the map's key. + auto simplifiedNode = buildSimplifiedDescriptorDemangling(node, dem); + if (!simplifiedNode) { + LOG("Failed to build simplified mangling for node %p.", node); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + auto simplifiedMangling = Demangle::mangleNode(simplifiedNode, resolver, dem); + if (!simplifiedMangling.isSuccess()) { + LOG("Failed to build demangling for simplified node %p.\n", node); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + // The map key is the simplified mangled name. + auto key = simplifiedMangling.result(); + + // Track how many descriptors we checked and how many were actually loaded, + // for logging. + unsigned numDescriptorsChecked = 0; + unsigned numDescriptorsLoaded = 0; + + // A descriptor is a match if it's actually loaded, and if it matches the node + // we're looking up. + auto isMatch = [&](auto pointers) { + auto *descriptor = *pointers.first; + uint16_t libraryIndex = *pointers.second; + + numDescriptorsChecked++; + + if (!isDescriptorLoaded(descriptor, libraryIndex)) + return false; + + numDescriptorsLoaded++; + + return _contextDescriptorMatchesMangling( + (const TypeContextDescriptor *)descriptor, node); + }; + + // Perform the lookup. + auto isNull = [](auto pointers) { return *pointers.first == nullptr; }; + auto found = descriptorMap->find(key.data(), key.size(), isMatch, isNull); + + LOG("Hash table lookup checked %u loaded entries, %u total entries.", + numDescriptorsLoaded, numDescriptorsChecked); + + // The pointers in `found` are pointers to the map entries, and should always + // be non-NULL. The only condition that returns NULL is if the map has no + // entries where `isMatch` or `isNull` return true, and the map should always + // have at least one NULL entry. + assert(found.first); + if (!found.first) { + LOG("Descriptor table lookup of '%.*s' returned NULL pointer to descriptor " + "pointer.", + (int)key.size(), key.data()); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + auto *foundDescriptor = *found.first; + + if (!foundDescriptor) { + LOG("Did not find descriptor for key '%.*s'.", (int)key.size(), key.data()); + + // This result is definitive if the descriptor map is comprehensive. If the + // map is not comprehensive, return NonDefinitiveNotFound to tell the caller + // that it needs to perform a full search. + if (data->getOptionFlags() & + LibPrespecializedData< + InProcess>::OptionFlagDescriptorMapNotComprehensive) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + return {LibPrespecializedLookupResult::DefinitiveNotFound, nullptr}; + } + + LOG("Found descriptor %p for key '%.*s'.", foundDescriptor, (int)key.size(), + key.data()); + return {LibPrespecializedLookupResult::Found, + (const TypeContextDescriptor *)foundDescriptor}; +} + void _swift_validatePrespecializedMetadata() { - auto *data = getLibPrespecializedData(); + auto *data = LibPrespecialized.get().data; if (!data) { return; } - disableForValidation = true; + LibPrespecialized.get().mapConfiguration.store( + LibPrespecializedState::MapConfiguration::Disabled, + std::memory_order_release); unsigned validated = 0; unsigned failed = 0; diff --git a/stdlib/public/runtime/MetadataLookup.cpp b/stdlib/public/runtime/MetadataLookup.cpp index 8c252da96caa7..a199b4663534e 100644 --- a/stdlib/public/runtime/MetadataLookup.cpp +++ b/stdlib/public/runtime/MetadataLookup.cpp @@ -57,6 +57,10 @@ using namespace reflection; #include #endif +#if __has_include() +#include +#endif + /// A Demangler suitable for resolving runtime type metadata strings. template class DemanglerForRuntimeTypeResolution : public Base { @@ -325,14 +329,38 @@ namespace { }; } // end anonymous namespace +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED +struct SharedCacheInfoState { + uintptr_t dyldSharedCacheStart; + uintptr_t dyldSharedCacheEnd; + + bool inSharedCache(const void *ptr) { + auto uintPtr = reinterpret_cast(ptr); + return dyldSharedCacheStart <= uintPtr && uintPtr < dyldSharedCacheEnd; + } + + SharedCacheInfoState() { + size_t length; + dyldSharedCacheStart = (uintptr_t)_dyld_get_shared_cache_range(&length); + dyldSharedCacheEnd = + dyldSharedCacheStart ? dyldSharedCacheStart + length : 0; + } +}; + +static Lazy SharedCacheInfo; +#endif + struct TypeMetadataPrivateState { ConcurrentReadableHashMap NominalCache; ConcurrentReadableArray SectionsToScan; - + +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + ConcurrentReadableArray SharedCacheSectionsToScan; +#endif + TypeMetadataPrivateState() { initializeTypeMetadataRecordLookup(); } - }; static Lazy TypeMetadataRecords; @@ -341,6 +369,12 @@ static void _registerTypeMetadataRecords(TypeMetadataPrivateState &T, const TypeMetadataRecord *begin, const TypeMetadataRecord *end) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + if (SharedCacheInfo.get().inSharedCache(begin)) { + T.SharedCacheSectionsToScan.push_back(TypeMetadataSection{begin, end}); + return; + } +#endif T.SectionsToScan.push_back(TypeMetadataSection{begin, end}); } @@ -775,6 +809,111 @@ swift::_contextDescriptorMatchesMangling(const ContextDescriptor *context, return true; } +// Helper functions to allow _searchTypeMetadataRecordsInSections to work with +// both type and protocol records. +static const ContextDescriptor * +getContextDescriptor(const TypeMetadataRecord &record) { + return record.getContextDescriptor(); +} + +static const ContextDescriptor * +getContextDescriptor(const ProtocolRecord &record) { + return record.Protocol.getPointer(); +} + +// Perform a linear scan of the given records section, searching for a +// descriptor that matches the mangling passed in `node`. `sectionsToScan` is a +// ConcurrentReadableHashMap containing sections of type/protocol records. +template +static const ContextDescriptor * +_searchTypeMetadataRecordsInSections(SectionsContainer §ionsToScan, + Demangle::NodePointer node) { + for (auto §ion : sectionsToScan.snapshot()) { + for (const auto &record : section) { + if (auto context = getContextDescriptor(record)) { + if (_contextDescriptorMatchesMangling(context, node)) { + return context; + } + } + } + } + + return nullptr; +} + +// Search for a context descriptor matching the mangling passed in `node`. +// `state` is `TypeMetadataPrivateState` or `ProtocolMetadataPrivateState` and +// the search will use the sections in those structures. `traceBegin` is a +// function returning a trace state which is called around the linear scans of +// type/protocol records. When available, the search will consult the +// LibPrespecialized table, and perform validation on the result when validation +// is enabled. +template +static const ContextDescriptor * +_searchForContextDescriptor(State &state, NodePointer node, + TraceBegin traceBegin) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + // Try LibPrespecialized first. + auto result = getLibPrespecializedTypeDescriptor(node); + + // Validate the result if requested. + if (SWIFT_UNLIKELY( + runtime::environment:: + SWIFT_DEBUG_VALIDATE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP())) { + // Only validate a definitive result. + if (result.first == LibPrespecializedLookupResult::Found || + result.first == LibPrespecializedLookupResult::DefinitiveNotFound) { + // Perform a scan of the shared cache sections and see if the result + // matches. + auto scanResult = _searchTypeMetadataRecordsInSections( + state.SharedCacheSectionsToScan, node); + + // Ignore a result that's outside the shared cache. This can happen for + // indirect descriptor records that get fixed up to point to a root. + if (SharedCacheInfo.get().inSharedCache(scanResult)) { + // We may find a different but equivalent context if they're not unique, + // as iteration order may be different between the two. Use + // equalContexts to compare distinct but equal non-unique contexts + // properly. + if (!equalContexts(result.second, scanResult)) { + auto tree = getNodeTreeAsString(node); + swift::fatalError( + 0, + "Searching for type descriptor, prespecialized descriptor map " + "returned %p, but scan returned %p. Node tree:\n%s", + result.second, scanResult, tree.c_str()); + } + } + } + } + + // If we found something, we're done, return it. + if (result.first == LibPrespecializedLookupResult::Found) { + assert(result.second); + return result.second; + } + + // If a negative result was not definitive, then we must search the shared + // cache sections. + if (result.first == LibPrespecializedLookupResult::NonDefinitiveNotFound) { + auto traceState = traceBegin(node); + auto descriptor = _searchTypeMetadataRecordsInSections( + state.SharedCacheSectionsToScan, node); + traceState.end(descriptor); + if (descriptor) + return descriptor; + } + + // If we didn't find anything in the shared cache, then search the rest. +#endif + + auto traceState = traceBegin(node); + auto foundDescriptor = + _searchTypeMetadataRecordsInSections(state.SectionsToScan, node); + traceState.end(foundDescriptor); + return foundDescriptor; +} + // returns the nominal type descriptor for the type named by typeName static const ContextDescriptor * _searchTypeMetadataRecords(TypeMetadataPrivateState &T, @@ -789,19 +928,8 @@ _searchTypeMetadataRecords(TypeMetadataPrivateState &T, return nullptr; #endif - auto traceState = runtime::trace::metadata_scan_begin(node); - - for (auto §ion : T.SectionsToScan.snapshot()) { - for (const auto &record : section) { - if (auto context = record.getContextDescriptor()) { - if (_contextDescriptorMatchesMangling(context, node)) { - return traceState.end(context); - } - } - } - } - - return nullptr; + return _searchForContextDescriptor(T, node, + runtime::trace::metadata_scan_begin); } #define DESCRIPTOR_MANGLING_SUFFIX_Structure Mn @@ -980,6 +1108,10 @@ namespace { ConcurrentReadableHashMap ProtocolCache; ConcurrentReadableArray SectionsToScan; +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + ConcurrentReadableArray SharedCacheSectionsToScan; +#endif + ProtocolMetadataPrivateState() { initializeProtocolLookup(); } @@ -992,6 +1124,12 @@ static void _registerProtocols(ProtocolMetadataPrivateState &C, const ProtocolRecord *begin, const ProtocolRecord *end) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + if (SharedCacheInfo.get().inSharedCache(begin)) { + C.SharedCacheSectionsToScan.push_back(ProtocolSection{begin, end}); + return; + } +#endif C.SectionsToScan.push_back(ProtocolSection{begin, end}); } @@ -1029,18 +1167,12 @@ void swift::swift_registerProtocols(const ProtocolRecord *begin, static const ProtocolDescriptor * _searchProtocolRecords(ProtocolMetadataPrivateState &C, NodePointer node) { - auto traceState = runtime::trace::protocol_scan_begin(node); - - for (auto §ion : C.SectionsToScan.snapshot()) { - for (const auto &record : section) { - if (auto protocol = record.Protocol.getPointer()) { - if (_contextDescriptorMatchesMangling(protocol, node)) - return traceState.end(protocol); - } - } - } - - return nullptr; + auto descriptor = + _searchForContextDescriptor(C, node, runtime::trace::protocol_scan_begin); + assert(!descriptor || + isa(descriptor) && + "Protocol record search found non-protocol descriptor."); + return reinterpret_cast(descriptor); } static const ProtocolDescriptor * diff --git a/unittests/runtime/PrebuiltStringMap.cpp b/unittests/runtime/PrebuiltStringMap.cpp index fb95a207e0fdb..28f4ad7b7d105 100644 --- a/unittests/runtime/PrebuiltStringMap.cpp +++ b/unittests/runtime/PrebuiltStringMap.cpp @@ -15,7 +15,7 @@ static bool stringIsNull(const char *str) { return str == nullptr; } -TEST(PrebuiltStringMapTest, basic) { +TEST(PrebuiltStringMapTest, PrebuiltStringMap) { auto testOnce = [&](unsigned testEntryCount) { std::vector> testVector; testVector.reserve(testEntryCount); @@ -32,6 +32,7 @@ TEST(PrebuiltStringMapTest, basic) { void *mapAllocation = calloc(1, Map::byteSize(mapSize)); Map *map = new (mapAllocation) Map(mapSize); + // Populate the map. for (auto &[key, value] : testVector) { const char *keyCStr = key.c_str(); auto *element = map->insert(keyCStr); @@ -41,6 +42,7 @@ TEST(PrebuiltStringMapTest, basic) { element->value = value; } + // Verify that we can find all the test values. for (auto &[key, value] : testVector) { const char *keyCStr = key.c_str(); auto *element = map->find(keyCStr); @@ -63,6 +65,11 @@ TEST(PrebuiltStringMapTest, basic) { EXPECT_EQ(element->value, value); } + // Verify that nonexistent keys are not found. + const char *nonexistentKey = "ceci n'est pas une clef"; + auto *element = map->find(nonexistentKey); + EXPECT_EQ(element, nullptr); + free(mapAllocation); }; @@ -70,3 +77,67 @@ TEST(PrebuiltStringMapTest, basic) { testOnce(100); testOnce(1000); } + +TEST(PrebuiltStringMapTest, PrebuiltAuxDataImplicitStringMap) { + auto testOnce = [&](unsigned testEntryCount) { + // Test a map containing positive integers, where the key is a string + // derived from the integer. The aux data is the negative of the integer. + using Map = swift::PrebuiltAuxDataImplicitStringMap; + + auto getKey = [](unsigned n) { + std::string key; + for (unsigned i = 0; i < n; i++) { + key += 'A' + (i % 26); + } + return key; + }; + + unsigned mapSize = testEntryCount * 4 / 3; + void *mapAllocation = calloc(1, Map::byteSize(mapSize)); + Map *map = new (mapAllocation) Map(mapSize); + + // Populate the map. + for (unsigned n = 0; n < testEntryCount; n++) { + auto key = getKey(n); + + auto isNull = [](auto pointers) { return *pointers.first == 0; }; + auto pointers = map->insert(key.c_str(), isNull); + EXPECT_NE(pointers.first, nullptr); + EXPECT_NE(pointers.second, nullptr); + + *pointers.first = n; + *pointers.second = -(int64_t)n; + } + + // Verify that we can find all the test values. + for (unsigned n = 0; n < testEntryCount; n++) { + auto key = getKey(n); + auto keyLength = key.size(); + + // Add some trash to the end to make sure the lookup doesn't look beyond + // the specified length. + key += "xyz"; + + auto isMatch = [n](auto pointers) { return *pointers.first == n; }; + auto isNull = [](auto pointers) { return *pointers.first == 0; }; + auto pointers = map->find(key.c_str(), keyLength, isMatch, isNull); + EXPECT_NE(pointers.first, nullptr); + EXPECT_NE(pointers.second, nullptr); + EXPECT_EQ(*pointers.first, n); + EXPECT_EQ(*pointers.second, -(int64_t)n); + } + + // Verfy a nonexistent value is not found. + const char *nonexistentKey = "ceci n'est pas une clef"; + auto isMatch = [](auto pointers) { return false; }; + auto isNull = [](auto pointers) { return *pointers.first == 0; }; + auto pointers = + map->find(nonexistentKey, strlen(nonexistentKey), isMatch, isNull); + EXPECT_EQ(*pointers.first, 0); + EXPECT_EQ(*pointers.second, 0); + }; + + testOnce(10); + testOnce(100); + testOnce(1000); +}