readonly mappedfile wrapper class, use_mapped_model_loading option

docs/how-to-use-and-FAQ/ncnn-load-model.md

@@ -19,8 +19,8 @@
 2. Never modify Net opt member after loading
 
 3. Most loading functions return 0 if success, except loading alexnet.param.bin and alexnet.bin from file memory, which returns the bytes consumed after loading
-    * int Net::load_param(const unsigned char*)
-    * int Net::load_model(const unsigned char*)
+    * size_t Net::load_param(const unsigned char*)
+    * size_t Net::load_model(const unsigned char*)
 
 4. It is recommended to load model from Android asset directly to avoid copying them to sdcard on Android platform
 

src/c_api.cpp

@@ -1541,12 +1541,12 @@ int ncnn_net_load_param_memory(ncnn_net_t net, const char* mem)
 #endif /* NCNN_STRING */
 #endif /* NCNN_STDIO */
 
-int ncnn_net_load_param_bin_memory(ncnn_net_t net, const unsigned char* mem)
+size_t ncnn_net_load_param_bin_memory(ncnn_net_t net, const unsigned char* mem)
 {
     return ((Net*)net->pthis)->load_param(mem);
 }
 
-int ncnn_net_load_model_memory(ncnn_net_t net, const unsigned char* mem)
+size_t ncnn_net_load_model_memory(ncnn_net_t net, const unsigned char* mem)
 {
     return ((Net*)net->pthis)->load_model(mem);
 }

src/c_api.h

@@ -324,8 +324,8 @@ NCNN_EXPORT int ncnn_net_load_model_w(ncnn_net_t net, const wchar_t* path);
 NCNN_EXPORT int ncnn_net_load_param_memory(ncnn_net_t net, const char* mem);
 #endif /* NCNN_STRING */
 #endif /* NCNN_STDIO */
-NCNN_EXPORT int ncnn_net_load_param_bin_memory(ncnn_net_t net, const unsigned char* mem);
-NCNN_EXPORT int ncnn_net_load_model_memory(ncnn_net_t net, const unsigned char* mem);
+NCNN_EXPORT size_t ncnn_net_load_param_bin_memory(ncnn_net_t net, const unsigned char* mem);
+NCNN_EXPORT size_t ncnn_net_load_model_memory(ncnn_net_t net, const unsigned char* mem);
 
 #if NCNN_STRING
 NCNN_EXPORT int ncnn_net_load_param_datareader(ncnn_net_t net, const ncnn_datareader_t dr);

src/net.cpp

@@ -69,6 +69,10 @@ class NetPrivate
     PoolAllocator* local_blob_allocator;
     PoolAllocator* local_workspace_allocator;
 
+#if defined _WIN32 || __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+    MappedFile mapped_model_file;
+#endif
+
 #if NCNN_VULKAN
     const VulkanDevice* vkdev;
 
@@ -1815,6 +1819,29 @@ int Net::load_model(FILE* fp)
 
 int Net::load_model(const char* modelpath)
 {
+#if defined _WIN32 || __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+    if (opt.use_mapped_model_loading)
+    {
+        int ret = d->mapped_model_file.open(modelpath);
+        if (ret == 0)
+        {
+            const void* ptr = d->mapped_model_file.mapped_ptr();
+            const size_t size = d->mapped_model_file.size();
+            size_t consumed = load_model((const unsigned char*)ptr);
+            if (consumed != size)
+            {
+                NCNN_LOGE("mapped_file consumed %zu != %zu", consumed, size);
+                d->mapped_model_file.close();
+                return -1;
+            }
+
+            return 0;
+        }
+
+        // fallback to regular file loading
+    }
+#endif // defined _WIN32 || __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+
     FILE* fp = fopen(modelpath, "rb");
     if (!fp)
     {
@@ -1830,6 +1857,29 @@ int Net::load_model(const char* modelpath)
 #if _WIN32
 int Net::load_model(const wchar_t* modelpath)
 {
+#if defined _WIN32 || __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+    if (opt.use_mapped_model_loading)
+    {
+        int ret = d->mapped_model_file.open(modelpath);
+        if (ret == 0)
+        {
+            const void* ptr = d->mapped_model_file.mapped_ptr();
+            const size_t size = d->mapped_model_file.size();
+            size_t consumed = load_model((const unsigned char*)ptr);
+            if (consumed != size)
+            {
+                NCNN_LOGE("mapped_file consumed %zu != %zu", consumed, size);
+                d->mapped_model_file.close();
+                return -1;
+            }
+
+            return 0;
+        }
+
+        // fallback to regular file loading
+    }
+#endif // defined _WIN32 || __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+
     FILE* fp = _wfopen(modelpath, L"rb");
     if (!fp)
     {
@@ -1844,20 +1894,20 @@ int Net::load_model(const wchar_t* modelpath)
 #endif
 #endif // NCNN_STDIO
 
-int Net::load_param(const unsigned char* _mem)
+size_t Net::load_param(const unsigned char* _mem)
 {
     const unsigned char* mem = _mem;
     DataReaderFromMemory dr(mem);
     load_param_bin(dr);
-    return static_cast<int>(mem - _mem);
+    return (size_t)(mem - _mem);
 }
 
-int Net::load_model(const unsigned char* _mem)
+size_t Net::load_model(const unsigned char* _mem)
 {
     const unsigned char* mem = _mem;
     DataReaderFromMemory dr(mem);
     load_model(dr);
-    return static_cast<int>(mem - _mem);
+    return (size_t)(mem - _mem);
 }
 
 #if NCNN_PLATFORM_API

src/net.h

@@ -98,14 +98,14 @@ class NCNN_EXPORT Net
     // load network structure from external memory
     // memory pointer must be 32-bit aligned
     // return bytes consumed
-    int load_param(const unsigned char* mem);
+    size_t load_param(const unsigned char* mem);
 
     // reference network weight data from external memory
     // weight data is not copied but referenced
     // so external memory should be retained when used
     // memory pointer must be 32-bit aligned
     // return bytes consumed
-    int load_model(const unsigned char* mem);
+    size_t load_model(const unsigned char* mem);
 
 #if NCNN_PLATFORM_API
 #if __ANDROID_API__ >= 9

src/option.cpp

@@ -52,6 +52,9 @@ Option::Option()
     use_weights_in_host_memory = false;
 
     flush_denormals = 3;
+    use_reserved_2f = false;
+    use_reserved_3f = false;
+    use_mapped_model_loading = false;
 
     use_local_pool_allocator = true;
 

src/option.h

@@ -118,7 +118,11 @@ class NCNN_EXPORT Option
     // 1 = DAZ ON , FTZ OFF
     // 2 = DAZ OFF, FTZ ON
     // 3 = DAZ ON,  FTZ ON
-    int flush_denormals;
+    unsigned char flush_denormals;
+
+    bool use_reserved_2f;
+    bool use_reserved_3f;
+    bool use_mapped_model_loading;
 
     bool use_local_pool_allocator;
 

src/platform.h.in

@@ -66,10 +66,19 @@
 
 #ifdef __cplusplus
 
-#if NCNN_THREADS
 #if defined _WIN32
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
+#elif defined __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+
+#if NCNN_THREADS
+#if defined _WIN32
 #include <process.h>
 #else
 #include <pthread.h>
@@ -82,6 +91,8 @@
 #endif
 #endif // __ANDROID_API__ >= 26
 
+#include <stddef.h>
+
 namespace ncnn {
 
 #if NCNN_THREADS
@@ -282,6 +293,117 @@ private:
     Mutex& mutex;
 };
 
+#if defined _WIN32
+class NCNN_EXPORT MappedFile
+{
+public:
+    MappedFile() { ptr = 0; _size = 0; file = INVALID_HANDLE_VALUE; mapping = 0; }
+    ~MappedFile() { close(); }
+    int open(const char* path)
+    {
+        close();
+
+        file = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+        if (file == INVALID_HANDLE_VALUE) return -1;
+
+        LARGE_INTEGER liSize;
+        if (!GetFileSizeEx(file, &liSize)) { close(); return -1; }
+
+        _size = (size_t)liSize.QuadPart;
+        if (_size == 0) { close(); return -1; }
+
+        mapping = CreateFileMapping(file, NULL, PAGE_READONLY, 0, 0, NULL);
+        if (!mapping) { close(); return -1; }
+
+        ptr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
+        if (!ptr) { close(); return -1; }
+        return 0;
+    }
+    int open(const wchar_t* path)
+    {
+        close();
+
+        file = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+        if (file == INVALID_HANDLE_VALUE) return -1;
+
+        LARGE_INTEGER liSize;
+        if (!GetFileSizeEx(file, &liSize)) { close(); return -1; }
+
+        _size = (size_t)liSize.QuadPart;
+        if (_size == 0) { close(); return -1; }
+
+        mapping = CreateFileMapping(file, NULL, PAGE_READONLY, 0, 0, NULL);
+        if (!mapping) { close(); return -1; }
+
+        ptr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
+        if (!ptr) { close(); return -1; }
+        return 0;
+    }
+    void close()
+    {
+        if (ptr) { UnmapViewOfFile(ptr); ptr = 0; }
+        if (mapping) { CloseHandle(mapping); mapping = 0; }
+        if (file != INVALID_HANDLE_VALUE) { CloseHandle(file); file = INVALID_HANDLE_VALUE; }
+        _size = 0;
+    }
+    const void* mapped_ptr() const { return ptr; }
+    size_t size() const { return _size; }
+private:
+    void* ptr;
+    size_t _size;
+    HANDLE file;
+    HANDLE mapping;
+};
+#elif defined __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+class NCNN_EXPORT MappedFile
+{
+public:
+    MappedFile() { ptr = 0; _size = 0; fd = -1; }
+    ~MappedFile() { close(); }
+    int open(const char* path)
+    {
+        close();
+
+        fd = ::open(path, O_RDONLY);
+        if (fd < 0) return -1;
+
+        struct stat st;
+        if (fstat(fd, &st) < 0) { close(); return -1; }
+
+        _size = (size_t)st.st_size;
+        if (_size == 0) { close(); return -1; }
+
+        ptr = mmap(NULL, _size, PROT_READ, MAP_PRIVATE, fd, 0);
+        if (ptr == MAP_FAILED) { close(); return -1; }
+        return 0;
+    }
+    void close()
+    {
+        if (ptr && ptr != MAP_FAILED) { munmap(ptr, _size); }
+        ptr = 0;
+        if (fd >= 0) { ::close(fd); fd = -1; }
+        _size = 0;
+    }
+    const void* mapped_ptr() const { return ptr; }
+    size_t size() const { return _size; }
+private:
+    void* ptr;
+    size_t _size;
+    int fd;
+};
+#else // defined _WIN32 || __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+class NCNN_EXPORT MappedFile
+{
+public:
+    MappedFile() {}
+    ~MappedFile() {}
+    int open(const char* /*path*/) { return -1; }
+    void close() {}
+    const void* mapped_ptr() const { return 0; }
+    size_t size() const { return 0; }
+};
+#endif // defined _WIN32 || __ANDROID__ || defined __OHOS__ || defined __linux__ || __APPLE__
+
 static inline void swap_endianness_16(void* x)
 {
     unsigned char* xx = (unsigned char*)x;

tests/test_squeezenet.cpp

@@ -436,6 +436,9 @@ int main()
     opts[2].use_bf16_storage = false; // FIXME enable me
     opts[2].blob_allocator = &g_blob_pool_allocator;
     opts[2].workspace_allocator = &g_workspace_pool_allocator;
+    opts[2].use_weights_in_host_memory = true;
+    opts[2].use_mapped_model_loading = true;
+    opts[2].use_local_pool_allocator = false;
 
     opts[3].use_packing_layout = true;
     opts[3].use_fp16_packed = true;