Add ublk-based block device frontend

The existing TCMU frontend requires a single resident daemon for all
devices on a host: netlink only notifies the registered handler
process, so one crash takes down every overlaybd disk, and the code
carries several netlink workarounds as the price of that architecture.

Introduce overlaybd-ublk, an alternative frontend based on the
kernel's io_uring-backed ublk driver, with no SCSI overhead and no
single point of failure: one process serves exactly one device, so
device create/delete collapses into process start/stop and failures
are isolated to a single disk.

* add/del/list CLI following ublk ecosystem conventions; add returns
  only after the device is usable and prints the device path
* per-queue event loop driven by photon: ring fd polled via epoll,
  CQEs reaped with the non-blocking libublksrv API, IO served by
  coroutines calling ImageFile directly
* IO mapping mirrors the TCMU frontend (READ/WRITE/FLUSH/DISCARD);
  writable devices honestly advertise a volatile cache, read-only
  devices do not
* works on kernels with partial io_uring feature sets: queue setup
  retries without IORING_SETUP_COOP_TASKRUN, device deletion prefers
  DEL_DEV_ASYNC and falls back to a bounded sync del
* libublksrv v1.7 and liburing 2.8 are fetched at configure time and
  statically linked (dual MIT/LGPL, used under MIT); building needs
  autotools, opt out with -DBUILD_UBLK_FRONTEND=off
* verified end-to-end on ublk-capable kernels: startup sequence, idle
  wakeup, full-queue fio pressure with crc32c verify, FLUSH path,
  add/del lifecycle, and read-only semantics; 13 unit tests cover the
  IO mapping and CLI layers

Signed-off-by: haolianglh <haolianglh@sina.com>

Author: haolianglh

.github/workflows/release/build.sh

@@ -73,11 +73,13 @@ elif [[ ${OS} =~ "centos" ]]; then
     fi
 
     yum install -y epel-release libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel
-    yum install -y rpm-build make git wget sudo autoconf automake libtool
+    # pkgconfig: ublksrv's configure uses PKG_CHECK_MODULES (needs pkg.m4 at autoreconf time)
+    yum install -y rpm-build make git wget sudo autoconf automake libtool pkgconfig
     yum install --skip-broken -y libzstd-static gcc gcc-c++ binutils libzstd-devel
 elif [[ ${OS} =~ "mariner" ]]; then
     yum install -y libaio-devel libcurl-devel openssl-devel libnl3-devel e2fsprogs-devel glibc-devel libzstd-devel binutils ca-certificates-microsoft build-essential
-    yum install -y rpm-build make git wget sudo tar gcc gcc-c++ autoconf automake libtool
+    # pkg-config: ublksrv's configure uses PKG_CHECK_MODULES (needs pkg.m4 at autoreconf time)
+    yum install -y rpm-build make git wget sudo tar gcc gcc-c++ autoconf automake libtool pkg-config
 
     DISTRO=${OS/:/.}
     PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.${DISTRO}"

CMake/Findliburing.cmake

@@ -0,0 +1,72 @@
+# liburing (hard dependency of ublksrv).
+# Distro liburing packages are commonly missing or too old (>= 2.2 required)
+# and behavior differs across versions, so like every other dependency of
+# this project it is always fetched at a pinned version and statically
+# linked -- never taken from the system.
+# FetchContent only downloads; the build uses liburing's own configure/make.
+include(FetchContent)
+set(FETCHCONTENT_QUIET false)
+
+# CMake 3.30+ deprecates single-argument FetchContent_Populate (CMP0169);
+# explicitly keep the old behavior
+if(POLICY CMP0169)
+  cmake_policy(SET CMP0169 OLD)
+endif()
+
+FetchContent_Declare(
+  liburing
+  GIT_REPOSITORY https://github.com/axboe/liburing.git
+  GIT_TAG        liburing-2.8
+)
+
+# download only, no add_subdirectory (liburing is a plain Makefile project)
+FetchContent_GetProperties(liburing)
+if(NOT liburing_POPULATED)
+  FetchContent_Populate(liburing)
+endif()
+
+if(NOT TARGET liburing_build)
+  # OUTPUT points at the final artifact => incremental build, skipped once present
+  add_custom_command(
+    OUTPUT ${liburing_SOURCE_DIR}/src/liburing.a
+    WORKING_DIRECTORY ${liburing_SOURCE_DIR}
+    COMMAND ./configure
+    COMMAND make -C src -j
+    COMMENT "Building liburing with its own configure/make"
+    VERBATIM
+  )
+  add_custom_target(liburing_build DEPENDS ${liburing_SOURCE_DIR}/src/liburing.a)
+endif()
+
+set(LIBURING_LIBRARIES    ${liburing_SOURCE_DIR}/src/liburing.a)
+set(LIBURING_INCLUDE_DIRS ${liburing_SOURCE_DIR}/src/include)
+
+# Generate an in-build-tree liburing.pc for ublksrv's configure check
+# PKG_CHECK_MODULES([LIBURING], [liburing >= 2.2]) (the official
+# build_with_liburing_src flow relies on pkg-config metadata)
+set(LIBURING_PC_DIR ${CMAKE_BINARY_DIR}/liburing-pkgconfig)
+file(WRITE ${LIBURING_PC_DIR}/liburing.pc
+"prefix=${liburing_SOURCE_DIR}
+libdir=${liburing_SOURCE_DIR}/src
+includedir=${liburing_SOURCE_DIR}/src/include
+
+Name: liburing
+Version: 2.8
+Description: io_uring library
+Libs: -L\${libdir} -luring
+Cflags: -I\${includedir}
+")
+
+if(NOT TARGET liburing_static)
+  add_library(liburing_static STATIC IMPORTED GLOBAL)
+  set_target_properties(liburing_static PROPERTIES
+    IMPORTED_LOCATION             ${LIBURING_LIBRARIES}
+    INTERFACE_INCLUDE_DIRECTORIES ${LIBURING_INCLUDE_DIRS}
+  )
+endif()
+add_dependencies(liburing_static liburing_build)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(liburing DEFAULT_MSG LIBURING_LIBRARIES LIBURING_INCLUDE_DIRS)
+
+mark_as_advanced(LIBURING_LIBRARIES LIBURING_INCLUDE_DIRS)

CMake/Findublksrv.cmake

@@ -0,0 +1,78 @@
+# ublksrv (ublk userspace library; autotools project, no CMake).
+# Only the lib/ directory (pure-C libublksrv) is built:
+#   - avoids the C++20/-fcoroutines requirement of the ublk CLI tool;
+#   - libnfs/libiscsi/gnutls only serve its bundled targets, all disabled.
+# liburing is built from source by Findliburing.cmake and fed to configure
+# via pkg-config/CFLAGS/LDFLAGS (the official build_with_liburing_src flow).
+# License: lib/ + include/ublksrv.h are dual MIT/LGPL; linked under MIT.
+include(FetchContent)
+set(FETCHCONTENT_QUIET false)
+
+if(POLICY CMP0169)
+  cmake_policy(SET CMP0169 OLD)
+endif()
+
+# the autotools build needs extra host tools; fail early with a clear message
+find_program(AUTORECONF_EXECUTABLE autoreconf)
+find_program(LIBTOOLIZE_EXECUTABLE libtoolize)
+find_program(AUTOMAKE_EXECUTABLE automake)
+if(NOT AUTORECONF_EXECUTABLE OR NOT LIBTOOLIZE_EXECUTABLE OR NOT AUTOMAKE_EXECUTABLE)
+  message(FATAL_ERROR
+    "BUILD_UBLK_FRONTEND requires autotools (autoconf/automake/libtool) "
+    "to build ublksrv. Install them or configure with -DBUILD_UBLK_FRONTEND=off")
+endif()
+
+FetchContent_Declare(
+  ublksrv
+  GIT_REPOSITORY https://github.com/ublk-org/ublksrv.git
+  GIT_TAG        f6c643952d1cdc7f6460630638fe6b5454ca1c4d   # v1.7
+)
+
+FetchContent_GetProperties(ublksrv)
+if(NOT ublksrv_POPULATED)
+  FetchContent_Populate(ublksrv)
+endif()
+
+if(NOT TARGET libublksrv_build)
+  add_custom_command(
+    OUTPUT ${ublksrv_SOURCE_DIR}/lib/.libs/libublksrv.a
+    WORKING_DIRECTORY ${ublksrv_SOURCE_DIR}
+    COMMAND autoreconf -i
+    COMMAND ${CMAKE_COMMAND} -E env PKG_CONFIG_PATH=${LIBURING_PC_DIR}
+            ./configure
+            --without-libnfs --without-libiscsi --without-gnutls
+            # configure appends -fcoroutines whenever $CXX matches *g++*
+            # (GCC 10+ only, and only the ublk tool needs it, lib/ does not);
+            # use the name c++ to sidestep that match
+            CXX=c++
+            "CFLAGS=-I${LIBURING_INCLUDE_DIRS} -O2"
+            "CXXFLAGS=-I${LIBURING_INCLUDE_DIRS} -O2"
+            "LDFLAGS=-L${liburing_SOURCE_DIR}/src"
+    COMMAND make -C lib -j
+    DEPENDS ${LIBURING_LIBRARIES}
+    COMMENT "Building libublksrv (lib/ only) with autotools"
+    VERBATIM
+  )
+  add_custom_target(libublksrv_build DEPENDS ${ublksrv_SOURCE_DIR}/lib/.libs/libublksrv.a)
+  add_dependencies(libublksrv_build liburing_build)
+endif()
+
+set(UBLKSRV_LIBRARIES    ${ublksrv_SOURCE_DIR}/lib/.libs/libublksrv.a)
+set(UBLKSRV_INCLUDE_DIRS ${ublksrv_SOURCE_DIR}/include)
+
+if(NOT TARGET libublksrv_static)
+  add_library(libublksrv_static STATIC IMPORTED GLOBAL)
+  set_target_properties(libublksrv_static PROPERTIES
+    IMPORTED_LOCATION             ${UBLKSRV_LIBRARIES}
+    INTERFACE_INCLUDE_DIRECTORIES ${UBLKSRV_INCLUDE_DIRS}
+    # libublksrv.a references liburing symbols; propagate via INTERFACE so
+    # consumers get the correct link order automatically
+    INTERFACE_LINK_LIBRARIES      liburing_static
+  )
+endif()
+add_dependencies(libublksrv_static libublksrv_build)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ublksrv DEFAULT_MSG UBLKSRV_LIBRARIES UBLKSRV_INCLUDE_DIRS)
+
+mark_as_advanced(UBLKSRV_LIBRARIES UBLKSRV_INCLUDE_DIRS)

CMakeLists.txt

@@ -53,10 +53,17 @@ set(ENABLE_MIMIC_VDSO off)
 option(BUILD_CURL_FROM_SOURCE "Compile static libcurl" off)
 option(BUILD_STREAM_CONVERTOR "Build the stream convertor" on)
 option(ORIGIN_EXT2FS "Use original libext2fs" off)
+# Building it requires autotools on the build machine; disable with -DBUILD_UBLK_FRONTEND=off
+option(BUILD_UBLK_FRONTEND "Build the ublk frontend (overlaybd-ublk), requires autotools" on)
 
 find_package(photon REQUIRED)
 find_package(tcmu REQUIRED)
 
+if(BUILD_UBLK_FRONTEND)
+    find_package(liburing REQUIRED)
+    find_package(ublksrv REQUIRED)
+endif()
+
 if(BUILD_STREAM_CONVERTOR)
     find_package(yaml-cpp)
     if (NOT yaml-cpp_FOUND)

README.md

@@ -119,6 +119,42 @@ cmake -D ENABLE_QAT=1 ..
 
 For more information go to `overlaybd/src/overlaybd/zfile/README.md`.
 
+If you want to build the ublk frontend (`overlaybd-ublk`), which exposes an
+image as `/dev/ublkbN` without going through TCMU/SCSI. It is built by default
+(disable with `-D BUILD_UBLK_FRONTEND=off`); building it additionally requires
+`autoconf`, `automake` and `libtool` (liburing and libublksrv are fetched and
+built from source automatically).
+
+```bash
+cmake -D BUILD_UBLK_FRONTEND=on ..
+```
+
+Running it requires a kernel with the `ublk_drv` driver (mainline >= 6.0, or a
+distro kernel with ublk backported):
+
+```bash
+sudo modprobe ublk_drv
+sudo overlaybd-ublk add --config /path/to/config.v1.json   # prints /dev/ublkbN when ready
+sudo overlaybd-ublk list
+sudo overlaybd-ublk del -n 0
+```
+
+Unlike `overlaybd-tcmu`, one `overlaybd-ublk` process serves exactly one
+device; killing the daemon removes the device. Each device can write to its
+own log file via `add --log-path ...` (recommended when running multiple
+devices; daemons otherwise share the global log file and are distinguished
+by a `ublk-<pid>` tag).
+
+Notes:
+
+* Always unmount filesystems on `/dev/ublkbN` before `del` (or before stopping
+  the daemon): the daemon serves the device's IO, so tearing it down with a
+  mounted filesystem aborts in-flight journal writes.
+* Runtime verified on a 5.10 kernel with ublk backported (Alinux
+  5.10.134); a full regression on mainline 6.x kernels is still pending.
+  Known backport-kernel caveats: DISCARD is unavailable there, and `del`
+  takes ~2s instead of milliseconds.
+
 Finally, setup a systemd service for overlaybd-tcmu backstore.
 
 ```bash

src/CMakeLists.txt

@@ -75,6 +75,9 @@ if (NOT ORIGIN_EXT2FS)
 endif()
 
 add_subdirectory(tools)
+if (BUILD_UBLK_FRONTEND)
+    add_subdirectory(ublk)
+endif ()
 if (BUILD_TESTING)
     add_subdirectory(test)
 endif ()

src/ublk/CMakeLists.txt

@@ -0,0 +1,35 @@
+# overlaybd-ublk: the ublk block device frontend (one process serves one device)
+# Entered only when BUILD_UBLK_FRONTEND=on
+
+add_library(ublk_frontend_lib
+    io_dispatch.cpp
+    ublk_device.cpp
+    cli.cpp
+)
+target_include_directories(ublk_frontend_lib PUBLIC
+    ${PHOTON_INCLUDE_DIR}
+    ${RAPIDJSON_INCLUDE_DIRS}
+)
+target_link_libraries(ublk_frontend_lib
+    photon_static
+    overlaybd_image_lib
+    libublksrv_static      # IMPORTED target; include dirs and liburing propagate via INTERFACE
+)
+
+add_executable(overlaybd-ublk main.cpp)
+target_link_libraries(overlaybd-ublk
+    ublk_frontend_lib
+    photon_static
+    overlaybd_image_lib
+    libublksrv_static
+    ${CURL_LIBRARIES}
+    ${OPENSSL_SSL_LIBRARY}
+    ${OPENSSL_CRYPTO_LIBRARY}
+    ${AIO_LIBRARIES}
+)
+
+install(TARGETS overlaybd-ublk DESTINATION /opt/overlaybd/bin)
+
+if (BUILD_TESTING)
+    add_subdirectory(test)
+endif ()

src/ublk/cli.cpp

@@ -0,0 +1,63 @@
+/*
+   Copyright The Overlaybd Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+#include "cli.h"
+
+#include "../tools/CLI11.hpp"
+
+int ublk_parse_cli(int argc, char **argv, UblkCliCmd &cmd) {
+    CLI::App app{"overlaybd-ublk: expose an overlaybd image as /dev/ublkbN "
+                 "(one process serves one device)"};
+    app.require_subcommand(1);
+
+    auto *add = app.add_subcommand("add", "create a ublk device from an image config");
+    add->add_option("--config", cmd.opts.image_config_path,
+                    "overlaybd image config path (config.v1.json)")
+        ->required()
+        ->check(CLI::ExistingFile);
+    add->add_option("-n,--dev-id", cmd.opts.dev_id,
+                    "device id (default -1: allocated by kernel)");
+    add->add_option("--depth", cmd.opts.queue_depth, "queue depth")
+        ->default_val(128)
+        ->check(CLI::Range(1, 4096));
+    add->add_option("--service-config", cmd.opts.service_config_path,
+                    "global service config path (default /etc/overlaybd/overlaybd.json)");
+    add->add_option("--log-path", cmd.opts.log_path,
+                    "per-device log file (default: shared log from service config; "
+                    "recommended when running multiple devices)");
+    add->add_flag("--foreground", cmd.foreground, "run in foreground (no daemonize)");
+
+    auto *del = app.add_subcommand("del", "stop the daemon serving /dev/ublkbN");
+    del->add_option("-n,--dev-id", cmd.del_dev_id, "device id")->required();
+
+    app.add_subcommand("list", "list overlaybd-ublk devices on this host");
+
+    try {
+        app.parse(argc, argv);
+    } catch (const CLI::ParseError &e) {
+        // help returns 0, errors return non-zero; cmd.kind stays NONE in
+        // both cases so the caller just exits with this code
+        cmd.kind = UblkCliCmd::Kind::NONE;
+        return app.exit(e);
+    }
+
+    if (app.got_subcommand("add"))
+        cmd.kind = UblkCliCmd::Kind::ADD;
+    else if (app.got_subcommand("del"))
+        cmd.kind = UblkCliCmd::Kind::DEL;
+    else
+        cmd.kind = UblkCliCmd::Kind::LIST;
+    return 0;
+}

src/ublk/cli.h

@@ -0,0 +1,34 @@
+/*
+   Copyright The Overlaybd Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+#pragma once
+
+#include "ublk_device.h"
+
+// Parsed CLI command, kept apart from execution for unit testing.
+struct UblkCliCmd {
+    enum class Kind { NONE, ADD, DEL, LIST };
+    Kind kind = Kind::NONE;
+
+    UblkDeviceOpts opts; // ADD
+    bool foreground = false;
+
+    int del_dev_id = -1; // DEL
+};
+
+// Parse argv into cmd. On success returns 0 with cmd.kind set; on
+// error/help prints the message (CLI11 behavior), leaves cmd.kind as NONE
+// and returns the intended process exit code (0 for help).
+int ublk_parse_cli(int argc, char **argv, UblkCliCmd &cmd);