Merge pull request #335 from salvete/main

Cleanup and minor fixes.
Skip the packing for centos 7 (arm)

Author: BigVan

.github/workflows/release.yml

@@ -16,6 +16,9 @@ jobs:
       matrix:
         images: [ubuntu_18.04, ubuntu_20.04, ubuntu_22.04, centos_7, centos_8, mcr.microsoft.com/cbl-mariner/base/core_2.0]
         platforms: [linux/amd64, linux/arm64]
+        exclude:
+          - images: centos_7
+            platforms: linux/arm64
     steps:
     - name: Set Release Version
       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')

.github/workflows/release/build.sh

@@ -39,9 +39,23 @@ if [[ ${OS} =~ "ubuntu" ]]; then
     PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.${DISTRO}"
 elif [[ ${OS} =~ "centos" ]]; then
     if [[ ${OS} == "centos:7" ]]; then
+        sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
+        sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+        sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+        yum clean all
+        rm -rf /var/cache/yum
+        yum -y update
+
         yum install -y centos-release-scl
-        yum install -y devtoolset-7-gcc-c++
 
+        sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
+        sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+        sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+        yum clean all
+        rm -rf /var/cache/yum
+        yum -y update
+
+        yum install -y devtoolset-7-gcc-c++
         export PATH="/opt/rh/devtoolset-7/root/usr/bin:$PATH"
         PACKAGE_RELEASE="-DPACKAGE_RELEASE=${RELEASE_NO}.el7"
         COMPILER="-DCMAKE_C_COMPILER=/opt/rh/devtoolset-7/root/usr/bin/gcc -DCMAKE_CXX_COMPILER=/opt/rh/devtoolset-7/root/usr/bin/g++"

src/overlaybd/tar/erofs/CMakeLists.txt

@@ -3,7 +3,7 @@ include(FetchContent)
 FetchContent_Declare(
     erofs-utils
     GIT_REPOSITORY https://gitee.com/anolis/erofs-utils.git
-    GIT_TAG      overlaybd-dev-3
+    GIT_TAG      6ae5eb64136aeea90bda46d286af4628ea35d974
 )
 
 FetchContent_MakeAvailable(erofs-utils)
@@ -13,7 +13,7 @@ execute_process(
     WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR}
 )
 execute_process(
-    COMMAND ./configure --disable-lz4 --without-liblzma --without-libzstd --without-uuid
+    COMMAND ./configure --disable-lz4 --disable-lzma --without-libzstd --without-uuid --disable-multithreading
     WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR}
 )
 execute_process(

src/overlaybd/tar/erofs/liberofs.cpp

@@ -16,7 +16,6 @@
 #define round_down_blk(addr) ((addr) & (~(SECTOR_SIZE - 1)))
 #define round_up_blk(addr) (round_down_blk((addr) + SECTOR_SIZE - 1))
 #define min(a, b) (a) < (b) ? (a) : (b)
-#define MAP_FILE_NAME "upper.map"
 
 #define EROFS_UNIMPLEMENTED 1
 
@@ -257,7 +256,7 @@ static ssize_t erofs_read_photon_file(void *buf, u64 offset, size_t len,
                 LOG_ERROR("Fail to read sector %lld.", end - SECTOR_SIZE);
                 return -1;
             }
-            memcpy(extra_buf, buf + end - SECTOR_SIZE - offset,
+            memcpy(extra_buf, (char*)buf + end - SECTOR_SIZE - offset,
                    offset + len + SECTOR_SIZE - end);
             if (cache->write_sector(end - SECTOR_SIZE, extra_buf)
                 != SECTOR_SIZE)
@@ -377,28 +376,13 @@ static int erofs_source_fallocate(struct erofs_vfile *vf,
 
 static int erofs_source_ftruncate(struct erofs_vfile *vf, u64 length)
 {
-    return EROFS_UNIMPLEMENTED;
+    return -EROFS_UNIMPLEMENTED;
 }
 
 static ssize_t erofs_source_read(struct erofs_vfile *vf, void *buf,
                                  size_t bytes)
 {
-    u64 i = 0;
-    while (bytes) {
-        u64 len = bytes > INT_MAX ? INT_MAX : bytes;
-        u64 ret;
-
-        ret = _source->read(buf + i, len);
-        if (ret < 1) {
-            if (ret == 0)
-                break;
-            else
-                return -1;
-        }
-        bytes -= ret;
-        i += ret;
-    }
-    return i;
+    return _source->read(buf, bytes);
 }
 
 static off_t erofs_source_lseek(struct erofs_vfile *vf, u64 offset, int whence)
@@ -411,6 +395,7 @@ struct erofs_mkfs_cfg {
     struct erofs_tarfile *erofstar;
     bool incremental;
     bool ovlfs_strip;
+    FILE *mp_fp;
 };
 
 static int rebuild_src_count;
@@ -421,18 +406,18 @@ int erofs_mkfs(struct erofs_mkfs_cfg *cfg)
     struct erofs_tarfile *erofstar;
     struct erofs_sb_info *sbi;
     struct erofs_buffer_head *sb_bh;
-    struct erofs_inode *root;
+    struct erofs_inode *root = NULL;
     erofs_blk_t nblocks;
 
     erofstar = cfg->erofstar;
     sbi = cfg->sbi;
     if (!erofstar || !sbi)
         return -EINVAL;
 
-    if (!erofstar->mapfile)
+    if (!cfg->mp_fp)
         return -EINVAL;
 
-    err = erofs_blocklist_open(erofstar->mapfile, true);
+    err = erofs_blocklist_open(cfg->mp_fp, true);
     if (err) {
         LOG_ERROR("[erofs] Fail to open erofs blocklist.");
         return -EINVAL;
@@ -444,7 +429,12 @@ int erofs_mkfs(struct erofs_mkfs_cfg *cfg)
     }
 
     if (!cfg->incremental) {
-        sb_bh = erofs_reserve_sb(sbi);
+        sbi->bmgr = erofs_buffer_init(sbi, 0);
+        if (!sbi->bmgr) {
+            err = -ENOMEM;
+            goto exit;
+        }
+        sb_bh = erofs_reserve_sb(sbi->bmgr);
         if (IS_ERR(sb_bh)) {
             LOG_ERROR("[erofs] Fail to reseve space for superblock.");
             err =  PTR_ERR(sb_bh);
@@ -456,7 +446,11 @@ int erofs_mkfs(struct erofs_mkfs_cfg *cfg)
             LOG_ERROR("[erofs] Fail to read superblock.");
             goto exit;
         }
-        erofs_buffer_init(sbi, sbi->primarydevice_blocks);
+        sbi->bmgr = erofs_buffer_init(sbi, sbi->primarydevice_blocks);
+        if (!sbi->bmgr) {
+            err = -ENOMEM;
+            goto exit;
+        }
         sb_bh = NULL;
     }
 
@@ -471,17 +465,17 @@ int erofs_mkfs(struct erofs_mkfs_cfg *cfg)
 
     while (!(err = tarerofs_parse_tar(root, erofstar)));
     if (err < 0) {
-        LOG_ERROR("[erofs] Fail to parse tar file.");
+        LOG_ERROR("[erofs] Fail to parse tar file.", err);
         goto exit;
     }
 
     err = erofs_rebuild_dump_tree(root, cfg->incremental);
     if (err < 0) {
-        LOG_ERROR("[erofs] Fail to dump tree.");
+        LOG_ERROR("[erofs] Fail to dump tree.", err);
         goto exit;
     }
 
-    err = erofs_bflush(NULL);
+    err = erofs_bflush(sbi->bmgr, NULL);
     if (err) {
         LOG_ERROR("[erofs] Bflush failed.");
         goto exit;
@@ -498,23 +492,22 @@ int erofs_mkfs(struct erofs_mkfs_cfg *cfg)
     }
 
     /* flush all remaining buffers */
-    err = erofs_bflush(NULL);
+    err = erofs_bflush(sbi->bmgr, NULL);
     if (err)
         goto exit;
 
     err = erofs_dev_resize(sbi, nblocks);
 exit:
     if (root)
         erofs_iput(root);
+    erofs_buffer_exit(sbi->bmgr);
     erofs_blocklist_close();
     return err;
 }
 
 static int erofs_init_sbi(struct erofs_sb_info *sbi, photon::fs::IFile *fout,
                           struct erofs_vfops *ops, int blkbits)
 {
-    int err;
-
     sbi->blkszbits = (char)blkbits;
     sbi->bdev.ops = ops;
     fout->lseek(0, 0);
@@ -524,26 +517,17 @@ static int erofs_init_sbi(struct erofs_sb_info *sbi, photon::fs::IFile *fout,
 }
 
 static int erofs_init_tar(struct erofs_tarfile *erofstar,
-                          photon::fs::IFile *tar_file, struct erofs_vfops *ops)
+                          struct erofs_vfops *ops)
 {
-    int err;
-    struct stat st;
-
     erofstar->global.xattrs = LIST_HEAD_INIT(erofstar->global.xattrs);
-    erofstar->mapfile = MAP_FILE_NAME;
     erofstar->aufs = true;
     erofstar->rvsp_mode = true;
     erofstar->dev = rebuild_src_count + 1;
 
     erofstar->ios.feof = false;
     erofstar->ios.tail = erofstar->ios.head = 0;
     erofstar->ios.dumpfd = -1;
-    err = tar_file->fstat(&st);
-    if (err) {
-        LOG_ERROR("Fail to fstat tar file.");
-        return err;
-    }
-    erofstar->ios.sz = st.st_size;
+    erofstar->ios.sz = 0;
     erofstar->ios.bufsize = 16384;
     do {
             erofstar->ios.buffer = (char*)malloc(erofstar->ios.bufsize);
@@ -560,18 +544,16 @@ static int erofs_init_tar(struct erofs_tarfile *erofstar,
     return 0;
 }
 
-static int erofs_write_map_file(photon::fs::IFile *fout, uint64_t blksz)
+static int erofs_write_map_file(photon::fs::IFile *fout, uint64_t blksz, FILE *fp)
 {
-    FILE *fp;
     uint64_t blkaddr, toff;
     uint32_t nblocks;
 
-    fp = fopen(MAP_FILE_NAME, "r");
     if (fp == NULL) {
        LOG_ERROR("unable to get upper.map, ignored");
        return -1;
     }
-
+    rewind(fp);
     while (fscanf(fp, "%" PRIx64" %x %" PRIx64 "\n", &blkaddr, &nblocks, &toff)
            >= 3)
     {
@@ -584,9 +566,8 @@ static int erofs_write_map_file(photon::fs::IFile *fout, uint64_t blksz)
             LOG_ERRNO_RETURN(0, -1, "failed to write lba");
         }
     }
-    fclose(fp);
 
-    return unlink(MAP_FILE_NAME);
+    return 0;
 }
 
 static int erofs_close_sbi(struct erofs_sb_info *sbi, ErofsCache *cache)
@@ -643,7 +624,7 @@ int LibErofs::extract_tar(photon::fs::IFile *source, bool meta_only, bool first_
         return err;
     }
     /* initialization of erofstar */
-    err = erofs_init_tar(&erofstar, _source, &source_vfops);
+    err = erofs_init_tar(&erofstar, &source_vfops);
     if (err) {
         LOG_ERROR("Failed to init tarerofs");
         goto exit;
@@ -654,6 +635,7 @@ int LibErofs::extract_tar(photon::fs::IFile *source, bool meta_only, bool first_
     cfg.incremental = !first_layer;
     erofs_cfg = erofs_get_configure();
     erofs_cfg->c_ovlfs_strip = true;
+    cfg.mp_fp = std::tmpfile();
 
     err = erofs_mkfs(&cfg);
     if (err) {
@@ -662,14 +644,15 @@ int LibErofs::extract_tar(photon::fs::IFile *source, bool meta_only, bool first_
     }
 
     /* write mapfile */
-    err = erofs_write_map_file(_target, blksize);
+    err = erofs_write_map_file(_target, blksize, cfg.mp_fp);
     if (err) {
         LOG_ERROR("Failed to write mapfile.");
         goto exit;
     }
 exit:
     err = erofs_close_sbi(&sbi, &erofs_cache);
     erofs_close_tar(&erofstar);
+    std::fclose(cfg.mp_fp);
     return err;
 }