Skip to content

Commit fa5d2fb

Browse files
author
Hongzhen Luo
committed
[EROFS] test: add test for same-name files or dirs
Signed-off-by: Hongzhen Luo <hongzhen@linux.alibaba.com>
1 parent ba3a364 commit fa5d2fb

3 files changed

Lines changed: 193 additions & 26 deletions

File tree

‎src/overlaybd/tar/erofs/test/erofs_stress.cpp‎

Lines changed: 125 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ class StressInterImpl: public StressGenInter {
4646
/* own */
4747
int own_id_min = 0;
4848
int own_id_max = UINT32_MAX / 3;
49+
/* dir or file name */
50+
std::map<int, std::set<std::string>> name_map;
4951

5052
/* generate file content in build phase */
5153
bool build_gen_content(StressNode *node, StressHostFile *file) override {
@@ -180,6 +182,25 @@ class StressInterImpl: public StressGenInter {
180182
node->own = std::to_string(st.st_uid) + std::to_string(st.st_gid);
181183
return true;
182184
}
185+
186+
/* generate a random dir or file name in the current layer */
187+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
188+
std::string res;
189+
int cnt = 0;
190+
191+
res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true);
192+
if (name_map.find(idx) ==name_map.end())
193+
name_map[idx] = std::set<std::string>();
194+
while (name_map[idx].find(res) != name_map[idx].end()) {
195+
res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true);
196+
cnt ++;
197+
/* try up to 1000 times */
198+
if (cnt > 1000)
199+
LOG_ERROR_RETURN(-1, "", "fail to generate a random name");
200+
}
201+
name_map[idx].insert(res);
202+
return res;
203+
}
183204
};
184205

185206
/*
@@ -190,7 +211,7 @@ class StressInterImpl: public StressGenInter {
190211
*
191212
* A simple test for verifying the integrity of the FS tree.
192213
*/
193-
class StressCase001: public StressBase {
214+
class StressCase001: public StressBase, public StressInterImpl {
194215
public:
195216
StressCase001(std::string path, int layers): StressBase(path, layers) {}
196217

@@ -207,8 +228,9 @@ class StressCase001: public StressBase {
207228
EROFS_STRESS_UNIMPLEMENTED_FUNC(bool, verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file), true)
208229

209230
/* simplely generate random dir and file names */
210-
EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \
211-
get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true))
231+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
232+
return StressInterImpl::generate_name(idx, depth, root_path, type);
233+
}
212234

213235
/*
214236
* each layer has two dirs:
@@ -253,8 +275,9 @@ class StressCase002: public StressBase, public StressInterImpl {
253275
}
254276

255277
/* simplely generate random dir and file names */
256-
EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \
257-
get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true))
278+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
279+
return StressInterImpl::generate_name(idx, depth, root_path, type);
280+
}
258281

259282
/*
260283
* each layer has two dirs:
@@ -298,8 +321,9 @@ class StressCase003: public StressBase, public StressInterImpl {
298321
}
299322

300323
/* simplely generate random dir and file names */
301-
EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \
302-
get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true))
324+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
325+
return StressInterImpl::generate_name(idx, depth, root_path, type);
326+
}
303327

304328
std::vector<int> layer_dirs(int idx) {
305329
std::vector<int> ret;
@@ -339,8 +363,9 @@ class StressCase004: public StressBase, public StressInterImpl {
339363
}
340364

341365
/* simplely generate random dir and file names */
342-
EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \
343-
get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true))
366+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
367+
return StressInterImpl::generate_name(idx, depth, root_path, type);
368+
}
344369

345370
std::vector<int> layer_dirs(int idx) {
346371
std::vector<int> ret;
@@ -379,8 +404,9 @@ class StressCase005: public StressBase, public StressInterImpl {
379404
}
380405

381406
/* simplely generate random dir and file names */
382-
EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \
383-
get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true))
407+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
408+
return StressInterImpl::generate_name(idx, depth, root_path, type);
409+
}
384410

385411
std::vector<int> layer_dirs(int idx) {
386412
std::vector<int> ret;
@@ -431,8 +457,9 @@ class StressCase006: public StressBase, public StressInterImpl {
431457
}
432458

433459
/* simplely generate random dir and file names */
434-
EROFS_STRESS_UNIMPLEMENTED_FUNC(std::string, generate_name(int idx, int depth, std::string _prefix, NODE_TYPE type), \
435-
get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true))
460+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
461+
return StressInterImpl::generate_name(idx, depth, root_path, type);
462+
}
436463

437464
std::vector<int> layer_dirs(int idx) {
438465
std::vector<int> ret;
@@ -443,6 +470,82 @@ class StressCase006: public StressBase, public StressInterImpl {
443470
}
444471
};
445472

473+
/*
474+
* TC007
475+
*
476+
* Create layers, each layer contains 10 dirs,
477+
* each dir contains 10 files.
478+
*
479+
* Test the scenario where the upper layer and lower
480+
* layer contain files or directories with the same name.
481+
*/
482+
class StressCase007: public StressBase, public StressInterImpl {
483+
private:
484+
std::map<int, std::set<std::string>> mp;
485+
public:
486+
StressCase007(std::string path, int layers): StressBase(path, layers) {}
487+
488+
bool build_gen_mod(StressNode *node, StressHostFile *file) override {
489+
return StressInterImpl::build_gen_mod(node, file);
490+
}
491+
bool build_gen_own(StressNode *node, StressHostFile *file) override {
492+
return StressInterImpl::build_gen_own(node, file);
493+
}
494+
bool build_gen_xattrs(StressNode *node, StressHostFile *file) override {
495+
return StressInterImpl::build_gen_xattrs(node, file);
496+
}
497+
bool build_gen_content(StressNode *node, StressHostFile *file) override {
498+
return StressInterImpl::build_gen_content(node, file);
499+
}
500+
501+
bool verify_gen_mod(StressNode *node, photon::fs::IFile *erofs_file) override {
502+
return StressInterImpl::verify_gen_mod(node, erofs_file);
503+
}
504+
bool verify_gen_own(StressNode *node, photon::fs::IFile *erofs_file) override {
505+
return StressInterImpl::verify_gen_own(node, erofs_file);
506+
}
507+
bool verify_gen_xattrs(StressNode *node, photon::fs::IFile *erofs_file) override {
508+
return StressInterImpl::verify_gen_xattrs(node, erofs_file);
509+
}
510+
bool verify_gen_content(StressNode *node, photon::fs::IFile *erofs_file) override {
511+
return StressInterImpl::verify_gen_content(node, erofs_file);
512+
}
513+
514+
std::string generate_name(int idx, int depth, std::string root_path, NODE_TYPE type) override {
515+
std::string res;
516+
int cnt = 0;
517+
518+
if (idx < 1) {
519+
res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true);
520+
goto out;
521+
}
522+
res = tree->get_same_name(idx, depth, root_path, type);
523+
/* fall back to a random name */
524+
if (res.length() == 0)
525+
res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true);
526+
if (mp.find(idx) == mp.end())
527+
mp[idx] = std::set<std::string>();
528+
/* already used in this layer, fall back to a random name */
529+
while (mp[idx].find(res) != mp[idx].end()) {
530+
res = get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true);
531+
cnt ++;
532+
if (cnt > 1000)
533+
LOG_ERROR_RETURN(-1, "", "fail to gernate name in TC007");
534+
}
535+
mp[idx].insert(res);
536+
out:
537+
return res;
538+
}
539+
540+
std::vector<int> layer_dirs(int idx) {
541+
std::vector<int> ret;
542+
543+
for (int i = 0; i < 10; i ++)
544+
ret.emplace_back(30);
545+
return ret;
546+
}
547+
};
548+
446549
TEST(ErofsStressTest, TC001) {
447550
std::srand(static_cast<unsigned int>(std::time(0)));
448551
StressCase001 *tc001 = new StressCase001("./erofs_stress_001", 20);
@@ -491,6 +594,15 @@ TEST(ErofsStressTest, TC006) {
491594
delete tc006;
492595
}
493596

597+
TEST(ErofsStressTest, TC007) {
598+
std::srand(static_cast<unsigned int>(std::time(0)));
599+
/* 50 layers */
600+
StressCase007 *tc007 = new StressCase007("./erofs_stress_007", 50);
601+
602+
ASSERT_EQ(tc007->run(), true);
603+
delete tc007;
604+
}
605+
494606
int main(int argc, char **argv) {
495607

496608
::testing::InitGoogleTest(&argc, argv);

‎src/overlaybd/tar/erofs/test/erofs_stress_base.cpp‎

Lines changed: 64 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
#include <chrono>
2020
#include <sys/stat.h>
2121
#include <dirent.h>
22+
#include <algorithm>
23+
#include <random>
2224
#include "../liberofs.h"
2325
#include "../erofs_fs.h"
2426
#include "../../../../tools/comm_func.h"
@@ -37,6 +39,31 @@ std::string get_randomstr(int max_length, bool range)
3739
}
3840
return res;
3941
}
42+
static bool is_substring(const std::string& str, const std::string& substring) {
43+
return str.find(substring) != std::string::npos;
44+
}
45+
46+
std::string StressFsTree::get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type) {
47+
std::vector<std::string> vec;
48+
for (const auto& pair : tree) {
49+
if (pair.first == "/" || pair.second->type != type ||
50+
!is_substring(pair.first, root_path) ||
51+
pair.first.length() == root_path.length())
52+
continue;
53+
std::string last_component = pair.first.substr(root_path.length() + 1);
54+
if (!is_substring(last_component, "/"))
55+
vec.emplace_back(last_component);
56+
}
57+
58+
if (vec.empty())
59+
return get_randomstr(type ? MAX_FILE_NAME : MAX_DIR_NAME, true);
60+
61+
std::random_device rd;
62+
std::default_random_engine engine(rd());
63+
std::shuffle(vec.begin(), vec.end(), engine);
64+
65+
return vec[0];
66+
}
4067

4168
struct LayerNode {
4269
std::string pwd;
@@ -66,6 +93,15 @@ static LayerNode *build_layer_tree(std::vector<int> &dirs) {
6693
}
6794

6895
bool StressBase::create_layer(int idx) {
96+
97+
#define MAX_TRY_TIME 10
98+
99+
std::string origin_prefix = prefix;
100+
// add a random prefix to avoid operations in the same dir
101+
prefix = prefix + "/" + get_randomstr(20, false);
102+
if (host_fs->mkdir(prefix.c_str(), 0755) != 0)
103+
LOG_ERROR_RETURN(-1, false, "fail to prepare for the current workdir `", prefix);
104+
69105
std::vector<int> dirs = layer_dirs(idx);
70106
LayerNode *layer_tree = build_layer_tree(dirs);
71107
std::vector<LayerNode*> q;
@@ -79,17 +115,17 @@ bool StressBase::create_layer(int idx) {
79115
layer_tree->pwd = root_path;
80116
q.emplace_back(layer_tree);
81117

118+
StressNode *node = new StressNode(layer_tree->pwd.substr(prefix.length()), NODE_DIR);
119+
if (host_fs->mkdir(layer_tree->pwd.c_str(), 0755) != 0)
120+
LOG_ERROR_RETURN(-1, false, "fail to mkdir `", layer_tree->pwd);
121+
tree->add_node(node);
122+
82123
// traverse the layer tree
83124
while (q.size()) {
84125
bool res;
85126
LayerNode *cur = q.front();
86127
q.erase(q.begin());
87128

88-
StressNode *node = new StressNode(cur->pwd.substr(prefix.length()), NODE_DIR);
89-
if (host_fs->mkdir(cur->pwd.c_str(), 0755) != 0)
90-
LOG_ERROR_RETURN(-1, false, "fail to mkdir `", cur->pwd);
91-
tree->add_node(node);
92-
93129
for (int i = 0; i < (int)cur->num_files; i ++) {
94130
std::string name_prefix = cur->pwd.substr(prefix.length());
95131
// generate filename for files in the current dir
@@ -106,22 +142,33 @@ bool StressBase::create_layer(int idx) {
106142
if (!tree->add_node(node))
107143
LOG_ERROR_RETURN(-1, false, "failt to add node `", filename);
108144
file_info->file->fsync();
145+
delete file_info;
109146
}
110147

111148
for (int i = 0; i < (int)cur->subdirs.size(); i ++) {
112149
LayerNode *next = cur->subdirs[i];
113150
next->depth = cur->depth + 1;
114151
// generate subdir name in the current dir
115-
next->pwd = cur->pwd + "/" + generate_name(idx, cur->depth, cur->pwd.substr(prefix.length()), NODE_DIR);
116-
q.emplace_back(next);
152+
for (int try_times = 0; try_times < MAX_TRY_TIME; try_times++) {
153+
next->pwd = cur->pwd + "/" + generate_name(idx, cur->depth, cur->pwd.substr(prefix.length()), NODE_DIR);
154+
if (host_fs->mkdir(next->pwd.c_str(), 0755) == 0) {
155+
StressNode *dir_node = new StressNode(next->pwd.substr(prefix.length()), NODE_DIR);
156+
tree->add_node(dir_node);
157+
q.emplace_back(next);
158+
break;
159+
}
160+
}
117161
}
118162
delete cur;
119163
}
120164

121-
std::string layer_name = prefix + "/layer" + std::to_string(idx);
165+
#undef MAX_TRY_TIME
166+
167+
std::string layer_name = origin_prefix + "/layer" + std::to_string(idx);
122168
std::string cmd = std::string(" sudo tar --xattrs --xattrs-include='*' -cf ") + layer_name + ".tar -C " + prefix + " " + root_dirname;
123169
if (system(cmd.c_str()))
124170
LOG_ERROR_RETURN(-1, false, "fail to prepare tar file, cmd: `", cmd);
171+
prefix = origin_prefix;
125172
return true;
126173
}
127174

@@ -220,7 +267,9 @@ bool StressBase::verify(photon::fs::IFileSystem *erofs_fs) {
220267
first = false;
221268
} while (!items.empty());
222269

223-
return tree->is_emtry();
270+
if (!tree->is_emtry())
271+
LOG_ERROR_RETURN(-1, false, "Mismatch: in-mem tree is not empty!");
272+
return true;
224273
}
225274

226275
bool StressBase::run()
@@ -246,9 +295,12 @@ bool StressBase::run()
246295

247296
bool ret = verify(erofs_fs);
248297

249-
std::string clear_cmd = std::string("rm -rf ") + prefix;
250-
if (system(clear_cmd.c_str()))
251-
LOG_ERROR_RETURN(-1, false, "fail to clear tmp workdir, cmd: `", clear_cmd);
298+
if (ret) {
299+
std::string clear_cmd = std::string("rm -rf ") + prefix;
300+
if (system(clear_cmd.c_str()))
301+
LOG_ERROR_RETURN(-1, false, "fail to clear tmp workdir, cmd: `", clear_cmd);
302+
303+
}
252304

253305
return ret;
254306
}

‎src/overlaybd/tar/erofs/test/erofs_stress_base.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,10 +203,14 @@ class StressFsTree {
203203
bool is_emtry() {
204204
return tree.empty();
205205
}
206+
207+
std::string get_same_name(int idx, int depth, std::string root_path, NODE_TYPE type);
206208
};
207209

208210
class StressBase: public StressGenInter {
209211
public:
212+
StressFsTree *tree;
213+
210214
StressBase(std::string path, int num): prefix(path), num_layers(num) {
211215
host_fs = photon::fs::new_localfs_adaptor();
212216
if (!host_fs)
@@ -235,7 +239,6 @@ class StressBase: public StressGenInter {
235239
photon::fs::IFileSystem *host_fs;
236240
bool workdir_exists;
237241

238-
StressFsTree *tree;
239242
bool create_layer(int idx);
240243
LSMT::IFileRW *mkfs();
241244
bool verify(photon::fs::IFileSystem *erofs_fs);

0 commit comments

Comments
 (0)