-
Notifications
You must be signed in to change notification settings - Fork 143
Expand file tree
/
Copy pathintegration.rs
More file actions
326 lines (279 loc) · 10.9 KB
/
integration.rs
File metadata and controls
326 lines (279 loc) · 10.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
use std::{fs, os::unix::fs::PermissionsExt, path::Path};
use serde::Deserialize;
use tempfile::tempdir;
datatest_stable::harness! { { test = run_integration_test, root = "tests/fixtures", pattern = "hermes.toml$" } }
#[derive(Deserialize)]
struct TestConfig {
#[serde(default)]
artifact: Vec<ArtifactExpectation>,
#[serde(default)]
command: Vec<CommandExpectation>,
}
#[derive(Deserialize)]
struct ArtifactExpectation {
package: String,
target: String,
should_exist: bool,
}
#[derive(Deserialize)]
struct CommandExpectation {
args: Vec<String>,
#[serde(default)]
should_not_exist: bool,
}
fn run_integration_test(path: &Path) -> datatest_stable::Result<()> {
// `path` is `tests/fixtures/<test_case>/hermes.toml`
let test_case_root = path.parent().unwrap();
let test_name = test_case_root.file_name().unwrap().to_str().unwrap();
let source_dir = test_case_root.join("source");
// Perform all work in a sandbox directory. This ensures that we don't
// get interference from `Cargo.toml` files in any parent directories.
let temp = tempdir()?;
let sandbox_root = temp.path().join(test_name);
copy_dir_contents(&source_dir, &sandbox_root)?;
// --- SETUP CHARON SHIM ---
let shim_dir = temp.path().join("bin");
fs::create_dir_all(&shim_dir)?;
let real_charon = which::which("charon").unwrap();
let log_file = sandbox_root.join("charon_args.log");
let shim_path = shim_dir.join("charon");
let shim_content = format!(
r#"#!/bin/sh
# Log each argument on a new line
for arg in "$@"; do
echo "ARG:$arg" >> "{0}"
done
echo "---END-INVOCATION---" >> "{0}"
# If mock JSON is set, bypass charon and return mock payload to stdout
if [ -n "$HERMES_MOCK_CHARON_JSON" ]; then
cat "$HERMES_MOCK_CHARON_JSON"
exit 101
fi
# Execute real charon
exec "{1}" "$@"
"#,
log_file.display(),
real_charon.display(),
);
fs::write(&shim_path, shim_content)?;
let mut perms = fs::metadata(&shim_path)?.permissions();
perms.set_mode(0o755);
fs::set_permissions(&shim_path, perms)?;
let mut cmd = assert_cmd::cargo_bin_cmd!("hermes");
cmd.env("HERMES_FORCE_TTY", "1");
cmd.env("FORCE_COLOR", "1");
// Prepend shim_dir to PATH (make sure our shim comes first).
let original_path = std::env::var_os("PATH").unwrap_or_default();
let new_path = std::env::join_paths(
std::iter::once(shim_dir).chain(std::env::split_paths(&original_path)),
)?;
cmd.env("CARGO_TARGET_DIR", sandbox_root.join("target"))
.env("PATH", new_path)
.env_remove("RUSTFLAGS")
// Forces deterministic output path: target/hermes/hermes_test_target
// (normally, the path includes a hash of the crate's path).
.env("HERMES_TEST_DIR_NAME", "hermes_test_target");
// Tests can specify the log level.
let env_log_file = test_case_root.join("rust_log.txt");
if env_log_file.exists() {
let log_level = fs::read_to_string(env_log_file)?;
cmd.env("RUST_LOG", log_level.trim());
}
// Mock JSON integration
let mock_json_file = test_case_root.join("mock_charon_output.json");
if mock_json_file.exists() {
let mock_src = fs::read_to_string(&mock_json_file).unwrap();
let processed_mock = mock_src.replace("[PROJECT_ROOT]", sandbox_root.to_str().unwrap());
let processed_mock_file = sandbox_root.join("mock_charon_output.json");
fs::write(&processed_mock_file, &processed_mock).unwrap();
let abs_processed = std::env::current_dir().unwrap().join(&processed_mock_file);
cmd.env("HERMES_MOCK_CHARON_JSON", abs_processed);
}
// Tests can specify the cwd to invoke from.
let cwd_file = test_case_root.join("cwd.txt");
if cwd_file.exists() {
let rel_path = fs::read_to_string(cwd_file)?;
cmd.current_dir(sandbox_root.join(rel_path.trim()));
} else {
cmd.current_dir(&sandbox_root);
}
// Tests can specify the arguments to pass to `hermes verify`.
let args_file = test_case_root.join("args.txt");
if args_file.exists() {
let args_str = fs::read_to_string(args_file)?;
cmd.args(args_str.trim().split_whitespace());
} else {
cmd.arg("verify");
}
let mut assert = cmd.assert();
// Tests can specify the expected exit status.
let expected_status_file = test_case_root.join("expected_status.txt");
if expected_status_file.exists() {
let status = fs::read_to_string(&expected_status_file)?;
if status.trim() == "failure" {
assert = assert.failure();
} else {
assert = assert.success();
}
} else {
assert = assert.success();
};
// Tests can specify the expected stderr.
let expected_stderr_file = test_case_root.join("expected_stderr.txt");
let stderr_regex_path = test_case_root.join("expected_stderr.regex.txt");
if stderr_regex_path.exists() {
let expected_regex = fs::read_to_string(&stderr_regex_path)?;
let output = assert.get_output();
let actual_stderr = String::from_utf8_lossy(&output.stderr);
let actual_stripped = strip_ansi_escapes::strip(&*actual_stderr);
let actual_str = String::from_utf8_lossy(&actual_stripped).into_owned().replace("\r", "");
let replace_path = sandbox_root.to_str().unwrap();
let stderr = actual_str.replace(replace_path, "[PROJECT_ROOT]");
let rx = regex::Regex::new(expected_regex.trim()).unwrap();
if !rx.is_match(&stderr) {
panic!(
"Stderr regex mismatch.\nExpected regex: {}\nActual stderr:\n{}",
expected_regex, stderr
);
}
} else if expected_stderr_file.exists() {
let needle = fs::read_to_string(expected_stderr_file)?;
let output = assert.get_output();
let actual_stderr = String::from_utf8_lossy(&output.stderr);
let actual_stripped = strip_ansi_escapes::strip(&*actual_stderr);
let actual_str = String::from_utf8_lossy(&actual_stripped).into_owned().replace("\r", "");
let replace_path = sandbox_root.to_str().unwrap();
let stderr = actual_str.replace(replace_path, "[PROJECT_ROOT]");
if !stderr.contains(needle.trim()) {
panic!(
"Stderr mismatch.\nExpected substring: {}\nActual stderr:\n{}",
needle.trim(),
stderr
);
}
}
// Load Config
let mut config = TestConfig { artifact: vec![], command: vec![] };
let config_file = test_case_root.join("expected_config.toml");
if config_file.exists() {
let content = fs::read_to_string(&config_file)?;
config = toml::from_str(&content)?;
}
// Backward compatibility for the previous step instructions
let artifacts_file = test_case_root.join("expected_artifacts.toml");
if artifacts_file.exists() {
let content = fs::read_to_string(&artifacts_file)?;
let partial: TestConfig = toml::from_str(&content)?;
config.artifact.extend(partial.artifact);
}
if !config.artifact.is_empty() {
let charon_dir = sandbox_root.join("target/hermes/hermes_test_target/charon");
assert_artifacts_match(&charon_dir, &config.artifact)?;
}
if !config.command.is_empty() {
if !log_file.exists() {
panic!("Command log file not found! Was the shim called?");
}
let log_content = fs::read_to_string(log_file)?;
let invocations = parse_command_log(&log_content);
assert_commands_match(&invocations, &config.command);
}
Ok(())
}
fn parse_command_log(content: &str) -> Vec<Vec<String>> {
let mut invocations = Vec::new();
let mut current_args = Vec::new();
for line in content.lines() {
if line == "---END-INVOCATION---" {
invocations.push(current_args);
current_args = Vec::new();
} else if let Some(arg) = line.strip_prefix("ARG:") {
current_args.push(arg.to_string());
}
}
invocations
}
fn assert_commands_match(invocations: &[Vec<String>], expectations: &[CommandExpectation]) {
for exp in expectations {
let mut found = false;
for cmd in invocations {
// Check if exp.args is a subsequence of cmd
if is_subsequence(cmd, &exp.args) {
found = true;
break;
}
}
if !exp.should_not_exist && !found {
panic!("Expected command invocation with args {:?} was not found.\nCaptured Invocations: {:#?}", exp.args, invocations);
} else if exp.should_not_exist && found {
panic!("Unexpected command invocation with args {:?} WAS found.", exp.args);
}
}
}
fn is_subsequence(haystack: &[String], needle: &[String]) -> bool {
if needle.is_empty() {
return true;
}
let mut needle_iter = needle.iter();
let mut next_needle = needle_iter.next();
for item in haystack {
if let Some(n) = next_needle {
if item == n {
next_needle = needle_iter.next();
}
} else {
return true;
}
}
next_needle.is_none()
}
fn assert_artifacts_match(
charon_dir: &Path,
expectations: &[ArtifactExpectation],
) -> std::io::Result<()> {
if !charon_dir.exists() {
if expectations.iter().any(|e| e.should_exist) {
panic!("Charon output directory does not exist: {:?}", charon_dir);
}
return Ok(());
}
// Collect all generated .llbc files
let mut generated_files = Vec::new();
for entry in fs::read_dir(charon_dir)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().to_string();
if name.ends_with(".llbc") {
generated_files.push(name);
}
}
for exp in expectations {
// We expect filenames format: "{package}-{target}-{hash}.llbc"
// Since hash is dynamic, we match on prefix.
let prefix = format!("{}-{}-", exp.package, exp.target);
let found = generated_files.iter().any(|f| f.starts_with(&prefix));
if exp.should_exist && !found {
panic!(
"Missing expected artifact for package='{}', target='{}'.\nExpected prefix: '{}'\nFound files: {:?}",
exp.package, exp.target, prefix, generated_files
);
} else if !exp.should_exist && found {
panic!(
"Found unexpected artifact for package='{}', target='{}'.\nMatched prefix: '{}'\nFound files: {:?}",
exp.package, exp.target, prefix, generated_files
);
}
}
Ok(())
}
fn copy_dir_contents(src: &Path, dst: &Path) -> std::io::Result<()> {
fs::create_dir_all(dst)?;
for entry in fs::read_dir(src)? {
let entry = entry?;
if entry.file_type()?.is_dir() {
copy_dir_contents(&entry.path(), &dst.join(entry.file_name()))?;
} else {
fs::copy(entry.path(), dst.join(entry.file_name()))?;
}
}
Ok(())
}