Skip to content

Commit 7c18f69

Browse files
ilovepiPeterChou1
ilovepi
and
PeterChou1
committed
[llvm] Add a tool to check mustache compliance against the public spec
This is a cli tool to that tests the conformance of LLVM's mustache implementation against the public Mustache spec, hosted at https://github.com/mustache/spec. This is a revised version of the patches in #111487. Co-authored-by: Peter Chou <[email protected]>
1 parent d1b0b4b commit 7c18f69

File tree

5 files changed

+312
-0
lines changed

5 files changed

+312
-0
lines changed

‎llvm/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,6 +1313,7 @@ if( LLVM_INCLUDE_UTILS )
13131313
add_subdirectory(utils/yaml-bench)
13141314
add_subdirectory(utils/split-file)
13151315
add_subdirectory(utils/mlgo-utils)
1316+
add_subdirectory(utils/llvm-test-mustache-spec)
13161317
if( LLVM_INCLUDE_TESTS )
13171318
set(LLVM_SUBPROJECT_TITLE "Third-Party/Google Test")
13181319
add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest)

‎llvm/docs/CommandGuide/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ Developer Tools
8787
llvm-exegesis
8888
llvm-ifs
8989
llvm-locstats
90+
llvm-test-mustache-spec
9091
llvm-pdbutil
9192
llvm-profgen
9293
llvm-tli-checker

‎llvm/docs/CommandGuide/llvm-test-mustache-spec.rst

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
llvm-test-mustache-spec - LLVM tool to test Mustache library compliance
2+
=======================================================================
3+
4+
.. program:: llvm-test-mustache-spec
5+
6+
SYNOPSIS
7+
--------
8+
9+
:program:`llvm-test-mustache-spec` [*inputs...*]
10+
11+
Description
12+
-----------
13+
14+
``llvm-test-mustache-spec`` tests the mustache spec conformance of the LLVM
15+
mustache library. The spec can be found here: https://github.com/mustache/spec
16+
17+
To test against the spec, simply download the spec and pass the test JSON files
18+
to the driver. Each spec file should have a list of tests for compliance with
19+
the spec. These are loaded as test cases, and rendered with our Mustache
20+
implementation, which is then compared against the expected output from the
21+
spec.
22+
23+
The current implementation only supports non-optional parts of the spec, so
24+
we do not expect any of the dynamic-names, inheritance, or lambda tests to
25+
pass. Additionally, Triple Mustache is not supported. Unsupported tests are
26+
marked as XFail and are removed from the XFail list as they are fixed.
27+
28+
The tool prints the number of test failures and successes in each of the test
29+
files to standard output.
30+
31+
EXAMPLE
32+
-------
33+
34+
.. code-block:: console
35+
36+
$ llvm-test-mustache-spec path/to/specs/\*.json
37+

‎llvm/utils/llvm-test-mustache-spec/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
add_llvm_utility(llvm-test-mustache-spec
2+
llvm-test-mustache-spec.cpp
3+
)
4+
5+
target_link_libraries(llvm-test-mustache-spec PRIVATE LLVMSupport)

‎llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// Simple drivers to test the mustache spec found at:
10+
// https://github.com/mustache/spec
11+
//
12+
// It is used to verify that the current implementation conforms to the spec.
13+
// Simply download the spec and pass the test JSON files to the driver. Each
14+
// spec file should have a list of tests for compliance with the spec. These
15+
// are loaded as test cases, and rendered with our Mustache implementation,
16+
// which is then compared against the expected output from the spec.
17+
//
18+
// The current implementation only supports non-optional parts of the spec, so
19+
// we do not expect any of the dynamic-names, inheritance, or lambda tests to
20+
// pass. Additionally, Triple Mustache is not supported. Unsupported tests are
21+
// marked as XFail and are removed from the XFail list as they are fixed.
22+
//
23+
// Usage:
24+
// llvm-test-mustache-spec path/to/test/file.json path/to/test/file2.json ...
25+
//===----------------------------------------------------------------------===//
26+
27+
#include "llvm/ADT/StringSet.h"
28+
#include "llvm/Support/CommandLine.h"
29+
#include "llvm/Support/Debug.h"
30+
#include "llvm/Support/Error.h"
31+
#include "llvm/Support/MemoryBuffer.h"
32+
#include "llvm/Support/Mustache.h"
33+
#include "llvm/Support/Path.h"
34+
#include <string>
35+
36+
using namespace llvm;
37+
using namespace llvm::json;
38+
using namespace llvm::mustache;
39+
40+
#define DEBUG_TYPE "llvm-test-mustache-spec"
41+
42+
static cl::OptionCategory Cat("llvm-test-mustache-spec Options");
43+
44+
static cl::list<std::string>
45+
InputFiles(cl::Positional, cl::desc("<input files>"), cl::OneOrMore);
46+
47+
static cl::opt<bool> ReportErrors("report-errors",
48+
cl::desc("Report errors in spec tests"),
49+
cl::cat(Cat));
50+
51+
static ExitOnError ExitOnErr;
52+
53+
static int NumXFail = 0;
54+
static int NumSuccess = 0;
55+
56+
static const StringMap<StringSet<>> XFailTestNames = {{
57+
{"delimiters.json",
58+
{
59+
"Pair Behavior",
60+
"Special Characters",
61+
"Sections",
62+
"Inverted Sections",
63+
"Partial Inheritence",
64+
"Post-Partial Behavior",
65+
"Standalone Tag",
66+
"Indented Standalone Tag",
67+
"Standalone Line Endings",
68+
"Standalone Without Previous Line",
69+
"Standalone Without Newline",
70+
}},
71+
{"~dynamic-names.json",
72+
{
73+
"Basic Behavior - Partial",
74+
"Basic Behavior - Name Resolution",
75+
"Context",
76+
"Dotted Names",
77+
"Dotted Names - Failed Lookup",
78+
"Dotted names - Context Stacking",
79+
"Dotted names - Context Stacking Under Repetition",
80+
"Dotted names - Context Stacking Failed Lookup",
81+
"Recursion",
82+
"Surrounding Whitespace",
83+
"Inline Indentation",
84+
"Standalone Line Endings",
85+
"Standalone Without Previous Line",
86+
"Standalone Without Newline",
87+
"Standalone Indentation",
88+
"Padding Whitespace",
89+
}},
90+
{"~inheritance.json",
91+
{
92+
"Default",
93+
"Variable",
94+
"Triple Mustache",
95+
"Sections",
96+
"Negative Sections",
97+
"Mustache Injection",
98+
"Inherit",
99+
"Overridden content",
100+
"Data does not override block default",
101+
"Two overridden parents",
102+
"Override parent with newlines",
103+
"Inherit indentation",
104+
"Only one override",
105+
"Parent template",
106+
"Recursion",
107+
"Multi-level inheritance, no sub child",
108+
"Text inside parent",
109+
"Text inside parent",
110+
"Block scope",
111+
"Standalone parent",
112+
"Standalone block",
113+
"Block reindentation",
114+
"Intrinsic indentation",
115+
"Nested block reindentation",
116+
117+
}},
118+
{"~lambdas.json",
119+
{
120+
"Interpolation",
121+
"Interpolation - Expansion",
122+
"Interpolation - Alternate Delimiters",
123+
"Interpolation - Multiple Calls",
124+
"Escaping",
125+
"Section",
126+
"Section - Expansion",
127+
"Section - Alternate Delimiters",
128+
"Section - Multiple Calls",
129+
130+
}},
131+
{"interpolation.json",
132+
{
133+
"Triple Mustache",
134+
"Triple Mustache Integer Interpolation",
135+
"Triple Mustache Decimal Interpolation",
136+
"Triple Mustache Null Interpolation",
137+
"Triple Mustache Context Miss Interpolation",
138+
"Dotted Names - Triple Mustache Interpolation",
139+
"Implicit Iterators - Triple Mustache",
140+
"Triple Mustache - Surrounding Whitespace",
141+
"Triple Mustache - Standalone",
142+
"Triple Mustache With Padding",
143+
}},
144+
{"partials.json", {"Standalone Indentation"}},
145+
{"sections.json", {"Implicit Iterator - Triple mustache"}},
146+
}};
147+
148+
struct TestData {
149+
static Expected<TestData> createTestData(json::Object *TestCase,
150+
StringRef InputFile) {
151+
// If any of the needed elements are missing, we cannot continue.
152+
// NOTE: partials are optional in the test schema.
153+
if (!TestCase || !TestCase->getString("template") ||
154+
!TestCase->getString("expected") || !TestCase->getString("name") ||
155+
!TestCase->get("data"))
156+
return createStringError(
157+
llvm::inconvertibleErrorCode(),
158+
"invalid JSON schema in test file: " + InputFile + "\n");
159+
160+
return TestData{TestCase->getString("template").value(),
161+
TestCase->getString("expected").value(),
162+
TestCase->getString("name").value(), TestCase->get("data"),
163+
TestCase->get("partials")};
164+
}
165+
166+
TestData() = default;
167+
168+
StringRef TemplateStr;
169+
StringRef ExpectedStr;
170+
StringRef Name;
171+
Value *Data;
172+
Value *Partials;
173+
};
174+
175+
static void reportTestFailure(const TestData &TD, StringRef ActualStr,
176+
bool IsXFail) {
177+
LLVM_DEBUG(dbgs() << "Template: " << TD.TemplateStr << "\n");
178+
if (TD.Partials) {
179+
LLVM_DEBUG(dbgs() << "Partial: ");
180+
LLVM_DEBUG(TD.Partials->print(dbgs()));
181+
LLVM_DEBUG(dbgs() << "\n");
182+
}
183+
LLVM_DEBUG(dbgs() << "JSON Data: ");
184+
LLVM_DEBUG(TD.Data->print(dbgs()));
185+
LLVM_DEBUG(dbgs() << "\n");
186+
outs() << formatv("Test {}: {}\n", (IsXFail ? "XFailed" : "Failed"), TD.Name);
187+
if (ReportErrors) {
188+
outs() << " Expected: \'" << TD.ExpectedStr << "\'\n"
189+
<< " Actual: \'" << ActualStr << "\'\n"
190+
<< " ====================\n";
191+
}
192+
}
193+
194+
static void registerPartials(Value *Partials, Template &T) {
195+
if (!Partials)
196+
return;
197+
for (const auto &[Partial, Str] : *Partials->getAsObject())
198+
T.registerPartial(Partial.str(), Str.getAsString()->str());
199+
}
200+
201+
static json::Value readJsonFromFile(StringRef &InputFile) {
202+
std::unique_ptr<MemoryBuffer> Buffer =
203+
ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile)));
204+
return ExitOnErr(parse(Buffer->getBuffer()));
205+
}
206+
207+
static bool isTestXFail(StringRef FileName, StringRef TestName) {
208+
auto P = llvm::sys::path::filename(FileName);
209+
auto It = XFailTestNames.find(P);
210+
return It != XFailTestNames.end() && It->second.contains(TestName);
211+
}
212+
213+
static bool evaluateTest(StringRef &InputFile, TestData &TestData,
214+
std::string &ActualStr) {
215+
bool IsXFail = isTestXFail(InputFile, TestData.Name);
216+
bool Matches = TestData.ExpectedStr == ActualStr;
217+
if ((Matches && IsXFail) || (!Matches && !IsXFail)) {
218+
reportTestFailure(TestData, ActualStr, IsXFail);
219+
return false;
220+
}
221+
IsXFail ? NumXFail++ : NumSuccess++;
222+
return true;
223+
}
224+
225+
static void runTest(StringRef InputFile) {
226+
NumXFail = 0;
227+
NumSuccess = 0;
228+
outs() << "Running Tests: " << InputFile << "\n";
229+
json::Value Json = readJsonFromFile(InputFile);
230+
231+
json::Object *Obj = Json.getAsObject();
232+
Array *TestArray = Obj->getArray("tests");
233+
// Even though we parsed the JSON, it can have a bad format, so check it.
234+
if (!TestArray)
235+
ExitOnErr(createStringError(
236+
llvm::inconvertibleErrorCode(),
237+
"invalid JSON schema in test file: " + InputFile + "\n"));
238+
239+
const size_t Total = TestArray->size();
240+
241+
for (Value V : *TestArray) {
242+
auto TestData =
243+
ExitOnErr(TestData::createTestData(V.getAsObject(), InputFile));
244+
Template T(TestData.TemplateStr);
245+
registerPartials(TestData.Partials, T);
246+
247+
std::string ActualStr;
248+
raw_string_ostream OS(ActualStr);
249+
T.render(*TestData.Data, OS);
250+
evaluateTest(InputFile, TestData, ActualStr);
251+
}
252+
253+
const int NumFailed = Total - NumSuccess - NumXFail;
254+
outs() << formatv("===Results===\n"
255+
" Suceeded: {}\n"
256+
" Expectedly Failed: {}\n"
257+
" Failed: {}\n"
258+
" Total: {}\n",
259+
NumSuccess, NumXFail, NumFailed, Total);
260+
}
261+
262+
int main(int argc, char **argv) {
263+
ExitOnErr.setBanner(std::string(argv[0]) + " error: ");
264+
cl::ParseCommandLineOptions(argc, argv);
265+
for (const auto &FileName : InputFiles)
266+
runTest(FileName);
267+
return 0;
268+
}

0 commit comments

Comments
 (0)