Skip to content

Commit b750cd9

Browse files
rsandellrsandell
authored
Merge pull request #206 from pascal-hofmann/feature/add-codepointlimit
[JENKINS-71077] Allow overriding codePointLimit to allow reading larger yaml files
2 parents a5cf2e4 + 78e3fac commit b750cd9

File tree

3 files changed

+132
-16
lines changed

3 files changed

+132
-16
lines changed

‎src/main/java/org/jenkinsci/plugins/pipeline/utility/steps/conf/ReadYamlStep.java‎

Lines changed: 76 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,16 @@
6060
*/
6161
public class ReadYamlStep extends AbstractFileOrTextStep {
6262

63+
public static final int LIBRARY_DEFAULT_CODE_POINT_LIMIT = new LoaderOptions().getCodePointLimit();
64+
public static final String MAX_CODE_POINT_LIMIT_PROPERTY = ReadYamlStep.class.getName() + ".MAX_CODE_POINT_LIMIT";
65+
@SuppressFBWarnings(value={"MS_SHOULD_BE_FINAL"}, justification="Non final so that an admin can adjust the value through the groovy script console without restarting the instance.")
66+
private static /*almost final*/ int MAX_CODE_POINT_LIMIT = Integer.getInteger(MAX_CODE_POINT_LIMIT_PROPERTY, LIBRARY_DEFAULT_CODE_POINT_LIMIT);
67+
public static final String DEFAULT_CODE_POINT_LIMIT_PROPERTY = ReadYamlStep.class.getName() + ".DEFAULT_CODE_POINT_LIMIT";
68+
@SuppressFBWarnings(value={"MS_SHOULD_BE_FINAL"}, justification="Non final so that an admin can adjust the value through the groovy script console without restarting the instance.")
69+
private static /*almost final*/ int DEFAULT_CODE_POINT_LIMIT = Integer.getInteger(DEFAULT_CODE_POINT_LIMIT_PROPERTY, -1);
70+
//By default, use whatever Yaml thinks is best
71+
private int codePointLimit = -1;
72+
6373
// the upper limit is hardcoded to 1000 to stop people shooting themselves in the foot
6474
public static final int HARDCODED_CEILING_MAX_ALIASES_FOR_COLLECTIONS = 1000;
6575
public static final int LIBRARY_DEFAULT_MAX_ALIASES_FOR_COLLECTIONS = new LoaderOptions().getMaxAliasesForCollections();
@@ -77,6 +87,29 @@ public class ReadYamlStep extends AbstractFileOrTextStep {
7787
public ReadYamlStep() {
7888
}
7989

90+
public static int getMaxCodePointLimit() {
91+
return MAX_CODE_POINT_LIMIT;
92+
}
93+
94+
/**
95+
* Setter with an added check to ensure the default does not exceed the max value.
96+
* @param defaultCodePointLimit the default value to set.
97+
* @return the actual value set after checking the max allowed.
98+
*/
99+
public static int setDefaultCodePointLimit(int defaultCodePointLimit) {
100+
if (defaultCodePointLimit > MAX_CODE_POINT_LIMIT) {
101+
throw new IllegalArgumentException(defaultCodePointLimit + " > " + MAX_CODE_POINT_LIMIT +
102+
". Reduce the required DEFAULT_CODE_POINT_LIMIT or convince your administrator to increase" +
103+
" the max allowed value with the system property \"" + MAX_CODE_POINT_LIMIT_PROPERTY + "\"");
104+
}
105+
DEFAULT_CODE_POINT_LIMIT = defaultCodePointLimit;
106+
return DEFAULT_CODE_POINT_LIMIT;
107+
}
108+
109+
public static int getDefaultCodePointLimit() {
110+
return DEFAULT_CODE_POINT_LIMIT;
111+
}
112+
80113
/**
81114
* Setter with an added check to ensure the default does not exceed the hardcoded max value.
82115
* TODO: decide if we want to add a message here before failing back.
@@ -116,6 +149,20 @@ public static int getDefaultMaxAliasesForCollections() {
116149
return DEFAULT_MAX_ALIASES_FOR_COLLECTIONS;
117150
}
118151

152+
public int getCodePointLimit() {
153+
return codePointLimit;
154+
}
155+
156+
@DataBoundSetter
157+
public void setCodePointLimit(final int codePointLimit) {
158+
if (codePointLimit > MAX_CODE_POINT_LIMIT) {
159+
throw new IllegalArgumentException(codePointLimit + " > " + MAX_CODE_POINT_LIMIT +
160+
". Reduce the code points in your yaml or convince your administrator to increase" +
161+
" the max allowed value with the system property \"" + MAX_CODE_POINT_LIMIT_PROPERTY + "\"");
162+
}
163+
this.codePointLimit = codePointLimit;
164+
}
165+
119166
public int getMaxAliasesForCollections() {
120167
return maxAliasesForCollections;
121168
}
@@ -138,7 +185,15 @@ public StepExecution start(StepContext context) throws Exception {
138185
} else if (DEFAULT_MAX_ALIASES_FOR_COLLECTIONS >= 0) {
139186
ac = DEFAULT_MAX_ALIASES_FOR_COLLECTIONS;
140187
}
141-
return new Execution(this, context, ac);
188+
189+
int cpl = -1;
190+
if (codePointLimit >= 0) {
191+
cpl = codePointLimit;
192+
} else if (DEFAULT_CODE_POINT_LIMIT >= 0) {
193+
cpl = DEFAULT_CODE_POINT_LIMIT;
194+
}
195+
196+
return new Execution(this, context, ac, cpl);
142197
}
143198

144199
@Extension
@@ -162,12 +217,16 @@ public String getDisplayName() {
162217
public static class Execution extends AbstractFileOrTextStepExecution<Object> {
163218
private static final long serialVersionUID = 1L;
164219
private transient ReadYamlStep step;
220+
221+
private final int codePointLimit;
222+
165223
private final int maxAliasesForCollections;
166224

167-
protected Execution(@NonNull ReadYamlStep step, @NonNull StepContext context, int maxAliasesForCollections) {
225+
protected Execution(@NonNull ReadYamlStep step, @NonNull StepContext context, int maxAliasesForCollections, int codePointLimit) {
168226
super(step, context);
169227
this.step = step;
170228
this.maxAliasesForCollections = maxAliasesForCollections;
229+
this.codePointLimit = codePointLimit;
171230
}
172231

173232
/**
@@ -220,23 +279,24 @@ protected Object doRun() throws Exception {
220279
}
221280

222281
protected Yaml newYaml() {
282+
//Need everything just to be able to specify constructor and LoaderOptions
283+
LoaderOptions loaderOptions = new LoaderOptions();
223284
if (maxAliasesForCollections >= 0) {
224-
//Need everything just to be able to specify constructor and LoaderOptions
225-
LoaderOptions loaderOptions = new LoaderOptions();
226285
loaderOptions.setMaxAliasesForCollections(maxAliasesForCollections);
227-
Representer representer = new Representer(new DumperOptions());
228-
//The Yaml constructors does this internally, so just in case...
229-
DumperOptions dumperOptions = new DumperOptions();
230-
dumperOptions.setDefaultFlowStyle(representer.getDefaultFlowStyle());
231-
dumperOptions.setDefaultScalarStyle(representer.getDefaultScalarStyle());
232-
dumperOptions.setAllowReadOnlyProperties(representer.getPropertyUtils().isAllowReadOnlyProperties());
233-
dumperOptions.setTimeZone(representer.getTimeZone());
234-
// Use SafeConstructor to limit objects to standard Java objects like List or Long
235-
return new Yaml(new SafeConstructor(new LoaderOptions()), representer, dumperOptions, loaderOptions);
236-
} else {
237-
// Use SafeConstructor to limit objects to standard Java objects like List or Long
238-
return new Yaml(new SafeConstructor(new LoaderOptions()));
239286
}
287+
if (codePointLimit >= 0) {
288+
loaderOptions.setCodePointLimit(codePointLimit);
289+
}
290+
291+
Representer representer = new Representer(new DumperOptions());
292+
// The Yaml constructor does this internally, so just in case...
293+
DumperOptions dumperOptions = new DumperOptions();
294+
dumperOptions.setDefaultFlowStyle(representer.getDefaultFlowStyle());
295+
dumperOptions.setDefaultScalarStyle(representer.getDefaultScalarStyle());
296+
dumperOptions.setAllowReadOnlyProperties(representer.getPropertyUtils().isAllowReadOnlyProperties());
297+
dumperOptions.setTimeZone(representer.getTimeZone());
298+
// Use SafeConstructor to limit objects to standard Java objects like List or Long
299+
return new Yaml(new SafeConstructor(new LoaderOptions()), representer, dumperOptions, loaderOptions);
240300
}
241301
}
242302
}

‎src/main/resources/org/jenkinsci/plugins/pipeline/utility/steps/conf/ReadYamlStep/help.groovy‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,27 @@ ul {
5757
text(' and so will overwrite any value already present if not a new YAML document.')
5858
}
5959
}
60+
li {
61+
code('codePointLimit: ')
62+
text('Limit for incoming data in bytes. ')
63+
text('Defaults to 3145728 (3MB) if not set.')
64+
br()
65+
em {
66+
String max_code_point_limit_property = org.jenkinsci.plugins.pipeline.utility.steps.conf.ReadYamlStep.MAX_CODE_POINT_LIMIT_PROPERTY
67+
String default_code_point_limit_property = org.jenkinsci.plugins.pipeline.utility.steps.conf.ReadYamlStep.DEFAULT_CODE_POINT_LIMIT_PROPERTY
68+
int max_code_point_limit = org.jenkinsci.plugins.pipeline.utility.steps.conf.ReadYamlStep.getMaxCodePointLimit()
69+
int default_code_point_limit = org.jenkinsci.plugins.pipeline.utility.steps.conf.ReadYamlStep.getDefaultCodePointLimit()
70+
text("""\
71+
There is a maximum value you can set on this controller: ${max_code_point_limit}.
72+
The administrator can change the max allowed value by setting the System property: ${max_code_point_limit_property}.
73+
The default can also be changed by setting the System property: ${default_code_point_limit_property}
74+
so that no pipeline code needs to be changed. """.stripIndent())
75+
if (default_code_point_limit >= -1) {
76+
br()
77+
text("On this controller the default is set to ${default_code_point_limit}.")
78+
}
79+
}
80+
}
6081
li {
6182
code('maxAliasesForCollections: ')
6283
text('Restrict the amount of aliases for collections (sequences and mappings) to avoid ')

‎src/test/java/org/jenkinsci/plugins/pipeline/utility/steps/conf/ReadYamlStepTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import hudson.model.Label;
2323
import hudson.model.Result;
24+
import org.yaml.snakeyaml.LoaderOptions;
2425

2526
public class ReadYamlStepTest {
2627
@Rule
@@ -55,6 +56,7 @@ public class ReadYamlStepTest {
5556

5657
@Before
5758
public void setup() throws Exception {
59+
System.setProperty("org.jenkinsci.plugins.pipeline.utility.steps.conf.ReadYamlStep.MAX_CODE_POINT_LIMIT", "10485760");
5860
System.setProperty("org.jenkinsci.plugins.pipeline.utility.steps.conf.ReadYamlStep.MAX_MAX_ALIASES_FOR_COLLECTIONS", "500");
5961
j.createOnlineSlave(Label.get("slaves"));
6062
}
@@ -187,6 +189,39 @@ public void readFileAndText() throws Exception {
187189
j.assertBuildStatusSuccess(p.scheduleBuild2(0));
188190
}
189191

192+
193+
@Test
194+
public void codePointLimits() throws Exception {
195+
StringBuilder str = new StringBuilder("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
196+
int desired = new LoaderOptions().getCodePointLimit();
197+
while (str.length() < desired) {
198+
str.append(str);
199+
}
200+
final String yaml = "a: " + str;
201+
File file = temp.newFile();
202+
FileUtils.writeStringToFile(file, yaml, Charset.defaultCharset());
203+
WorkflowJob p = j.jenkins.createProject(WorkflowJob.class, "p");
204+
p.setDefinition(new CpsFlowDefinition(
205+
"node('"+j.jenkins.getSelfLabel()+"') { def yaml = readYaml file: '" + separatorsToSystemEscaped(file.getAbsolutePath()) + "'}",
206+
true));
207+
j.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0));
208+
p.setDefinition(new CpsFlowDefinition(
209+
"node('"+j.jenkins.getSelfLabel()+"') { def yaml = readYaml codePointLimit: 10485760, file: '" + separatorsToSystemEscaped(file.getAbsolutePath()) + "'}",
210+
true));
211+
j.assertBuildStatus(Result.SUCCESS, p.scheduleBuild2(0));
212+
}
213+
214+
@Test
215+
public void setDefaultCodePointLimitHigherThanMaxFailsWithException() throws Exception {
216+
Exception exception = assertThrows(IllegalArgumentException.class, () -> {
217+
ReadYamlStep readYamlStep = new ReadYamlStep();
218+
readYamlStep.setDefaultCodePointLimit(ReadYamlStep.getMaxCodePointLimit() + 1);
219+
});
220+
String expectedMessage = "Reduce the required DEFAULT_CODE_POINT_LIMIT or convince your administrator to increase";
221+
String actualMessage = exception.getMessage();
222+
assertTrue(actualMessage + " <<<< DOES NOT CONTAIN >>>> " + expectedMessage, actualMessage.contains(expectedMessage));
223+
}
224+
190225
@Test
191226
public void millionLaughs() throws Exception {
192227
final String lol = "a: &a [\"lol\",\"lol\",\"lol\",\"lol\",\"lol\",\"lol\",\"lol\",\"lol\",\"lol\"]\n" +

0 commit comments

Comments
 (0)