toml: fix hex values starting with a, e or E and comments ending with crlf (#12367)

Author: larpon

vlib/toml/checker/checker.v

@@ -141,12 +141,6 @@ fn (c Checker) check_number(num ast.Number) ? {
 		is_oct = lit_sans_sign.starts_with('0o')
 		is_hex = lit_sans_sign.starts_with('0x')
 
-		third := lit[2]
-		if third in scanner.digit_extras {
-			ascii = byte(third).ascii_str()
-			return error(@MOD + '.' + @STRUCT + '.' + @FN +
-				' numbers like "$lit" (hex, octal and binary) can not have `$ascii` in ...${c.excerpt(num.pos)}...')
-		}
 		lit_sans_sign_and_type_prefix := lit_sans_sign[2..]
 
 		if lit_sans_sign_and_type_prefix.starts_with('_')

vlib/toml/scanner/scanner.v

@@ -273,6 +273,12 @@ pub fn (s &Scanner) at() byte {
 	return byte(-1)
 }
 
+// at_crlf returns `true` if the scanner is at a `\r` character
+// and the next character is a `\n`.
+fn (s Scanner) at_crlf() bool {
+	return s.at() == `\r` && s.peek(1) == `\n`
+}
+
 // peek returns the character code from the input text at position + `n`.
 // peek returns `-1` if it can't peek `n` characters ahead.
 [direct_array_access; inline]
@@ -322,6 +328,10 @@ fn (mut s Scanner) ignore_line() ?string {
 	for c := s.at(); c != -1 && c != `\n`; c = s.at() {
 		s.next()
 		util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'skipping "${byte(c).ascii_str()}"')
+		if s.at_crlf() {
+			util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'letting `\\r\\n` slip through')
+			return s.text[start..s.pos]
+		}
 	}
 	return s.text[start..s.pos]
 }

vlib/toml/tests/crlf_test.v

@@ -0,0 +1,13 @@
+import toml
+
+fn test_crlf() {
+	str_value := 'test string'
+	mut toml_txt := 'crlf_string = "test string"
+# Comment with CRLF\r\n'
+	toml_doc := toml.parse(toml_txt) or { panic(err) }
+
+	value := toml_doc.value('crlf_string')
+	assert value == toml.Any(str_value)
+	assert value as string == str_value
+	assert value.string() == str_value
+}

vlib/toml/tests/types_test.v

@@ -68,3 +68,14 @@ open_sourced = "Jun 22 2019 20:20:28"'
 	value := toml_doc.value('v.open_sourced').string()
 	assert value == 'Jun 22 2019 20:20:28'
 }
+
+fn test_hex_values() {
+	// Regression test
+	// '0xb' is carefully chosen to include the 'b' character that also denotes binary via 0b prefix.
+	toml_txt := 'hex = 0xb'
+	toml_doc := toml.parse(toml_txt) or { panic(err) }
+
+	value := toml_doc.value('hex')
+	assert value as i64 == 11
+	assert value.i64() == 11
+}