Skip to content

Commit 174aae8

Browse files
tomtauTomas Tauber
tomtau
and
Tomas Tauber
authored
fix: restrict the factorizer case to only atomic and compoundatomic rules (#766)
* fix: restrict the factorizer case to only atomic and compoundatomic rules closes #762 the full fix would require a bigger overhaul / refactoring of the optimizer * add alloc Co-authored-by: Tomas Tauber <me@tomtau.be>
1 parent c9867cf commit 174aae8

File tree

11 files changed

+65
-22
lines changed

11 files changed

+65
-22
lines changed

‎debugger/Cargo.toml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "pest_debugger"
33
description = "pest grammar debugger"
4-
version = "2.5.2"
4+
version = "2.5.3"
55
edition = "2021"
66
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
77
homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
1414
rust-version = "1.56"
1515

1616
[dependencies]
17-
pest = { path = "../pest", version = "2.5.2" }
18-
pest_meta = { path = "../meta", version = "2.5.2" }
19-
pest_vm = { path = "../vm", version = "2.5.2" }
17+
pest = { path = "../pest", version = "2.5.3" }
18+
pest_meta = { path = "../meta", version = "2.5.3" }
19+
pest_vm = { path = "../vm", version = "2.5.3" }
2020
rustyline = "10"
2121
thiserror = "1"
2222

‎derive/Cargo.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "pest_derive"
33
description = "pest's derive macro"
4-
version = "2.5.2"
4+
version = "2.5.3"
55
edition = "2021"
66
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
77
homepage = "https://pest.rs/"
@@ -23,5 +23,5 @@ std = ["pest/std", "pest_generator/std"]
2323

2424
[dependencies]
2525
# for tests, included transitively anyway
26-
pest = { path = "../pest", version = "2.5.2", default-features = false }
27-
pest_generator = { path = "../generator", version = "2.5.2", default-features = false }
26+
pest = { path = "../pest", version = "2.5.3", default-features = false }
27+
pest_generator = { path = "../generator", version = "2.5.3", default-features = false }

‎derive/tests/implicit.pest‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
program = _{ SOI ~ implicit ~ EOI }
2+
implicit= ${ or ~ (WHITESPACE+ ~ or )* }
3+
4+
or = !{ and ~ (or_op ~ and)+ | and }
5+
and = { comp ~ (and_op ~ comp)+ | comp }
6+
comp = { array ~ eq_op ~ array | array }
7+
8+
array = ${ term }
9+
10+
term = _{ ASCII_ALPHANUMERIC+ }
11+
or_op = { "||" }
12+
and_op = { "&&" }
13+
eq_op = { "=" }
14+
WHITESPACE = _{ " " | "\t" | NEWLINE }

‎derive/tests/implicit.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Licensed under the Apache License, Version 2.0
2+
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
3+
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
4+
// option. All files in the project carrying such notice may not be copied,
5+
// modified, or distributed except according to those terms.
6+
7+
#![cfg_attr(not(feature = "std"), no_std)]
8+
extern crate alloc;
9+
extern crate pest;
10+
extern crate pest_derive;
11+
12+
use pest::Parser;
13+
use pest_derive::Parser;
14+
15+
#[derive(Parser)]
16+
#[grammar = "../tests/implicit.pest"]
17+
struct TestImplicitParser;
18+
19+
#[test]
20+
fn test_implicit_whitespace() {
21+
// this failed to parse due to a bug in the optimizer
22+
// see: https://github.com/pest-parser/pest/issues/762#issuecomment-1375374868
23+
let successful_parse = TestImplicitParser::parse(Rule::program, "a a");
24+
assert!(successful_parse.is_ok());
25+
}

‎generator/Cargo.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "pest_generator"
33
description = "pest code generator"
4-
version = "2.5.2"
4+
version = "2.5.3"
55
edition = "2021"
66
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
77
homepage = "https://pest.rs/"
@@ -18,8 +18,8 @@ default = ["std"]
1818
std = ["pest/std"]
1919

2020
[dependencies]
21-
pest = { path = "../pest", version = "2.5.2", default-features = false }
22-
pest_meta = { path = "../meta", version = "2.5.2" }
21+
pest = { path = "../pest", version = "2.5.3", default-features = false }
22+
pest_meta = { path = "../meta", version = "2.5.3" }
2323
proc-macro2 = "1.0"
2424
quote = "1.0"
2525
syn = "1.0"

‎grammars/Cargo.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "pest_grammars"
33
description = "pest popular grammar implementations"
4-
version = "2.5.2"
4+
version = "2.5.3"
55
edition = "2021"
66
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
77
homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
1414
rust-version = "1.56"
1515

1616
[dependencies]
17-
pest = { path = "../pest", version = "2.5.2" }
18-
pest_derive = { path = "../derive", version = "2.5.2" }
17+
pest = { path = "../pest", version = "2.5.3" }
18+
pest_derive = { path = "../derive", version = "2.5.3" }
1919

2020
[dev-dependencies]
2121
criterion = "0.3"

‎meta/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "pest_meta"
33
description = "pest meta language parser and validator"
4-
version = "2.5.2"
4+
version = "2.5.3"
55
edition = "2021"
66
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
77
homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
1616
rust-version = "1.56"
1717

1818
[dependencies]
19-
pest = { path = "../pest", version = "2.5.2" }
19+
pest = { path = "../pest", version = "2.5.3" }
2020
once_cell = "1.8.0"
2121

2222
[build-dependencies]

‎meta/src/optimizer/factorizer.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@ pub fn factor(rule: Rule) -> Rule {
2626
}
2727
}
2828
// Converts `(rule ~ rest) | rule` to `rule ~ rest?`, avoiding trying to match `rule` twice.
29-
(Expr::Seq(l1, l2), r) => {
29+
// This is only done for atomic rules, because other rule types have implicit whitespaces.
30+
// FIXME: "desugar" implicit whitespace rules before applying any optimizations
31+
(Expr::Seq(l1, l2), r)
32+
if matches!(ty, RuleType::Atomic | RuleType::CompoundAtomic) =>
33+
{
3034
if *l1 == r {
3135
Expr::Seq(l1, Box::new(Expr::Opt(l2)))
3236
} else {

‎meta/src/optimizer/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -543,7 +543,7 @@ mod tests {
543543
use crate::ast::Expr::*;
544544
vec![Rule {
545545
name: "rule".to_owned(),
546-
ty: RuleType::Silent,
546+
ty: RuleType::Atomic,
547547
expr: box_tree!(Choice(
548548
Seq(Ident(String::from("a")), Ident(String::from("b"))),
549549
Ident(String::from("a"))
@@ -554,7 +554,7 @@ mod tests {
554554
use crate::optimizer::OptimizedExpr::*;
555555
vec![OptimizedRule {
556556
name: "rule".to_owned(),
557-
ty: RuleType::Silent,
557+
ty: RuleType::Atomic,
558558
expr: box_tree!(Seq(Ident(String::from("a")), Opt(Ident(String::from("b"))))),
559559
}]
560560
};

‎pest/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "pest"
33
description = "The Elegant Parser"
4-
version = "2.5.2"
4+
version = "2.5.3"
55
edition = "2021"
66
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
77
homepage = "https://pest.rs/"

0 commit comments

Comments
 (0)