Skip to content

Commit eed542b

Browse files
denbezrukovdenbezrukov
committed
refactor(formatter): Introduce Token element
1 parent 3e54958 commit eed542b

File tree

66 files changed

+973
-949
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+973
-949
lines changed

‎.changeset/lemon-planes-knock.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
"@biomejs/biome": patch
3+
---
4+
5+
Refactored formatter to use strict `Token` element for better performance. The new `Token` variant is optimized for static, ASCII-only text (keywords, operators, punctuation) with the following constraints:
6+
7+
- ASCII only (no Unicode characters)
8+
- No newlines (`\n`, `\r`)
9+
- No tab characters (`\t`)
10+
11+
This enables faster printing and fitting logic by using bulk string operations (`push_str`, `len()`) instead of character-by-character iteration with Unicode width calculations.

‎crates/biome_css_formatter/src/comments.rs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ impl FormatRule<SourceComment<CssLanguage>> for FormatCssLeadingComment {
3333

3434
// SAFETY: Safe, `is_doc_comment` only returns `true` for multiline comments
3535
let first_line = lines.next().unwrap();
36-
write!(f, [dynamic_text(first_line.trim_end(), source_offset)])?;
36+
write!(f, [text(first_line.trim_end(), source_offset)])?;
3737

3838
source_offset += first_line.text_len();
3939

@@ -44,10 +44,7 @@ impl FormatRule<SourceComment<CssLanguage>> for FormatCssLeadingComment {
4444
1,
4545
&format_once(|f| {
4646
for line in lines {
47-
write!(
48-
f,
49-
[hard_line_break(), dynamic_text(line.trim(), source_offset)]
50-
)?;
47+
write!(f, [hard_line_break(), text(line.trim(), source_offset)])?;
5148

5249
source_offset += line.text_len();
5350
}

‎crates/biome_css_formatter/src/css/auxiliary/declaration_with_semicolon.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ impl FormatNodeRule<CssDeclarationWithSemicolon> for FormatCssDeclarationWithSem
2121
// if semicolon is present, use the token's format to keep the comments
2222
write!(f, [semicolon_token.format()])
2323
} else {
24-
write!(f, [text(";")])
24+
write!(f, [token(";")])
2525
}
2626
}
2727
}

‎crates/biome_css_formatter/src/css/value/url_value_raw.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ impl FormatNodeRule<CssUrlValueRaw> for FormatCssUrlValueRaw {
88
fn fmt_fields(&self, node: &CssUrlValueRaw, f: &mut CssFormatter) -> FormatResult<()> {
99
let CssUrlValueRawFields { value_token } = node.as_fields();
1010
let value_token = value_token?;
11-
let text = value_token.token_text();
11+
let token_text = value_token.token_text();
1212
write!(
1313
f,
1414
[format_replaced(
1515
&value_token,
16-
&dynamic_text(text.trim(), value_token.text_trimmed_range().start())
16+
&text(token_text.trim(), value_token.text_trimmed_range().start())
1717
)]
1818
)
1919
}

‎crates/biome_css_formatter/src/lib.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -316,10 +316,9 @@ impl FormatRule<CssSyntaxToken> for FormatCssSyntaxToken {
316316
let original = token.text_trimmed();
317317
match original.to_ascii_lowercase_cow() {
318318
Cow::Borrowed(_) => self.format_trimmed_token_trivia(token, f),
319-
Cow::Owned(lowercase) => write!(
320-
f,
321-
[dynamic_text(&lowercase, token.text_trimmed_range().start())]
322-
),
319+
Cow::Owned(lowercase) => {
320+
write!(f, [text(&lowercase, token.text_trimmed_range().start())])
321+
}
323322
}
324323
} else {
325324
self.format_trimmed_token_trivia(token, f)

‎crates/biome_css_formatter/src/utils/string_utils.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use biome_formatter::QuoteStyle;
99
use biome_formatter::token::string::normalize_string;
1010
use biome_formatter::{
1111
Format, FormatResult,
12-
prelude::{dynamic_text, write},
12+
prelude::{text, write},
1313
};
1414
use biome_rowan::SyntaxToken;
1515
use biome_string_case::StrLikeExtension;
@@ -35,7 +35,7 @@ impl Format<CssFormatContext> for FormatTokenAsLowercase {
3535
f,
3636
[format_replaced(
3737
&self.token,
38-
&dynamic_text(&lowercase, self.token.text_trimmed_range().start()),
38+
&text(&lowercase, self.token.text_trimmed_range().start()),
3939
)]
4040
),
4141
}

‎crates/biome_css_formatter/src/verbatim.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use crate::context::CssFormatContext;
22
use biome_css_syntax::{CssLanguage, CssSyntaxNode};
33
use biome_formatter::format_element::tag::VerbatimKind;
44
use biome_formatter::formatter::Formatter;
5-
use biome_formatter::prelude::{Tag, dynamic_text};
5+
use biome_formatter::prelude::{Tag, text};
66
use biome_formatter::trivia::{FormatLeadingComments, FormatTrailingComments};
77
use biome_formatter::{
88
Buffer, CstFormatContext, Format, FormatContext, FormatElement, FormatError, FormatResult,
@@ -114,7 +114,7 @@ impl Format<CssFormatContext> for FormatCssVerbatimNode<'_> {
114114
},
115115
);
116116

117-
dynamic_text(
117+
text(
118118
&normalize_newlines(&original_source, LINE_TERMINATORS),
119119
self.node.text_trimmed_range().start(),
120120
)

‎crates/biome_css_semantic/src/format_semantic_model.rs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,11 @@ impl Format<FormatSemanticModelContext> for Rule {
9090
write!(
9191
f,
9292
[
93-
dynamic_text(
93+
text(
9494
self.node().syntax().text_trimmed().into_text().text(),
9595
self.node().syntax().text_trimmed_range().start()
9696
),
97-
text(":"),
97+
token(":"),
9898
space(),
9999
&self.specificity(),
100100
]
@@ -108,13 +108,13 @@ impl Format<FormatSemanticModelContext> for Selector {
108108
write!(
109109
f,
110110
[
111-
dynamic_text(self.text().into_text().text(), self.range().start()),
112-
text(":"),
111+
text(self.text().into_text().text(), self.range().start()),
112+
token(":"),
113113
space(),
114114
&self.specificity(),
115115
space(),
116-
text(" @ "),
117-
dynamic_text(range.as_str(), TextSize::default()),
116+
token(" @ "),
117+
text(range.as_str(), TextSize::default()),
118118
]
119119
)
120120
}
@@ -125,15 +125,15 @@ impl Format<FormatSemanticModelContext> for Specificity {
125125
write!(
126126
f,
127127
[
128-
text("("),
129-
dynamic_text(self.0.to_string().as_str(), TextSize::default()),
130-
text(","),
128+
token("("),
129+
text(self.0.to_string().as_str(), TextSize::default()),
130+
token(","),
131131
space(),
132-
dynamic_text(self.1.to_string().as_str(), TextSize::default()),
133-
text(","),
132+
text(self.1.to_string().as_str(), TextSize::default()),
133+
token(","),
134134
space(),
135-
dynamic_text(self.2.to_string().as_str(), TextSize::default()),
136-
text(")")
135+
text(self.2.to_string().as_str(), TextSize::default()),
136+
token(")")
137137
]
138138
)
139139
}

‎crates/biome_formatter/src/arguments.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ impl<Context> Format<Context> for Argument<'_, Context> {
7676
///
7777
/// # fn main() -> FormatResult<()> {
7878
/// let formatted = format!(SimpleFormatContext::default(), [
79-
/// format_args!(text("a"), space(), text("b"))
79+
/// format_args!(token("a"), space(), token("b"))
8080
/// ])?;
8181
///
8282
/// assert_eq!("a b", formatted.print()?.as_code());
@@ -140,26 +140,26 @@ mod tests {
140140
write!(
141141
&mut buffer,
142142
[
143-
text("function"),
143+
token("function"),
144144
space(),
145-
text("a"),
145+
token("a"),
146146
space(),
147-
group(&format_args!(text("("), text(")")))
147+
group(&format_args!(token("("), token(")")))
148148
]
149149
)
150150
.unwrap();
151151

152152
assert_eq!(
153153
buffer.into_vec(),
154154
vec![
155-
FormatElement::StaticText { text: "function" },
155+
FormatElement::Token { text: "function" },
156156
FormatElement::Space,
157-
FormatElement::StaticText { text: "a" },
157+
FormatElement::Token { text: "a" },
158158
FormatElement::Space,
159159
// Group
160160
FormatElement::Tag(Tag::StartGroup(tag::Group::new())),
161-
FormatElement::StaticText { text: "(" },
162-
FormatElement::StaticText { text: ")" },
161+
FormatElement::Token { text: "(" },
162+
FormatElement::Token { text: ")" },
163163
FormatElement::Tag(Tag::EndGroup)
164164
]
165165
);

‎crates/biome_formatter/src/buffer.rs‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ pub trait Buffer {
2727
/// let mut state = FormatState::new(SimpleFormatContext::default());
2828
/// let mut buffer = VecBuffer::new(&mut state);
2929
///
30-
/// buffer.write_element(FormatElement::StaticText { text: "test"}).unwrap();
30+
/// buffer.write_element(FormatElement::Token { text: "test"}).unwrap();
3131
///
32-
/// assert_eq!(buffer.into_vec(), vec![FormatElement::StaticText { text: "test" }]);
32+
/// assert_eq!(buffer.into_vec(), vec![FormatElement::Token { text: "test" }]);
3333
/// ```
3434
///
3535
fn write_element(&mut self, element: FormatElement) -> FormatResult<()>;
@@ -53,9 +53,9 @@ pub trait Buffer {
5353
/// let mut state = FormatState::new(SimpleFormatContext::default());
5454
/// let mut buffer = VecBuffer::new(&mut state);
5555
///
56-
/// buffer.write_fmt(format_args!(text("Hello World"))).unwrap();
56+
/// buffer.write_fmt(format_args!(token("Hello World"))).unwrap();
5757
///
58-
/// assert_eq!(buffer.into_vec(), vec![FormatElement::StaticText{ text: "Hello World" }]);
58+
/// assert_eq!(buffer.into_vec(), vec![FormatElement::Token{ text: "Hello World" }]);
5959
/// ```
6060
fn write_fmt(mut self: &mut Self, arguments: Arguments<Self::Context>) -> FormatResult<()> {
6161
write(&mut self, arguments)
@@ -269,7 +269,7 @@ Make sure that you take and restore the snapshot in order and that this snapshot
269269
///
270270
/// impl Format<SimpleFormatContext> for Preamble {
271271
/// fn fmt(&self, f: &mut Formatter<SimpleFormatContext>) -> FormatResult<()> {
272-
/// write!(f, [text("# heading"), hard_line_break()])
272+
/// write!(f, [token("# heading"), hard_line_break()])
273273
/// }
274274
/// }
275275
///
@@ -280,7 +280,7 @@ Make sure that you take and restore the snapshot in order and that this snapshot
280280
/// {
281281
/// let mut with_preamble = PreambleBuffer::new(&mut buffer, Preamble);
282282
///
283-
/// write!(&mut with_preamble, [text("this text will be on a new line")])?;
283+
/// write!(&mut with_preamble, [token("this text will be on a new line")])?;
284284
/// }
285285
///
286286
/// let formatted = Formatted::new(Document::from(buffer.into_vec()), SimpleFormatContext::default());
@@ -300,7 +300,7 @@ Make sure that you take and restore the snapshot in order and that this snapshot
300300
///
301301
/// impl Format<SimpleFormatContext> for Preamble {
302302
/// fn fmt(&self, f: &mut Formatter<SimpleFormatContext>) -> FormatResult<()> {
303-
/// write!(f, [text("# heading"), hard_line_break()])
303+
/// write!(f, [token("# heading"), hard_line_break()])
304304
/// }
305305
/// }
306306
///
@@ -455,11 +455,11 @@ where
455455
/// write!(
456456
/// buffer,
457457
/// [
458-
/// text("The next soft line or space gets replaced by a space"),
458+
/// token("The next soft line or space gets replaced by a space"),
459459
/// soft_line_break_or_space(),
460-
/// text("and the line here"),
460+
/// token("and the line here"),
461461
/// soft_line_break(),
462-
/// text("is removed entirely.")
462+
/// token("is removed entirely.")
463463
/// ]
464464
/// )
465465
/// })]
@@ -468,10 +468,10 @@ where
468468
/// assert_eq!(
469469
/// formatted.document().as_ref(),
470470
/// &[
471-
/// FormatElement::StaticText { text: "The next soft line or space gets replaced by a space" },
471+
/// FormatElement::Token { text: "The next soft line or space gets replaced by a space" },
472472
/// FormatElement::Space,
473-
/// FormatElement::StaticText { text: "and the line here" },
474-
/// FormatElement::StaticText { text: "is removed entirely." }
473+
/// FormatElement::Token { text: "and the line here" },
474+
/// FormatElement::Token { text: "is removed entirely." }
475475
/// ]
476476
/// );
477477
///
@@ -706,19 +706,19 @@ pub trait BufferExtensions: Buffer + Sized {
706706
/// let formatted = format!(SimpleFormatContext::default(), [format_with(|f| {
707707
/// let mut recording = f.start_recording();
708708
///
709-
/// write!(recording, [text("A")])?;
710-
/// write!(recording, [text("B")])?;
709+
/// write!(recording, [token("A")])?;
710+
/// write!(recording, [token("B")])?;
711711
///
712-
/// write!(recording, [format_with(|f| write!(f, [text("C"), text("D")]))])?;
712+
/// write!(recording, [format_with(|f| write!(f, [token("C"), token("D")]))])?;
713713
///
714714
/// let recorded = recording.stop();
715715
/// assert_eq!(
716716
/// recorded.deref(),
717717
/// &[
718-
/// FormatElement::StaticText{ text: "A" },
719-
/// FormatElement::StaticText{ text: "B" },
720-
/// FormatElement::StaticText{ text: "C" },
721-
/// FormatElement::StaticText{ text: "D" }
718+
/// FormatElement::Token{ text: "A" },
719+
/// FormatElement::Token{ text: "B" },
720+
/// FormatElement::Token{ text: "C" },
721+
/// FormatElement::Token{ text: "D" }
722722
/// ]
723723
/// );
724724
///

0 commit comments

Comments
 (0)