Skip to content

Commit 46f1350

Browse files
committed
Remove tokenizer restrictions on header lengths for verbatim sections. Issue #49
1 parent e2c02eb commit 46f1350

File tree

3 files changed

+52
-6
lines changed

3 files changed

+52
-6
lines changed

History.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
* `ri` now has space between class comments from multiple files.
1616
* :stopdoc: no longer creates Object references. Issue #55 by Simon Chiang
1717
* :nodoc: works on class aliases now. Issue #51 by Steven G. Harms
18+
* Remove tokenizer restriction on header lengths for verbatim sections.
19+
Issue #49 by trans
1820

1921
=== 3.8 / 2011-06-29
2022

lib/rdoc/markup/parser.rb

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -405,13 +405,19 @@ def tokenize input
405405
@line += 1
406406
token
407407
# === text => :HEADER then :TEXT
408-
when s.scan(/(=+)\s*/) then
408+
when s.scan(/(=+)(\s*)/) then
409409
level = s[1].length
410-
level = 6 if level > 6
411-
@tokens << [:HEADER, level, *token_pos(pos)]
412-
pos = s.pos
413-
s.scan(/.*/)
414-
[:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
410+
header = [:HEADER, level, *token_pos(pos)]
411+
412+
if s[2] =~ /^\r?\n/ then
413+
s.pos -= s[2].length
414+
header
415+
else
416+
pos = s.pos
417+
s.scan(/.*/)
418+
@tokens << header
419+
[:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
420+
end
415421
# --- (at least 3) and nothing else on the line => :RULE
416422
when s.scan(/(-{3,}) *$/) then
417423
[:RULE, s[1].length - 2, *token_pos(pos)]

test/test_rdoc_markup_parser.rb

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1366,6 +1366,44 @@ def test_tokenize_verbatim_heading
13661366
assert_equal expected, @RMP.tokenize(str)
13671367
end
13681368

1369+
def test_tokenize_verbatim_rule
1370+
str = <<-STR
1371+
Verbatim section here that is double-underlined
1372+
===============================================
1373+
STR
1374+
1375+
expected = [
1376+
[:TEXT, 'Verbatim section here that is double-underlined', 2, 0],
1377+
[:NEWLINE, "\n", 49, 0],
1378+
[:HEADER, 47, 2, 1],
1379+
[:NEWLINE, "\n", 49, 1],
1380+
]
1381+
1382+
assert_equal expected, @RMP.tokenize(str)
1383+
end
1384+
1385+
def test_tokenize_verbatim_rule_fancy
1386+
str = <<-STR
1387+
A
1388+
b
1389+
===============================================
1390+
c
1391+
STR
1392+
1393+
expected = [
1394+
[:TEXT, 'A', 2, 0],
1395+
[:NEWLINE, "\n", 3, 0],
1396+
[:TEXT, 'b', 4, 1],
1397+
[:NEWLINE, "\n", 5, 1],
1398+
[:HEADER, 47, 2, 2],
1399+
[:NEWLINE, "\n", 49, 2],
1400+
[:TEXT, 'c', 4, 3],
1401+
[:NEWLINE, "\n", 5, 3],
1402+
]
1403+
1404+
assert_equal expected, @RMP.tokenize(str)
1405+
end
1406+
13691407
# HACK move to Verbatim test case
13701408
def test_verbatim_normalize
13711409
v = @RM::Verbatim.new "foo\n", "\n", "\n", "bar\n"

0 commit comments

Comments
 (0)