-
Notifications
You must be signed in to change notification settings - Fork 481
Fix incorrect date parsing for multi-word number representations #1280
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -426,11 +426,39 @@ def _token_with_digits_is_ok(self, token): | |||||||
| def _simplify(self, date_string, settings=None): | ||||||||
| date_string = date_string.lower() | ||||||||
| simplifications = self._get_simplifications(settings=settings) | ||||||||
|
|
||||||||
| if self.info.get("name") == "ru": | ||||||||
| date_string = self._process_russian_compound_ordinals( | ||||||||
| date_string, simplifications | ||||||||
| ) | ||||||||
| else: | ||||||||
| date_string = self._apply_simplifications(date_string, simplifications) | ||||||||
|
|
||||||||
| return date_string | ||||||||
|
|
||||||||
| def _apply_simplifications(self, date_string, simplifications): | ||||||||
| for simplification in simplifications: | ||||||||
| pattern, replacement = list(simplification.items())[0] | ||||||||
| date_string = pattern.sub(replacement, date_string).lower() | ||||||||
| return date_string | ||||||||
|
|
||||||||
| def _process_russian_compound_ordinals(self, date_string, simplifications): | ||||||||
| """Process Russian compound ordinals mathematically (двадцать + первое = 21).""" | ||||||||
| date_string = self._apply_simplifications(date_string, simplifications) | ||||||||
|
|
||||||||
| def replace_number_pairs(match): | ||||||||
| first_num = int(match.group(1)) | ||||||||
| second_num = int(match.group(2)) | ||||||||
| result = first_num + second_num | ||||||||
| if 1 <= result <= 31 and first_num in [20, 30] and 1 <= second_num <= 9: | ||||||||
| return str(result) | ||||||||
| return match.group(0) | ||||||||
|
|
||||||||
| number_pair_pattern = r"\b(\d+)\s+(\d+)\b" | ||||||||
| date_string = re.sub(number_pair_pattern, replace_number_pairs, date_string) | ||||||||
|
Comment on lines
+457
to
+458
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The regex pattern should be compiled as a module-level constant since it's used repeatedly, which would improve performance and maintainability.
Suggested change
Copilot uses AI. Check for mistakes. Positive FeedbackNegative Feedback |
||||||||
|
|
||||||||
| return date_string | ||||||||
|
|
||||||||
| def _get_simplifications(self, settings=None): | ||||||||
| no_word_spacing = eval(self.info.get("no_word_spacing", "False")) | ||||||||
| if settings.NORMALIZE: | ||||||||
|
|
||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -1103,3 +1103,113 @@ def test_search_dates_with_prepositions(self): | |||||
| ("30 апреля", datetime.datetime(2025, 4, 30, 0, 0), "ru"), | ||||||
| ] | ||||||
| assert result == expected | ||||||
|
|
||||||
| @parameterized.expand( | ||||||
| [ | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцатое февраля. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцатое февраля", | ||||||
| expected_day=20, | ||||||
| expected_month=2, | ||||||
| description="20th February", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать первое февраля. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать первое февраля", | ||||||
| expected_day=21, | ||||||
| expected_month=2, | ||||||
| description="21st February", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать второе февраля. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать второе февраля", | ||||||
| expected_day=22, | ||||||
| expected_month=2, | ||||||
| description="22nd February", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать третье февраля. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать третье февраля", | ||||||
| expected_day=23, | ||||||
| expected_month=2, | ||||||
| description="23rd February", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать четвёртое февраля. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать четвёртое февраля", | ||||||
| expected_day=24, | ||||||
| expected_month=2, | ||||||
| description="24th February (with ё)", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать четвертое февраля. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать четвертое февраля", | ||||||
| expected_day=24, | ||||||
| expected_month=2, | ||||||
| description="24th February (without ё)", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать пятое марта. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать пятое марта", | ||||||
| expected_day=25, | ||||||
| expected_month=3, | ||||||
| description="25th March", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать шестое марта. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать шестое марта", | ||||||
| expected_day=26, | ||||||
| expected_month=3, | ||||||
| description="26th March", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать седьмое марта. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать седьмое марта", | ||||||
| expected_day=27, | ||||||
| expected_month=3, | ||||||
| description="27th March", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать восьмое марта. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать восьмое марта", | ||||||
| expected_day=28, | ||||||
| expected_month=3, | ||||||
| description="28th March", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Двадцать девятое марта. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Двадцать девятое марта", | ||||||
| expected_day=29, | ||||||
| expected_month=3, | ||||||
| description="29th March", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Тридцатое марта. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Тридцатое марта", | ||||||
| expected_day=30, | ||||||
| expected_month=3, | ||||||
| description="30th March", | ||||||
| ), | ||||||
| param( | ||||||
| text="Ужасное событие произошло в тот день. Тридцать первое марта. Вспоминаю тот день с ужасом.", | ||||||
| expected_text="Тридцать первое марта", | ||||||
| expected_day=31, | ||||||
| expected_month=3, | ||||||
| description="31st March", | ||||||
| ), | ||||||
| ] | ||||||
| ) | ||||||
| def test_search_dates_multi_word_expression( | ||||||
| self, text, expected_text, expected_day, expected_month, description | ||||||
| ): | ||||||
| """Test parsing of multi-word date expressions in Russian.""" | ||||||
| result = search_dates(text, languages=["ru"]) | ||||||
| expected = [ | ||||||
| ( | ||||||
| expected_text, | ||||||
| datetime.datetime( | ||||||
| datetime.datetime.now().year, expected_month, expected_day, 0, 0 | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Using
Suggested change
Copilot uses AI. Check for mistakes. Positive FeedbackNegative Feedback |
||||||
| ), | ||||||
| ) | ||||||
| ] | ||||||
| self.assertEqual(result, expected) | ||||||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.