Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions tests/test_normalizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,29 @@ def test_spelling_normalizer():
assert std("cancelation") == "cancellation"


@pytest.mark.parametrize("std", [EnglishNumberNormalizer(), EnglishTextNormalizer()])
@pytest.mark.parametrize(
"text, expected",
[
("the positive and negative effects", "the positive and negative effects"),
("the plus and minus signs", "the plus and minus signs"),
("a positive point of view", "a positive point of view"),
("a negative percent change", "a negative percent change"),
("plus dollars for shipping", "plus dollars for shipping"),
("positive double vision", "positive double vision"),
("positive and negative five", "positive and -5"),
("minus point five", "-.5"),
("negative point 5", "-.5"),
("minus double five", "-55"),
("plus triple oh", "+000"),
("positive twenty thousand", "+20000"),
("negative zero", "-0"),
],
)
def test_numeric_prefix_requires_a_number(std, text, expected):
assert std(text) == expected


def test_text_normalizer():
std = EnglishTextNormalizer()
assert std("Let's") == "let us"
Expand Down
23 changes: 21 additions & 2 deletions whisper/normalizers/english.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,12 @@ def __init__(self):
]
)
self.literal_words = {"one", "ones"}
self.number_starters = self.words.difference(
self.preceding_prefixers,
self.following_prefixers,
self.suffixers,
self.specials,
)

def process_words(self, words: List[str]) -> Iterator[str]:
prefix: Optional[str] = None
Expand All @@ -185,7 +191,9 @@ def output(result: Union[str, int]):
if len(words) == 0:
return

for prev, current, next in windowed([None] + words + [None], 3):
for prev, current, next, after_next in windowed(
[None] + words + [None, None], 4
):
if skip:
skip = False
continue
Expand Down Expand Up @@ -324,7 +332,18 @@ def output(result: Union[str, int]):
if value is not None:
yield output(value)

if next in self.words or next_is_numeric:
next_starts_number = next in self.number_starters or next_is_numeric
if next == "point":
next_starts_number = after_next in self.decimals or (
after_next is not None
and re.match(r"^\d+(\.\d+)?$", after_next)
)
elif next in {"double", "triple"}:
next_starts_number = (
after_next in self.ones or after_next in self.zeros
)

if next_starts_number:
prefix = self.preceding_prefixers[current]
else:
yield output(current)
Expand Down