diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index da7316d0fe..9f3416bfa7 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -32,6 +32,12 @@ class PlagiarismScorer(MessageFloatScaleScorer): _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"]) + # Separators that are kept when they sit between two decimal digits, mapped to the + # form they take in the token. Dropping them would join the digits on either side, + # so "2.5" would collide with "25" and "2,500" with "2500". Fraction slashes fold + # to ASCII "/"; decimal points and thousands separators are kept as written. + _NUMERIC_SEPARATORS: dict[str, str] = {"/": "/", "\u2044": "/", ".": ".", ",": ","} + def __init__( self, *, @@ -90,16 +96,18 @@ def _tokenize(self, text: str) -> list[str]: Text is normalized to NFKC first so that compatibility forms (fullwidth letters, mathematical alphanumerics, decomposed accents) fold to a single - representation. Fraction separators between decimal digits are preserved - as ``/`` instead of joining the numerator and denominator. Compact - fractions remain separate from a preceding whole number. + representation. Numeric separators between decimal digits are preserved + instead of joining the digits on either side: fraction separators as ``/``, + and decimal points and thousands separators (``.`` and ``,``) as written, + so that ``2.5`` does not tokenize the same as ``25``. Compact fractions + remain separate from a preceding whole number. Combining marks are kept only after a retained word character or another attached mark, so meaningful vowel signs survive without keeping orphaned marks from removed emoji, punctuation, or whitespace. Returns: - list[str]: Lowercase word tokens with numeric fraction separators preserved. + list[str]: Lowercase word tokens with numeric separators preserved. """ text = self._normalize_text(text) characters: list[str] = [] @@ -116,12 +124,12 @@ def _tokenize(self, text: str) -> list[str]: if char.isspace(): characters.append(char) elif ( - char in {"/", "\u2044"} + char in self._NUMERIC_SEPARATORS and 0 < index < len(text) - 1 and text[index - 1].isdecimal() and text[index + 1].isdecimal() ): - characters.append("/") + characters.append(self._NUMERIC_SEPARATORS[char]) return "".join(characters).split() def _normalize_text(self, text: str) -> str: diff --git a/tests/unit/score/test_plagiarism_scorer.py b/tests/unit/score/test_plagiarism_scorer.py index 643262179b..7dfd9a0d35 100644 --- a/tests/unit/score/test_plagiarism_scorer.py +++ b/tests/unit/score/test_plagiarism_scorer.py @@ -289,7 +289,17 @@ async def test_score_text_async_integration(self): ) @pytest.mark.parametrize( ("reference", "response"), - [("½", "12"), ("¼", "14"), ("⅔", "23"), ("⅟2", "12"), ("1/2", "12")], + [ + ("½", "12"), + ("¼", "14"), + ("⅔", "23"), + ("⅟2", "12"), + ("1/2", "12"), + ("2.5", "25"), + ("25.00", "2500"), + ("2,500", "2500"), + ("25.00", "2,500"), + ], ) async def test_score_text_fraction_not_concatenated_async( self, @@ -411,6 +421,17 @@ def test_tokenize_keeps_combining_marks(self, scorer): ("a/", ["a"]), ("/a", ["a"]), ("a//b", ["ab"]), + ("2.5", ["2.5"]), + ("25.00", ["25.00"]), + ("2,500", ["2,500"]), + ("1.2.3", ["1.2.3"]), + ("2.5", ["2.5"]), + ("2.", ["2"]), + (".5", ["5"]), + ("2..5", ["25"]), + ("a.b", ["ab"]), + ("a,b", ["ab"]), + ("1,a", ["1a"]), ("a-\u0301b", ["ab"]), ("सिस्टम", ["सिस्टम"]), ("สวัสดี", ["สวัสดี"]),