Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
openSUSE:Factory
python-nltk
nltk-pr3207-py312.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File nltk-pr3207-py312.patch of Package python-nltk
From 25d35fc4283dedd2053ec6d821f4b707fff8d72c Mon Sep 17 00:00:00 2001 From: Konstantin Chernyshev <k4black@ya.ru> Date: Thu, 16 Nov 2023 19:00:15 +0100 Subject: [PATCH 1/8] ci: enable 3.12 in ci tests --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) Index: nltk-3.8.1/nltk/test/unit/translate/test_bleu.py =================================================================== --- nltk-3.8.1.orig/nltk/test/unit/translate/test_bleu.py +++ nltk-3.8.1/nltk/test/unit/translate/test_bleu.py @@ -2,7 +2,6 @@ Tests for BLEU translation evaluation metric """ -import io import unittest from nltk.data import find Index: nltk-3.8.1/nltk/translate/bleu_score.py =================================================================== --- nltk-3.8.1.orig/nltk/translate/bleu_score.py +++ nltk-3.8.1/nltk/translate/bleu_score.py @@ -1,685 +1,710 @@ -# Natural Language Toolkit: BLEU Score -# -# Copyright (C) 2001-2023 NLTK Project -# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim -# Contributors: Björn Mattsson, Dmitrijs Milajevs, Liling Tan -# URL: <https://www.nltk.org/> -# For license information, see LICENSE.TXT - -"""BLEU score implementation.""" - -import math -import sys -import warnings -from collections import Counter -from fractions import Fraction - -from nltk.util import ngrams - - -def sentence_bleu( - references, - hypothesis, - weights=(0.25, 0.25, 0.25, 0.25), - smoothing_function=None, - auto_reweigh=False, -): - """ - Calculate BLEU score (Bilingual Evaluation Understudy) from - Papineni, Kishore, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. - "BLEU: a method for automatic evaluation of machine translation." - In Proceedings of ACL. https://www.aclweb.org/anthology/P02-1040.pdf - - >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'military', 'always', - ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] - - >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops', - ... 'forever', 'hearing', 'the', 'activity', 'guidebook', - ... 'that', 'party', 'direct'] - - >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'military', 'will', 'forever', - ... 'heed', 'Party', 'commands'] - - >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'military', 'forces', 'always', - ... 'being', 'under', 'the', 'command', 'of', 'the', - ... 'Party'] - - >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'army', 'always', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'party'] - - >>> sentence_bleu([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS - 0.5045... - - If there is no ngrams overlap for any order of n-grams, BLEU returns the - value 0. This is because the precision for the order of n-grams without - overlap is 0, and the geometric mean in the final BLEU score computation - multiplies the 0 with the precision of other n-grams. This results in 0 - (independently of the precision of the other n-gram orders). The following - example has zero 3-gram and 4-gram overlaps: - - >>> round(sentence_bleu([reference1, reference2, reference3], hypothesis2),4) # doctest: +ELLIPSIS - 0.0 - - To avoid this harsh behaviour when no ngram overlaps are found a smoothing - function can be used. - - >>> chencherry = SmoothingFunction() - >>> sentence_bleu([reference1, reference2, reference3], hypothesis2, - ... smoothing_function=chencherry.method1) # doctest: +ELLIPSIS - 0.0370... - - The default BLEU calculates a score for up to 4-grams using uniform - weights (this is called BLEU-4). To evaluate your translations with - higher/lower order ngrams, use customized weights. E.g. when accounting - for up to 5-grams with uniform weights (this is called BLEU-5) use: - - >>> weights = (1./5., 1./5., 1./5., 1./5., 1./5.) - >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS - 0.3920... - - Multiple BLEU scores can be computed at once, by supplying a list of weights. - E.g. for computing BLEU-2, BLEU-3 *and* BLEU-4 in one computation, use: - >>> weights = [ - ... (1./2., 1./2.), - ... (1./3., 1./3., 1./3.), - ... (1./4., 1./4., 1./4., 1./4.) - ... ] - >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS - [0.7453..., 0.6240..., 0.5045...] - - :param references: reference sentences - :type references: list(list(str)) - :param hypothesis: a hypothesis sentence - :type hypothesis: list(str) - :param weights: weights for unigrams, bigrams, trigrams and so on (one or a list of weights) - :type weights: tuple(float) / list(tuple(float)) - :param smoothing_function: - :type smoothing_function: SmoothingFunction - :param auto_reweigh: Option to re-normalize the weights uniformly. - :type auto_reweigh: bool - :return: The sentence-level BLEU score. Returns a list if multiple weights were supplied. - :rtype: float / list(float) - """ - return corpus_bleu( - [references], [hypothesis], weights, smoothing_function, auto_reweigh - ) - - -def corpus_bleu( - list_of_references, - hypotheses, - weights=(0.25, 0.25, 0.25, 0.25), - smoothing_function=None, - auto_reweigh=False, -): - """ - Calculate a single corpus-level BLEU score (aka. system-level BLEU) for all - the hypotheses and their respective references. - - Instead of averaging the sentence level BLEU scores (i.e. macro-average - precision), the original BLEU metric (Papineni et al. 2002) accounts for - the micro-average precision (i.e. summing the numerators and denominators - for each hypothesis-reference(s) pairs before the division). - - >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'military', 'always', - ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] - >>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'military', 'will', 'forever', - ... 'heed', 'Party', 'commands'] - >>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'military', 'forces', 'always', - ... 'being', 'under', 'the', 'command', 'of', 'the', 'Party'] - >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'army', 'always', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'party'] - - >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', - ... 'interested', 'in', 'world', 'history'] - >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', - ... 'because', 'he', 'read', 'the', 'book'] - - >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] - >>> hypotheses = [hyp1, hyp2] - >>> corpus_bleu(list_of_references, hypotheses) # doctest: +ELLIPSIS - 0.5920... - - The example below show that corpus_bleu() is different from averaging - sentence_bleu() for hypotheses - - >>> score1 = sentence_bleu([ref1a, ref1b, ref1c], hyp1) - >>> score2 = sentence_bleu([ref2a], hyp2) - >>> (score1 + score2) / 2 # doctest: +ELLIPSIS - 0.6223... - - Custom weights may be supplied to fine-tune the BLEU score further. - A tuple of float weights for unigrams, bigrams, trigrams and so on can be given. - >>> weights = (0.1, 0.3, 0.5, 0.1) - >>> corpus_bleu(list_of_references, hypotheses, weights=weights) # doctest: +ELLIPSIS - 0.5818... - - This particular weight gave extra value to trigrams. - Furthermore, multiple weights can be given, resulting in multiple BLEU scores. - >>> weights = [ - ... (0.5, 0.5), - ... (0.333, 0.333, 0.334), - ... (0.25, 0.25, 0.25, 0.25), - ... (0.2, 0.2, 0.2, 0.2, 0.2) - ... ] - >>> corpus_bleu(list_of_references, hypotheses, weights=weights) # doctest: +ELLIPSIS - [0.8242..., 0.7067..., 0.5920..., 0.4719...] - - :param list_of_references: a corpus of lists of reference sentences, w.r.t. hypotheses - :type list_of_references: list(list(list(str))) - :param hypotheses: a list of hypothesis sentences - :type hypotheses: list(list(str)) - :param weights: weights for unigrams, bigrams, trigrams and so on (one or a list of weights) - :type weights: tuple(float) / list(tuple(float)) - :param smoothing_function: - :type smoothing_function: SmoothingFunction - :param auto_reweigh: Option to re-normalize the weights uniformly. - :type auto_reweigh: bool - :return: The corpus-level BLEU score. - :rtype: float - """ - # Before proceeding to compute BLEU, perform sanity checks. - - p_numerators = Counter() # Key = ngram order, and value = no. of ngram matches. - p_denominators = Counter() # Key = ngram order, and value = no. of ngram in ref. - hyp_lengths, ref_lengths = 0, 0 - - assert len(list_of_references) == len(hypotheses), ( - "The number of hypotheses and their reference(s) should be the " "same " - ) - - try: - weights[0][0] - except TypeError: - weights = [weights] - max_weight_length = max(len(weight) for weight in weights) - - # Iterate through each hypothesis and their corresponding references. - for references, hypothesis in zip(list_of_references, hypotheses): - # For each order of ngram, calculate the numerator and - # denominator for the corpus-level modified precision. - for i in range(1, max_weight_length + 1): - p_i = modified_precision(references, hypothesis, i) - p_numerators[i] += p_i.numerator - p_denominators[i] += p_i.denominator - - # Calculate the hypothesis length and the closest reference length. - # Adds them to the corpus-level hypothesis and reference counts. - hyp_len = len(hypothesis) - hyp_lengths += hyp_len - ref_lengths += closest_ref_length(references, hyp_len) - - # Calculate corpus-level brevity penalty. - bp = brevity_penalty(ref_lengths, hyp_lengths) - - # Collects the various precision values for the different ngram orders. - p_n = [ - Fraction(p_numerators[i], p_denominators[i], _normalize=False) - for i in range(1, max_weight_length + 1) - ] - - # Returns 0 if there's no matching n-grams - # We only need to check for p_numerators[1] == 0, since if there's - # no unigrams, there won't be any higher order ngrams. - if p_numerators[1] == 0: - return 0 if len(weights) == 1 else [0] * len(weights) - - # If there's no smoothing, set use method0 from SmoothinFunction class. - if not smoothing_function: - smoothing_function = SmoothingFunction().method0 - # Smoothen the modified precision. - # Note: smoothing_function() may convert values into floats; - # it tries to retain the Fraction object as much as the - # smoothing method allows. - p_n = smoothing_function( - p_n, references=references, hypothesis=hypothesis, hyp_len=hyp_lengths - ) - - bleu_scores = [] - for weight in weights: - # Uniformly re-weighting based on maximum hypothesis lengths if largest - # order of n-grams < 4 and weights is set at default. - if auto_reweigh: - if hyp_lengths < 4 and weight == (0.25, 0.25, 0.25, 0.25): - weight = (1 / hyp_lengths,) * hyp_lengths - - s = (w_i * math.log(p_i) for w_i, p_i in zip(weight, p_n) if p_i > 0) - s = bp * math.exp(math.fsum(s)) - bleu_scores.append(s) - return bleu_scores[0] if len(weights) == 1 else bleu_scores - - -def modified_precision(references, hypothesis, n): - """ - Calculate modified ngram precision. - - The normal precision method may lead to some wrong translations with - high-precision, e.g., the translation, in which a word of reference - repeats several times, has very high precision. - - This function only returns the Fraction object that contains the numerator - and denominator necessary to calculate the corpus-level precision. - To calculate the modified precision for a single pair of hypothesis and - references, cast the Fraction object into a float. - - The famous "the the the ... " example shows that you can get BLEU precision - by duplicating high frequency words. - - >>> reference1 = 'the cat is on the mat'.split() - >>> reference2 = 'there is a cat on the mat'.split() - >>> hypothesis1 = 'the the the the the the the'.split() - >>> references = [reference1, reference2] - >>> float(modified_precision(references, hypothesis1, n=1)) # doctest: +ELLIPSIS - 0.2857... - - In the modified n-gram precision, a reference word will be considered - exhausted after a matching hypothesis word is identified, e.g. - - >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'military', 'will', - ... 'forever', 'heed', 'Party', 'commands'] - >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'military', 'forces', 'always', - ... 'being', 'under', 'the', 'command', 'of', 'the', - ... 'Party'] - >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'army', 'always', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'party'] - >>> hypothesis = 'of the'.split() - >>> references = [reference1, reference2, reference3] - >>> float(modified_precision(references, hypothesis, n=1)) - 1.0 - >>> float(modified_precision(references, hypothesis, n=2)) - 1.0 - - An example of a normal machine translation hypothesis: - - >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', - ... 'ensures', 'that', 'the', 'military', 'always', - ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] - - >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops', - ... 'forever', 'hearing', 'the', 'activity', 'guidebook', - ... 'that', 'party', 'direct'] - - >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', - ... 'ensures', 'that', 'the', 'military', 'will', - ... 'forever', 'heed', 'Party', 'commands'] - - >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', - ... 'guarantees', 'the', 'military', 'forces', 'always', - ... 'being', 'under', 'the', 'command', 'of', 'the', - ... 'Party'] - - >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', - ... 'army', 'always', 'to', 'heed', 'the', 'directions', - ... 'of', 'the', 'party'] - >>> references = [reference1, reference2, reference3] - >>> float(modified_precision(references, hypothesis1, n=1)) # doctest: +ELLIPSIS - 0.9444... - >>> float(modified_precision(references, hypothesis2, n=1)) # doctest: +ELLIPSIS - 0.5714... - >>> float(modified_precision(references, hypothesis1, n=2)) # doctest: +ELLIPSIS - 0.5882352941176471 - >>> float(modified_precision(references, hypothesis2, n=2)) # doctest: +ELLIPSIS - 0.07692... - - - :param references: A list of reference translations. - :type references: list(list(str)) - :param hypothesis: A hypothesis translation. - :type hypothesis: list(str) - :param n: The ngram order. - :type n: int - :return: BLEU's modified precision for the nth order ngram. - :rtype: Fraction - """ - # Extracts all ngrams in hypothesis - # Set an empty Counter if hypothesis is empty. - counts = Counter(ngrams(hypothesis, n)) if len(hypothesis) >= n else Counter() - # Extract a union of references' counts. - # max_counts = reduce(or_, [Counter(ngrams(ref, n)) for ref in references]) - max_counts = {} - for reference in references: - reference_counts = ( - Counter(ngrams(reference, n)) if len(reference) >= n else Counter() - ) - for ngram in counts: - max_counts[ngram] = max(max_counts.get(ngram, 0), reference_counts[ngram]) - - # Assigns the intersection between hypothesis and references' counts. - clipped_counts = { - ngram: min(count, max_counts[ngram]) for ngram, count in counts.items() - } - - numerator = sum(clipped_counts.values()) - # Ensures that denominator is minimum 1 to avoid ZeroDivisionError. - # Usually this happens when the ngram order is > len(reference). - denominator = max(1, sum(counts.values())) - - return Fraction(numerator, denominator, _normalize=False) - - -def closest_ref_length(references, hyp_len): - """ - This function finds the reference that is the closest length to the - hypothesis. The closest reference length is referred to as *r* variable - from the brevity penalty formula in Papineni et. al. (2002) - - :param references: A list of reference translations. - :type references: list(list(str)) - :param hyp_len: The length of the hypothesis. - :type hyp_len: int - :return: The length of the reference that's closest to the hypothesis. - :rtype: int - """ - ref_lens = (len(reference) for reference in references) - closest_ref_len = min( - ref_lens, key=lambda ref_len: (abs(ref_len - hyp_len), ref_len) - ) - return closest_ref_len - - -def brevity_penalty(closest_ref_len, hyp_len): - """ - Calculate brevity penalty. - - As the modified n-gram precision still has the problem from the short - length sentence, brevity penalty is used to modify the overall BLEU - score according to length. - - An example from the paper. There are three references with length 12, 15 - and 17. And a concise hypothesis of the length 12. The brevity penalty is 1. - - >>> reference1 = list('aaaaaaaaaaaa') # i.e. ['a'] * 12 - >>> reference2 = list('aaaaaaaaaaaaaaa') # i.e. ['a'] * 15 - >>> reference3 = list('aaaaaaaaaaaaaaaaa') # i.e. ['a'] * 17 - >>> hypothesis = list('aaaaaaaaaaaa') # i.e. ['a'] * 12 - >>> references = [reference1, reference2, reference3] - >>> hyp_len = len(hypothesis) - >>> closest_ref_len = closest_ref_length(references, hyp_len) - >>> brevity_penalty(closest_ref_len, hyp_len) - 1.0 - - In case a hypothesis translation is shorter than the references, penalty is - applied. - - >>> references = [['a'] * 28, ['a'] * 28] - >>> hypothesis = ['a'] * 12 - >>> hyp_len = len(hypothesis) - >>> closest_ref_len = closest_ref_length(references, hyp_len) - >>> brevity_penalty(closest_ref_len, hyp_len) - 0.2635971381157267 - - The length of the closest reference is used to compute the penalty. If the - length of a hypothesis is 12, and the reference lengths are 13 and 2, the - penalty is applied because the hypothesis length (12) is less then the - closest reference length (13). - - >>> references = [['a'] * 13, ['a'] * 2] - >>> hypothesis = ['a'] * 12 - >>> hyp_len = len(hypothesis) - >>> closest_ref_len = closest_ref_length(references, hyp_len) - >>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS - 0.9200... - - The brevity penalty doesn't depend on reference order. More importantly, - when two reference sentences are at the same distance, the shortest - reference sentence length is used. - - >>> references = [['a'] * 13, ['a'] * 11] - >>> hypothesis = ['a'] * 12 - >>> hyp_len = len(hypothesis) - >>> closest_ref_len = closest_ref_length(references, hyp_len) - >>> bp1 = brevity_penalty(closest_ref_len, hyp_len) - >>> hyp_len = len(hypothesis) - >>> closest_ref_len = closest_ref_length(reversed(references), hyp_len) - >>> bp2 = brevity_penalty(closest_ref_len, hyp_len) - >>> bp1 == bp2 == 1 - True - - A test example from mteval-v13a.pl (starting from the line 705): - - >>> references = [['a'] * 11, ['a'] * 8] - >>> hypothesis = ['a'] * 7 - >>> hyp_len = len(hypothesis) - >>> closest_ref_len = closest_ref_length(references, hyp_len) - >>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS - 0.8668... - - >>> references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7] - >>> hypothesis = ['a'] * 7 - >>> hyp_len = len(hypothesis) - >>> closest_ref_len = closest_ref_length(references, hyp_len) - >>> brevity_penalty(closest_ref_len, hyp_len) - 1.0 - - :param hyp_len: The length of the hypothesis for a single sentence OR the - sum of all the hypotheses' lengths for a corpus - :type hyp_len: int - :param closest_ref_len: The length of the closest reference for a single - hypothesis OR the sum of all the closest references for every hypotheses. - :type closest_ref_len: int - :return: BLEU's brevity penalty. - :rtype: float - """ - if hyp_len > closest_ref_len: - return 1 - # If hypothesis is empty, brevity penalty = 0 should result in BLEU = 0.0 - elif hyp_len == 0: - return 0 - else: - return math.exp(1 - closest_ref_len / hyp_len) - - -class SmoothingFunction: - """ - This is an implementation of the smoothing techniques - for segment-level BLEU scores that was presented in - Boxing Chen and Collin Cherry (2014) A Systematic Comparison of - Smoothing Techniques for Sentence-Level BLEU. In WMT14. - http://acl2014.org/acl2014/W14-33/pdf/W14-3346.pdf - """ - - def __init__(self, epsilon=0.1, alpha=5, k=5): - """ - This will initialize the parameters required for the various smoothing - techniques, the default values are set to the numbers used in the - experiments from Chen and Cherry (2014). - - >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', 'ensures', - ... 'that', 'the', 'military', 'always', 'obeys', 'the', - ... 'commands', 'of', 'the', 'party'] - >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', 'ensures', - ... 'that', 'the', 'military', 'will', 'forever', 'heed', - ... 'Party', 'commands'] - - >>> chencherry = SmoothingFunction() - >>> print(sentence_bleu([reference1], hypothesis1)) # doctest: +ELLIPSIS - 0.4118... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method0)) # doctest: +ELLIPSIS - 0.4118... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method1)) # doctest: +ELLIPSIS - 0.4118... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method2)) # doctest: +ELLIPSIS - 0.4452... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method3)) # doctest: +ELLIPSIS - 0.4118... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method4)) # doctest: +ELLIPSIS - 0.4118... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method5)) # doctest: +ELLIPSIS - 0.4905... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method6)) # doctest: +ELLIPSIS - 0.4135... - >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method7)) # doctest: +ELLIPSIS - 0.4905... - - :param epsilon: the epsilon value use in method 1 - :type epsilon: float - :param alpha: the alpha value use in method 6 - :type alpha: int - :param k: the k value use in method 4 - :type k: int - """ - self.epsilon = epsilon - self.alpha = alpha - self.k = k - - def method0(self, p_n, *args, **kwargs): - """ - No smoothing. - """ - p_n_new = [] - for i, p_i in enumerate(p_n): - if p_i.numerator != 0: - p_n_new.append(p_i) - else: - _msg = str( - "\nThe hypothesis contains 0 counts of {}-gram overlaps.\n" - "Therefore the BLEU score evaluates to 0, independently of\n" - "how many N-gram overlaps of lower order it contains.\n" - "Consider using lower n-gram order or use " - "SmoothingFunction()" - ).format(i + 1) - warnings.warn(_msg) - # When numerator==0 where denonminator==0 or !=0, the result - # for the precision score should be equal to 0 or undefined. - # Due to BLEU geometric mean computation in logarithm space, - # we we need to take the return sys.float_info.min such that - # math.log(sys.float_info.min) returns a 0 precision score. - p_n_new.append(sys.float_info.min) - return p_n_new - - def method1(self, p_n, *args, **kwargs): - """ - Smoothing method 1: Add *epsilon* counts to precision with 0 counts. - """ - return [ - (p_i.numerator + self.epsilon) / p_i.denominator - if p_i.numerator == 0 - else p_i - for p_i in p_n - ] - - def method2(self, p_n, *args, **kwargs): - """ - Smoothing method 2: Add 1 to both numerator and denominator from - Chin-Yew Lin and Franz Josef Och (2004) ORANGE: a Method for - Evaluating Automatic Evaluation Metrics for Machine Translation. - In COLING 2004. - """ - return [ - Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1, _normalize=False) - if i != 0 - else p_n[0] - for i in range(len(p_n)) - ] - - def method3(self, p_n, *args, **kwargs): - """ - Smoothing method 3: NIST geometric sequence smoothing - The smoothing is computed by taking 1 / ( 2^k ), instead of 0, for each - precision score whose matching n-gram count is null. - k is 1 for the first 'n' value for which the n-gram match count is null/ - - For example, if the text contains: - - - one 2-gram match - - and (consequently) two 1-gram matches - - the n-gram count for each individual precision score would be: - - - n=1 => prec_count = 2 (two unigrams) - - n=2 => prec_count = 1 (one bigram) - - n=3 => prec_count = 1/2 (no trigram, taking 'smoothed' value of 1 / ( 2^k ), with k=1) - - n=4 => prec_count = 1/4 (no fourgram, taking 'smoothed' value of 1 / ( 2^k ), with k=2) - """ - incvnt = 1 # From the mteval-v13a.pl, it's referred to as k. - for i, p_i in enumerate(p_n): - if p_i.numerator == 0: - p_n[i] = 1 / (2**incvnt * p_i.denominator) - incvnt += 1 - return p_n - - def method4(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): - """ - Smoothing method 4: - Shorter translations may have inflated precision values due to having - smaller denominators; therefore, we give them proportionally - smaller smoothed counts. Instead of scaling to 1/(2^k), Chen and Cherry - suggests dividing by 1/ln(len(T)), where T is the length of the translation. - """ - incvnt = 1 - hyp_len = hyp_len if hyp_len else len(hypothesis) - for i, p_i in enumerate(p_n): - if p_i.numerator == 0 and hyp_len > 1: - # incvnt = i + 1 * self.k / math.log( - # hyp_len - # ) # Note that this K is different from the K from NIST. - # p_n[i] = incvnt / p_i.denominator\ - numerator = 1 / (2**incvnt * self.k / math.log(hyp_len)) - p_n[i] = numerator / p_i.denominator - incvnt += 1 - return p_n - - def method5(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): - """ - Smoothing method 5: - The matched counts for similar values of n should be similar. To a - calculate the n-gram matched count, it averages the n−1, n and n+1 gram - matched counts. - """ - hyp_len = hyp_len if hyp_len else len(hypothesis) - m = {} - # Requires an precision value for an addition ngram order. - p_n_plus1 = p_n + [modified_precision(references, hypothesis, 5)] - m[-1] = p_n[0] + 1 - for i, p_i in enumerate(p_n): - p_n[i] = (m[i - 1] + p_i + p_n_plus1[i + 1]) / 3 - m[i] = p_n[i] - return p_n - - def method6(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): - """ - Smoothing method 6: - Interpolates the maximum likelihood estimate of the precision *p_n* with - a prior estimate *pi0*. The prior is estimated by assuming that the ratio - between pn and pn−1 will be the same as that between pn−1 and pn−2; from - Gao and He (2013) Training MRF-Based Phrase Translation Models using - Gradient Ascent. In NAACL. - """ - hyp_len = hyp_len if hyp_len else len(hypothesis) - # This smoothing only works when p_1 and p_2 is non-zero. - # Raise an error with an appropriate message when the input is too short - # to use this smoothing technique. - assert p_n[2], "This smoothing method requires non-zero precision for bigrams." - for i, p_i in enumerate(p_n): - if i in [0, 1]: # Skips the first 2 orders of ngrams. - continue - else: - pi0 = 0 if p_n[i - 2] == 0 else p_n[i - 1] ** 2 / p_n[i - 2] - # No. of ngrams in translation that matches the reference. - m = p_i.numerator - # No. of ngrams in translation. - l = sum(1 for _ in ngrams(hypothesis, i + 1)) - # Calculates the interpolated precision. - p_n[i] = (m + self.alpha * pi0) / (l + self.alpha) - return p_n - - def method7(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): - """ - Smoothing method 7: - Interpolates methods 4 and 5. - """ - hyp_len = hyp_len if hyp_len else len(hypothesis) - p_n = self.method4(p_n, references, hypothesis, hyp_len) - p_n = self.method5(p_n, references, hypothesis, hyp_len) - return p_n +# Natural Language Toolkit: BLEU Score +# +# Copyright (C) 2001-2023 NLTK Project +# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim +# Contributors: Björn Mattsson, Dmitrijs Milajevs, Liling Tan +# URL: <https://www.nltk.org/> +# For license information, see LICENSE.TXT + +"""BLEU score implementation.""" +import math +import sys +import warnings +from collections import Counter +from fractions import Fraction as _Fraction + +from nltk.util import ngrams + + +class Fraction(_Fraction): + """Fraction with _normalize=False support for 3.12""" + + def __new__(cls, numerator=0, denominator=None, _normalize=False): + if sys.version_info >= (3, 12): + self = super().__new__(cls, numerator, denominator) + else: + self = super().__new__(cls, numerator, denominator, _normalize=_normalize) + self._normalize = _normalize + self._original_numerator = numerator + self._original_denominator = denominator + return self + + @property + def numerator(self): + if not self._normalize: + return self._original_numerator + return super().numerator + + @property + def denominator(self): + if not self._normalize: + return self._original_denominator + return super().denominator + + +def sentence_bleu( + references, + hypothesis, + weights=(0.25, 0.25, 0.25, 0.25), + smoothing_function=None, + auto_reweigh=False, +): + """ + Calculate BLEU score (Bilingual Evaluation Understudy) from + Papineni, Kishore, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. + "BLEU: a method for automatic evaluation of machine translation." + In Proceedings of ACL. https://www.aclweb.org/anthology/P02-1040.pdf + + >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', + ... 'ensures', 'that', 'the', 'military', 'always', + ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] + + >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops', + ... 'forever', 'hearing', 'the', 'activity', 'guidebook', + ... 'that', 'party', 'direct'] + + >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', + ... 'ensures', 'that', 'the', 'military', 'will', 'forever', + ... 'heed', 'Party', 'commands'] + + >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', + ... 'guarantees', 'the', 'military', 'forces', 'always', + ... 'being', 'under', 'the', 'command', 'of', 'the', + ... 'Party'] + + >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', + ... 'army', 'always', 'to', 'heed', 'the', 'directions', + ... 'of', 'the', 'party'] + + >>> sentence_bleu([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS + 0.5045... + + If there is no ngrams overlap for any order of n-grams, BLEU returns the + value 0. This is because the precision for the order of n-grams without + overlap is 0, and the geometric mean in the final BLEU score computation + multiplies the 0 with the precision of other n-grams. This results in 0 + (independently of the precision of the other n-gram orders). The following + example has zero 3-gram and 4-gram overlaps: + + >>> round(sentence_bleu([reference1, reference2, reference3], hypothesis2),4) # doctest: +ELLIPSIS + 0.0 + + To avoid this harsh behaviour when no ngram overlaps are found a smoothing + function can be used. + + >>> chencherry = SmoothingFunction() + >>> sentence_bleu([reference1, reference2, reference3], hypothesis2, + ... smoothing_function=chencherry.method1) # doctest: +ELLIPSIS + 0.0370... + + The default BLEU calculates a score for up to 4-grams using uniform + weights (this is called BLEU-4). To evaluate your translations with + higher/lower order ngrams, use customized weights. E.g. when accounting + for up to 5-grams with uniform weights (this is called BLEU-5) use: + + >>> weights = (1./5., 1./5., 1./5., 1./5., 1./5.) + >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS + 0.3920... + + Multiple BLEU scores can be computed at once, by supplying a list of weights. + E.g. for computing BLEU-2, BLEU-3 *and* BLEU-4 in one computation, use: + >>> weights = [ + ... (1./2., 1./2.), + ... (1./3., 1./3., 1./3.), + ... (1./4., 1./4., 1./4., 1./4.) + ... ] + >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS + [0.7453..., 0.6240..., 0.5045...] + + :param references: reference sentences + :type references: list(list(str)) + :param hypothesis: a hypothesis sentence + :type hypothesis: list(str) + :param weights: weights for unigrams, bigrams, trigrams and so on (one or a list of weights) + :type weights: tuple(float) / list(tuple(float)) + :param smoothing_function: + :type smoothing_function: SmoothingFunction + :param auto_reweigh: Option to re-normalize the weights uniformly. + :type auto_reweigh: bool + :return: The sentence-level BLEU score. Returns a list if multiple weights were supplied. + :rtype: float / list(float) + """ + return corpus_bleu( + [references], [hypothesis], weights, smoothing_function, auto_reweigh + ) + + +def corpus_bleu( + list_of_references, + hypotheses, + weights=(0.25, 0.25, 0.25, 0.25), + smoothing_function=None, + auto_reweigh=False, +): + """ + Calculate a single corpus-level BLEU score (aka. system-level BLEU) for all + the hypotheses and their respective references. + + Instead of averaging the sentence level BLEU scores (i.e. macro-average + precision), the original BLEU metric (Papineni et al. 2002) accounts for + the micro-average precision (i.e. summing the numerators and denominators + for each hypothesis-reference(s) pairs before the division). + + >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', + ... 'ensures', 'that', 'the', 'military', 'always', + ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] + >>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', + ... 'ensures', 'that', 'the', 'military', 'will', 'forever', + ... 'heed', 'Party', 'commands'] + >>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which', + ... 'guarantees', 'the', 'military', 'forces', 'always', + ... 'being', 'under', 'the', 'command', 'of', 'the', 'Party'] + >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', + ... 'army', 'always', 'to', 'heed', 'the', 'directions', + ... 'of', 'the', 'party'] + + >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', + ... 'interested', 'in', 'world', 'history'] + >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', + ... 'because', 'he', 'read', 'the', 'book'] + + >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] + >>> hypotheses = [hyp1, hyp2] + >>> corpus_bleu(list_of_references, hypotheses) # doctest: +ELLIPSIS + 0.5920... + + The example below show that corpus_bleu() is different from averaging + sentence_bleu() for hypotheses + + >>> score1 = sentence_bleu([ref1a, ref1b, ref1c], hyp1) + >>> score2 = sentence_bleu([ref2a], hyp2) + >>> (score1 + score2) / 2 # doctest: +ELLIPSIS + 0.6223... + + Custom weights may be supplied to fine-tune the BLEU score further. + A tuple of float weights for unigrams, bigrams, trigrams and so on can be given. + >>> weights = (0.1, 0.3, 0.5, 0.1) + >>> corpus_bleu(list_of_references, hypotheses, weights=weights) # doctest: +ELLIPSIS + 0.5818... + + This particular weight gave extra value to trigrams. + Furthermore, multiple weights can be given, resulting in multiple BLEU scores. + >>> weights = [ + ... (0.5, 0.5), + ... (0.333, 0.333, 0.334), + ... (0.25, 0.25, 0.25, 0.25), + ... (0.2, 0.2, 0.2, 0.2, 0.2) + ... ] + >>> corpus_bleu(list_of_references, hypotheses, weights=weights) # doctest: +ELLIPSIS + [0.8242..., 0.7067..., 0.5920..., 0.4719...] + + :param list_of_references: a corpus of lists of reference sentences, w.r.t. hypotheses + :type list_of_references: list(list(list(str))) + :param hypotheses: a list of hypothesis sentences + :type hypotheses: list(list(str)) + :param weights: weights for unigrams, bigrams, trigrams and so on (one or a list of weights) + :type weights: tuple(float) / list(tuple(float)) + :param smoothing_function: + :type smoothing_function: SmoothingFunction + :param auto_reweigh: Option to re-normalize the weights uniformly. + :type auto_reweigh: bool + :return: The corpus-level BLEU score. + :rtype: float + """ + # Before proceeding to compute BLEU, perform sanity checks. + + p_numerators = Counter() # Key = ngram order, and value = no. of ngram matches. + p_denominators = Counter() # Key = ngram order, and value = no. of ngram in ref. + hyp_lengths, ref_lengths = 0, 0 + + assert len(list_of_references) == len(hypotheses), ( + "The number of hypotheses and their reference(s) should be the " "same " + ) + + try: + weights[0][0] + except TypeError: + weights = [weights] + max_weight_length = max(len(weight) for weight in weights) + + # Iterate through each hypothesis and their corresponding references. + for references, hypothesis in zip(list_of_references, hypotheses): + # For each order of ngram, calculate the numerator and + # denominator for the corpus-level modified precision. + for i in range(1, max_weight_length + 1): + p_i = modified_precision(references, hypothesis, i) + p_numerators[i] += p_i.numerator + p_denominators[i] += p_i.denominator + + # Calculate the hypothesis length and the closest reference length. + # Adds them to the corpus-level hypothesis and reference counts. + hyp_len = len(hypothesis) + hyp_lengths += hyp_len + ref_lengths += closest_ref_length(references, hyp_len) + + # Calculate corpus-level brevity penalty. + bp = brevity_penalty(ref_lengths, hyp_lengths) + + # Collects the various precision values for the different ngram orders. + p_n = [ + Fraction(p_numerators[i], p_denominators[i], _normalize=False) + for i in range(1, max_weight_length + 1) + ] + + # Returns 0 if there's no matching n-grams + # We only need to check for p_numerators[1] == 0, since if there's + # no unigrams, there won't be any higher order ngrams. + if p_numerators[1] == 0: + return 0 if len(weights) == 1 else [0] * len(weights) + + # If there's no smoothing, set use method0 from SmoothinFunction class. + if not smoothing_function: + smoothing_function = SmoothingFunction().method0 + # Smoothen the modified precision. + # Note: smoothing_function() may convert values into floats; + # it tries to retain the Fraction object as much as the + # smoothing method allows. + p_n = smoothing_function( + p_n, references=references, hypothesis=hypothesis, hyp_len=hyp_lengths + ) + + bleu_scores = [] + for weight in weights: + # Uniformly re-weighting based on maximum hypothesis lengths if largest + # order of n-grams < 4 and weights is set at default. + if auto_reweigh: + if hyp_lengths < 4 and weight == (0.25, 0.25, 0.25, 0.25): + weight = (1 / hyp_lengths,) * hyp_lengths + + s = (w_i * math.log(p_i) for w_i, p_i in zip(weight, p_n) if p_i > 0) + s = bp * math.exp(math.fsum(s)) + bleu_scores.append(s) + return bleu_scores[0] if len(weights) == 1 else bleu_scores + + +def modified_precision(references, hypothesis, n): + """ + Calculate modified ngram precision. + + The normal precision method may lead to some wrong translations with + high-precision, e.g., the translation, in which a word of reference + repeats several times, has very high precision. + + This function only returns the Fraction object that contains the numerator + and denominator necessary to calculate the corpus-level precision. + To calculate the modified precision for a single pair of hypothesis and + references, cast the Fraction object into a float. + + The famous "the the the ... " example shows that you can get BLEU precision + by duplicating high frequency words. + + >>> reference1 = 'the cat is on the mat'.split() + >>> reference2 = 'there is a cat on the mat'.split() + >>> hypothesis1 = 'the the the the the the the'.split() + >>> references = [reference1, reference2] + >>> float(modified_precision(references, hypothesis1, n=1)) # doctest: +ELLIPSIS + 0.2857... + + In the modified n-gram precision, a reference word will be considered + exhausted after a matching hypothesis word is identified, e.g. + + >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', + ... 'ensures', 'that', 'the', 'military', 'will', + ... 'forever', 'heed', 'Party', 'commands'] + >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', + ... 'guarantees', 'the', 'military', 'forces', 'always', + ... 'being', 'under', 'the', 'command', 'of', 'the', + ... 'Party'] + >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', + ... 'army', 'always', 'to', 'heed', 'the', 'directions', + ... 'of', 'the', 'party'] + >>> hypothesis = 'of the'.split() + >>> references = [reference1, reference2, reference3] + >>> float(modified_precision(references, hypothesis, n=1)) + 1.0 + >>> float(modified_precision(references, hypothesis, n=2)) + 1.0 + + An example of a normal machine translation hypothesis: + + >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', + ... 'ensures', 'that', 'the', 'military', 'always', + ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] + + >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops', + ... 'forever', 'hearing', 'the', 'activity', 'guidebook', + ... 'that', 'party', 'direct'] + + >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', + ... 'ensures', 'that', 'the', 'military', 'will', + ... 'forever', 'heed', 'Party', 'commands'] + + >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', + ... 'guarantees', 'the', 'military', 'forces', 'always', + ... 'being', 'under', 'the', 'command', 'of', 'the', + ... 'Party'] + + >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', + ... 'army', 'always', 'to', 'heed', 'the', 'directions', + ... 'of', 'the', 'party'] + >>> references = [reference1, reference2, reference3] + >>> float(modified_precision(references, hypothesis1, n=1)) # doctest: +ELLIPSIS + 0.9444... + >>> float(modified_precision(references, hypothesis2, n=1)) # doctest: +ELLIPSIS + 0.5714... + >>> float(modified_precision(references, hypothesis1, n=2)) # doctest: +ELLIPSIS + 0.5882352941176471 + >>> float(modified_precision(references, hypothesis2, n=2)) # doctest: +ELLIPSIS + 0.07692... + + + :param references: A list of reference translations. + :type references: list(list(str)) + :param hypothesis: A hypothesis translation. + :type hypothesis: list(str) + :param n: The ngram order. + :type n: int + :return: BLEU's modified precision for the nth order ngram. + :rtype: Fraction + """ + # Extracts all ngrams in hypothesis + # Set an empty Counter if hypothesis is empty. + counts = Counter(ngrams(hypothesis, n)) if len(hypothesis) >= n else Counter() + # Extract a union of references' counts. + # max_counts = reduce(or_, [Counter(ngrams(ref, n)) for ref in references]) + max_counts = {} + for reference in references: + reference_counts = ( + Counter(ngrams(reference, n)) if len(reference) >= n else Counter() + ) + for ngram in counts: + max_counts[ngram] = max(max_counts.get(ngram, 0), reference_counts[ngram]) + + # Assigns the intersection between hypothesis and references' counts. + clipped_counts = { + ngram: min(count, max_counts[ngram]) for ngram, count in counts.items() + } + + numerator = sum(clipped_counts.values()) + # Ensures that denominator is minimum 1 to avoid ZeroDivisionError. + # Usually this happens when the ngram order is > len(reference). + denominator = max(1, sum(counts.values())) + + return Fraction(numerator, denominator, _normalize=False) + + +def closest_ref_length(references, hyp_len): + """ + This function finds the reference that is the closest length to the + hypothesis. The closest reference length is referred to as *r* variable + from the brevity penalty formula in Papineni et. al. (2002) + + :param references: A list of reference translations. + :type references: list(list(str)) + :param hyp_len: The length of the hypothesis. + :type hyp_len: int + :return: The length of the reference that's closest to the hypothesis. + :rtype: int + """ + ref_lens = (len(reference) for reference in references) + closest_ref_len = min( + ref_lens, key=lambda ref_len: (abs(ref_len - hyp_len), ref_len) + ) + return closest_ref_len + + +def brevity_penalty(closest_ref_len, hyp_len): + """ + Calculate brevity penalty. + + As the modified n-gram precision still has the problem from the short + length sentence, brevity penalty is used to modify the overall BLEU + score according to length. + + An example from the paper. There are three references with length 12, 15 + and 17. And a concise hypothesis of the length 12. The brevity penalty is 1. + + >>> reference1 = list('aaaaaaaaaaaa') # i.e. ['a'] * 12 + >>> reference2 = list('aaaaaaaaaaaaaaa') # i.e. ['a'] * 15 + >>> reference3 = list('aaaaaaaaaaaaaaaaa') # i.e. ['a'] * 17 + >>> hypothesis = list('aaaaaaaaaaaa') # i.e. ['a'] * 12 + >>> references = [reference1, reference2, reference3] + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) + 1.0 + + In case a hypothesis translation is shorter than the references, penalty is + applied. + + >>> references = [['a'] * 28, ['a'] * 28] + >>> hypothesis = ['a'] * 12 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) + 0.2635971381157267 + + The length of the closest reference is used to compute the penalty. If the + length of a hypothesis is 12, and the reference lengths are 13 and 2, the + penalty is applied because the hypothesis length (12) is less then the + closest reference length (13). + + >>> references = [['a'] * 13, ['a'] * 2] + >>> hypothesis = ['a'] * 12 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS + 0.9200... + + The brevity penalty doesn't depend on reference order. More importantly, + when two reference sentences are at the same distance, the shortest + reference sentence length is used. + + >>> references = [['a'] * 13, ['a'] * 11] + >>> hypothesis = ['a'] * 12 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> bp1 = brevity_penalty(closest_ref_len, hyp_len) + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(reversed(references), hyp_len) + >>> bp2 = brevity_penalty(closest_ref_len, hyp_len) + >>> bp1 == bp2 == 1 + True + + A test example from mteval-v13a.pl (starting from the line 705): + + >>> references = [['a'] * 11, ['a'] * 8] + >>> hypothesis = ['a'] * 7 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS + 0.8668... + + >>> references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7] + >>> hypothesis = ['a'] * 7 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) + 1.0 + + :param hyp_len: The length of the hypothesis for a single sentence OR the + sum of all the hypotheses' lengths for a corpus + :type hyp_len: int + :param closest_ref_len: The length of the closest reference for a single + hypothesis OR the sum of all the closest references for every hypotheses. + :type closest_ref_len: int + :return: BLEU's brevity penalty. + :rtype: float + """ + if hyp_len > closest_ref_len: + return 1 + # If hypothesis is empty, brevity penalty = 0 should result in BLEU = 0.0 + elif hyp_len == 0: + return 0 + else: + return math.exp(1 - closest_ref_len / hyp_len) + + +class SmoothingFunction: + """ + This is an implementation of the smoothing techniques + for segment-level BLEU scores that was presented in + Boxing Chen and Collin Cherry (2014) A Systematic Comparison of + Smoothing Techniques for Sentence-Level BLEU. In WMT14. + http://acl2014.org/acl2014/W14-33/pdf/W14-3346.pdf + """ + + def __init__(self, epsilon=0.1, alpha=5, k=5): + """ + This will initialize the parameters required for the various smoothing + techniques, the default values are set to the numbers used in the + experiments from Chen and Cherry (2014). + + >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', 'ensures', + ... 'that', 'the', 'military', 'always', 'obeys', 'the', + ... 'commands', 'of', 'the', 'party'] + >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', 'ensures', + ... 'that', 'the', 'military', 'will', 'forever', 'heed', + ... 'Party', 'commands'] + + >>> chencherry = SmoothingFunction() + >>> print(sentence_bleu([reference1], hypothesis1)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method0)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method1)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method2)) # doctest: +ELLIPSIS + 0.4452... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method3)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method4)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method5)) # doctest: +ELLIPSIS + 0.4905... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method6)) # doctest: +ELLIPSIS + 0.4135... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method7)) # doctest: +ELLIPSIS + 0.4905... + + :param epsilon: the epsilon value use in method 1 + :type epsilon: float + :param alpha: the alpha value use in method 6 + :type alpha: int + :param k: the k value use in method 4 + :type k: int + """ + self.epsilon = epsilon + self.alpha = alpha + self.k = k + + def method0(self, p_n, *args, **kwargs): + """ + No smoothing. + """ + p_n_new = [] + for i, p_i in enumerate(p_n): + if p_i.numerator != 0: + p_n_new.append(p_i) + else: + _msg = str( + "\nThe hypothesis contains 0 counts of {}-gram overlaps.\n" + "Therefore the BLEU score evaluates to 0, independently of\n" + "how many N-gram overlaps of lower order it contains.\n" + "Consider using lower n-gram order or use " + "SmoothingFunction()" + ).format(i + 1) + warnings.warn(_msg) + # When numerator==0 where denonminator==0 or !=0, the result + # for the precision score should be equal to 0 or undefined. + # Due to BLEU geometric mean computation in logarithm space, + # we we need to take the return sys.float_info.min such that + # math.log(sys.float_info.min) returns a 0 precision score. + p_n_new.append(sys.float_info.min) + return p_n_new + + def method1(self, p_n, *args, **kwargs): + """ + Smoothing method 1: Add *epsilon* counts to precision with 0 counts. + """ + return [ + (p_i.numerator + self.epsilon) / p_i.denominator + if p_i.numerator == 0 + else p_i + for p_i in p_n + ] + + def method2(self, p_n, *args, **kwargs): + """ + Smoothing method 2: Add 1 to both numerator and denominator from + Chin-Yew Lin and Franz Josef Och (2004) ORANGE: a Method for + Evaluating Automatic Evaluation Metrics for Machine Translation. + In COLING 2004. + """ + return [ + Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1, _normalize=False) + if i != 0 + else p_n[0] + for i in range(len(p_n)) + ] + + def method3(self, p_n, *args, **kwargs): + """ + Smoothing method 3: NIST geometric sequence smoothing + The smoothing is computed by taking 1 / ( 2^k ), instead of 0, for each + precision score whose matching n-gram count is null. + k is 1 for the first 'n' value for which the n-gram match count is null/ + + For example, if the text contains: + + - one 2-gram match + - and (consequently) two 1-gram matches + + the n-gram count for each individual precision score would be: + + - n=1 => prec_count = 2 (two unigrams) + - n=2 => prec_count = 1 (one bigram) + - n=3 => prec_count = 1/2 (no trigram, taking 'smoothed' value of 1 / ( 2^k ), with k=1) + - n=4 => prec_count = 1/4 (no fourgram, taking 'smoothed' value of 1 / ( 2^k ), with k=2) + """ + incvnt = 1 # From the mteval-v13a.pl, it's referred to as k. + for i, p_i in enumerate(p_n): + if p_i.numerator == 0: + p_n[i] = 1 / (2**incvnt * p_i.denominator) + incvnt += 1 + return p_n + + def method4(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 4: + Shorter translations may have inflated precision values due to having + smaller denominators; therefore, we give them proportionally + smaller smoothed counts. Instead of scaling to 1/(2^k), Chen and Cherry + suggests dividing by 1/ln(len(T)), where T is the length of the translation. + """ + incvnt = 1 + hyp_len = hyp_len if hyp_len else len(hypothesis) + for i, p_i in enumerate(p_n): + if p_i.numerator == 0 and hyp_len > 1: + # incvnt = i + 1 * self.k / math.log( + # hyp_len + # ) # Note that this K is different from the K from NIST. + # p_n[i] = incvnt / p_i.denominator\ + numerator = 1 / (2**incvnt * self.k / math.log(hyp_len)) + p_n[i] = numerator / p_i.denominator + incvnt += 1 + return p_n + + def method5(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 5: + The matched counts for similar values of n should be similar. To a + calculate the n-gram matched count, it averages the n−1, n and n+1 gram + matched counts. + """ + hyp_len = hyp_len if hyp_len else len(hypothesis) + m = {} + # Requires an precision value for an addition ngram order. + p_n_plus1 = p_n + [modified_precision(references, hypothesis, 5)] + m[-1] = p_n[0] + 1 + for i, p_i in enumerate(p_n): + p_n[i] = (m[i - 1] + p_i + p_n_plus1[i + 1]) / 3 + m[i] = p_n[i] + return p_n + + def method6(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 6: + Interpolates the maximum likelihood estimate of the precision *p_n* with + a prior estimate *pi0*. The prior is estimated by assuming that the ratio + between pn and pn−1 will be the same as that between pn−1 and pn−2; from + Gao and He (2013) Training MRF-Based Phrase Translation Models using + Gradient Ascent. In NAACL. + """ + hyp_len = hyp_len if hyp_len else len(hypothesis) + # This smoothing only works when p_1 and p_2 is non-zero. + # Raise an error with an appropriate message when the input is too short + # to use this smoothing technique. + assert p_n[2], "This smoothing method requires non-zero precision for bigrams." + for i, p_i in enumerate(p_n): + if i in [0, 1]: # Skips the first 2 orders of ngrams. + continue + else: + pi0 = 0 if p_n[i - 2] == 0 else p_n[i - 1] ** 2 / p_n[i - 2] + # No. of ngrams in translation that matches the reference. + m = p_i.numerator + # No. of ngrams in translation. + l = sum(1 for _ in ngrams(hypothesis, i + 1)) + # Calculates the interpolated precision. + p_n[i] = (m + self.alpha * pi0) / (l + self.alpha) + return p_n + + def method7(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 7: + Interpolates methods 4 and 5. + """ + hyp_len = hyp_len if hyp_len else len(hypothesis) + p_n = self.method4(p_n, references, hypothesis, hyp_len) + p_n = self.method5(p_n, references, hypothesis, hyp_len) + return p_n Index: nltk-3.8.1/README.md =================================================================== --- nltk-3.8.1.orig/README.md +++ nltk-3.8.1/README.md @@ -1,50 +1,50 @@ -# Natural Language Toolkit (NLTK) -[](https://pypi.python.org/pypi/nltk) - - -NLTK -- the Natural Language Toolkit -- is a suite of open source Python -modules, data sets, and tutorials supporting research and development in Natural -Language Processing. NLTK requires Python version 3.7, 3.8, 3.9, 3.10 or 3.11. - -For documentation, please visit [nltk.org](https://www.nltk.org/). - - -## Contributing - -Do you want to contribute to NLTK development? Great! -Please read [CONTRIBUTING.md](CONTRIBUTING.md) for more details. - -See also [how to contribute to NLTK](https://www.nltk.org/contribute.html). - - -## Donate - -Have you found the toolkit helpful? Please support NLTK development by donating -to the project via PayPal, using the link on the NLTK homepage. - - -## Citing - -If you publish work that uses NLTK, please cite the NLTK book, as follows: - - Bird, Steven, Edward Loper and Ewan Klein (2009). - Natural Language Processing with Python. O'Reilly Media Inc. - - -## Copyright - -Copyright (C) 2001-2023 NLTK Project - -For license information, see [LICENSE.txt](LICENSE.txt). - -[AUTHORS.md](AUTHORS.md) contains a list of everyone who has contributed to NLTK. - - -### Redistributing - -- NLTK source code is distributed under the Apache 2.0 License. -- NLTK documentation is distributed under the Creative Commons - Attribution-Noncommercial-No Derivative Works 3.0 United States license. -- NLTK corpora are provided under the terms given in the README file for each - corpus; all are redistributable and available for non-commercial use. -- NLTK may be freely redistributed, subject to the provisions of these licenses. +# Natural Language Toolkit (NLTK) +[](https://pypi.python.org/pypi/nltk) + + +NLTK -- the Natural Language Toolkit -- is a suite of open source Python +modules, data sets, and tutorials supporting research and development in Natural +Language Processing. NLTK requires Python version 3.7, 3.8, 3.9, 3.10, 3.11 or 3.12. + +For documentation, please visit [nltk.org](https://www.nltk.org/). + + +## Contributing + +Do you want to contribute to NLTK development? Great! +Please read [CONTRIBUTING.md](CONTRIBUTING.md) for more details. + +See also [how to contribute to NLTK](https://www.nltk.org/contribute.html). + + +## Donate + +Have you found the toolkit helpful? Please support NLTK development by donating +to the project via PayPal, using the link on the NLTK homepage. + + +## Citing + +If you publish work that uses NLTK, please cite the NLTK book, as follows: + + Bird, Steven, Edward Loper and Ewan Klein (2009). + Natural Language Processing with Python. O'Reilly Media Inc. + + +## Copyright + +Copyright (C) 2001-2023 NLTK Project + +For license information, see [LICENSE.txt](LICENSE.txt). + +[AUTHORS.md](AUTHORS.md) contains a list of everyone who has contributed to NLTK. + + +### Redistributing + +- NLTK source code is distributed under the Apache 2.0 License. +- NLTK documentation is distributed under the Creative Commons + Attribution-Noncommercial-No Derivative Works 3.0 United States license. +- NLTK corpora are provided under the terms given in the README file for each + corpus; all are redistributable and available for non-commercial use. +- NLTK may be freely redistributed, subject to the provisions of these licenses. Index: nltk-3.8.1/setup.py =================================================================== --- nltk-3.8.1.orig/setup.py +++ nltk-3.8.1/setup.py @@ -1,125 +1,126 @@ -#!/usr/bin/env python -# -# Setup script for the Natural Language Toolkit -# -# Copyright (C) 2001-2023 NLTK Project -# Author: NLTK Team <nltk.team@gmail.com> -# URL: <https://www.nltk.org/> -# For license information, see LICENSE.TXT - -# Work around mbcs bug in distutils. -# https://bugs.python.org/issue10945 -import codecs - -try: - codecs.lookup("mbcs") -except LookupError: - ascii = codecs.lookup("ascii") - func = lambda name, enc=ascii: {True: enc}.get(name == "mbcs") - codecs.register(func) - -import os - -# Use the VERSION file to get NLTK version -version_file = os.path.join(os.path.dirname(__file__), "nltk", "VERSION") -with open(version_file) as fh: - nltk_version = fh.read().strip() - -# setuptools -from setuptools import find_packages, setup - -# Specify groups of optional dependencies -extras_require = { - "machine_learning": [ - "numpy", - "python-crfsuite", - "scikit-learn", - "scipy", - ], - "plot": ["matplotlib"], - "tgrep": ["pyparsing"], - "twitter": ["twython"], - "corenlp": ["requests"], -} - -# Add a group made up of all optional dependencies -extras_require["all"] = { - package for group in extras_require.values() for package in group -} - -# Adds CLI commands -console_scripts = """ -[console_scripts] -nltk=nltk.cli:cli -""" - -_project_homepage = "https://www.nltk.org/" - -setup( - name="nltk", - description="Natural Language Toolkit", - version=nltk_version, - url=_project_homepage, - project_urls={ - "Documentation": _project_homepage, - "Source Code": "https://github.com/nltk/nltk", - "Issue Tracker": "https://github.com/nltk/nltk/issues", - }, - long_description="""\ -The Natural Language Toolkit (NLTK) is a Python package for -natural language processing. NLTK requires Python 3.7, 3.8, 3.9, 3.10 or 3.11.""", - license="Apache License, Version 2.0", - keywords=[ - "NLP", - "CL", - "natural language processing", - "computational linguistics", - "parsing", - "tagging", - "tokenizing", - "syntax", - "linguistics", - "language", - "natural language", - "text analytics", - ], - maintainer="NLTK Team", - maintainer_email="nltk.team@gmail.com", - author="NLTK Team", - author_email="nltk.team@gmail.com", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Scientific/Engineering :: Human Machine Interfaces", - "Topic :: Scientific/Engineering :: Information Analysis", - "Topic :: Text Processing", - "Topic :: Text Processing :: Filters", - "Topic :: Text Processing :: General", - "Topic :: Text Processing :: Indexing", - "Topic :: Text Processing :: Linguistic", - ], - package_data={"nltk": ["test/*.doctest", "VERSION"]}, - python_requires=">=3.7", - install_requires=[ - "click", - "joblib", - "regex>=2021.8.3", - "tqdm", - ], - extras_require=extras_require, - packages=find_packages(), - zip_safe=False, # since normal files will be present too? - entry_points=console_scripts, -) +#!/usr/bin/env python +# +# Setup script for the Natural Language Toolkit +# +# Copyright (C) 2001-2023 NLTK Project +# Author: NLTK Team <nltk.team@gmail.com> +# URL: <https://www.nltk.org/> +# For license information, see LICENSE.TXT + +# Work around mbcs bug in distutils. +# https://bugs.python.org/issue10945 +import codecs + +try: + codecs.lookup("mbcs") +except LookupError: + ascii = codecs.lookup("ascii") + func = lambda name, enc=ascii: {True: enc}.get(name == "mbcs") + codecs.register(func) + +import os + +# Use the VERSION file to get NLTK version +version_file = os.path.join(os.path.dirname(__file__), "nltk", "VERSION") +with open(version_file) as fh: + nltk_version = fh.read().strip() + +# setuptools +from setuptools import find_packages, setup + +# Specify groups of optional dependencies +extras_require = { + "machine_learning": [ + "numpy", + "python-crfsuite", + "scikit-learn", + "scipy", + ], + "plot": ["matplotlib"], + "tgrep": ["pyparsing"], + "twitter": ["twython"], + "corenlp": ["requests"], +} + +# Add a group made up of all optional dependencies +extras_require["all"] = { + package for group in extras_require.values() for package in group +} + +# Adds CLI commands +console_scripts = """ +[console_scripts] +nltk=nltk.cli:cli +""" + +_project_homepage = "https://www.nltk.org/" + +setup( + name="nltk", + description="Natural Language Toolkit", + version=nltk_version, + url=_project_homepage, + project_urls={ + "Documentation": _project_homepage, + "Source Code": "https://github.com/nltk/nltk", + "Issue Tracker": "https://github.com/nltk/nltk/issues", + }, + long_description="""\ +The Natural Language Toolkit (NLTK) is a Python package for +natural language processing. NLTK requires Python 3.7, 3.8, 3.9, 3.10, 3.11 or 3.12.""", + license="Apache License, Version 2.0", + keywords=[ + "NLP", + "CL", + "natural language processing", + "computational linguistics", + "parsing", + "tagging", + "tokenizing", + "syntax", + "linguistics", + "language", + "natural language", + "text analytics", + ], + maintainer="NLTK Team", + maintainer_email="nltk.team@gmail.com", + author="NLTK Team", + author_email="nltk.team@gmail.com", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Human Machine Interfaces", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Text Processing", + "Topic :: Text Processing :: Filters", + "Topic :: Text Processing :: General", + "Topic :: Text Processing :: Indexing", + "Topic :: Text Processing :: Linguistic", + ], + package_data={"nltk": ["test/*.doctest", "VERSION"]}, + python_requires=">=3.7", + install_requires=[ + "click", + "joblib", + "regex>=2021.8.3", + "tqdm", + ], + extras_require=extras_require, + packages=find_packages(), + zip_safe=False, # since normal files will be present too? + entry_points=console_scripts, +)
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
