Update to CLDR data v44 (#310)

v2/goi18n/merge_command_test.go

@@ -297,6 +297,7 @@ other = "{{.Count}} unread emails"
 [UnreadEmails]
 description = "Message that tells the user how many unread emails they have"
 hash = "sha1-5afbc91dfedb9755627655c365eb47a89e541099"
+many = "{{.Count}} unread emails"
 one = "{{.Count}} unread email"
 other = "{{.Count}} unread emails"
 `),
@@ -435,6 +436,7 @@ other = "{{.Count}} unread emails!"
 [UnreadEmails]
 description = "Message that tells the user how many unread emails they have"
 hash = "sha1-92a24983c5bbc0c42462cdc252dca68ebdb46501"
+many = "{{.Count}} unread emails!"
 one = "{{.Count}} unread emails!"
 other = "{{.Count}} unread emails!"
 `),

v2/internal/plural/codegen/README.md

@@ -1,6 +1,5 @@
 # How to upgrade CLDR data
 
-1.  Go to http://cldr.unicode.org/index/downloads to find the latest version.
-1.  Download the latest version of cldr-common (e.g. https://unicode.org/Public/cldr/39/cldr-common-39.0.zip)
+1.  Go to https://github.com/unicode-org/cldr/releases to find the latest release and download the source code.
 1.  Unzip and copy `common/supplemental/plurals.xml` to this directory.
 1.  Run `generate.sh`.

v2/internal/plural/codegen/xml.go

@@ -4,6 +4,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
 )
 
@@ -47,36 +48,47 @@ func (pr *PluralRule) Condition() string {
 	return pr.Rule[:i]
 }
 
-// Examples returns the integer and decimal exmaples for the PLuralRule.
-func (pr *PluralRule) Examples() (integer []string, decimal []string) {
-	ex := strings.Replace(pr.Rule, ", …", "", -1)
+// Examples returns the integer and decimal examples for the PluralRule.
+func (pr *PluralRule) Examples() (integers []string, decimals []string) {
+	ex := strings.ReplaceAll(pr.Rule, ", …", "")
 	ddelim := "@decimal"
 	if i := strings.Index(ex, ddelim); i > 0 {
 		dex := strings.TrimSpace(ex[i+len(ddelim):])
-		decimal = strings.Split(dex, ", ")
+		dex = strings.ReplaceAll(dex, "c", "e")
+		decimals = strings.Split(dex, ", ")
 		ex = ex[:i]
 	}
 	idelim := "@integer"
 	if i := strings.Index(ex, idelim); i > 0 {
 		iex := strings.TrimSpace(ex[i+len(idelim):])
-		integer = strings.Split(iex, ", ")
+		integers = strings.Split(iex, ", ")
+		for j, integer := range integers {
+			ii := strings.IndexAny(integer, "eEcC")
+			if ii > 0 {
+				zeros, err := strconv.ParseInt(integer[ii+1:], 10, 0)
+				if err != nil {
+					panic(err)
+				}
+				integers[j] = integer[:ii] + strings.Repeat("0", int(zeros))
+			}
+		}
 	}
-	return integer, decimal
+	return integers, decimals
 }
 
-// IntegerExamples returns the integer exmaples for the PLuralRule.
+// IntegerExamples returns the integer examples for the PluralRule.
 func (pr *PluralRule) IntegerExamples() []string {
 	integer, _ := pr.Examples()
 	return integer
 }
 
-// DecimalExamples returns the decimal exmaples for the PLuralRule.
+// DecimalExamples returns the decimal examples for the PluralRule.
 func (pr *PluralRule) DecimalExamples() []string {
 	_, decimal := pr.Examples()
 	return decimal
 }
 
-var relationRegexp = regexp.MustCompile(`([niftvw])(?:\s*%\s*([0-9]+))?\s*(!=|=)(.*)`)
+var relationRegexp = regexp.MustCompile(`([niftvwce])(?:\s*%\s*([0-9]+))?\s*(!=|=)(.*)`)
 
 // GoCondition converts the XML condition to valid Go code.
 func (pr *PluralRule) GoCondition() string {
@@ -92,6 +104,11 @@ func (pr *PluralRule) GoCondition() string {
 			if op == "=" {
 				op = "=="
 			}
+			if lvar == "E" {
+				// E is a deprecated symbol for C
+				// https://unicode.org/reports/tr35/tr35-numbers.html#Plural_Operand_Meanings
+				lvar = "C"
+			}
 			lvar = "ops." + lvar
 			var rhor []string
 			var rany []string

v2/internal/plural/operands.go

@@ -7,13 +7,17 @@ import (
 )
 
 // Operands is a representation of http://unicode.org/reports/tr35/tr35-numbers.html#Operands
+// If there is a compact decimal exponent value C, then the N, I, V, W, F, and T values are computed after shifting the decimal point in the original by the ‘c’ value.
+// So for 1.2c3, the values are the same as those of 1200: i=1200 and f=0.
+// Similarly, 1.2005c3 has i=1200 and f=5 (corresponding to 1200.5).
 type Operands struct {
 	N float64 // absolute value of the source number (integer and decimals)
 	I int64   // integer digits of n
 	V int64   // number of visible fraction digits in n, with trailing zeros
 	W int64   // number of visible fraction digits in n, without trailing zeros
 	F int64   // visible fractional digits in n, with trailing zeros
 	T int64   // visible fractional digits in n, without trailing zeros
+	C int64   // compact decimal exponent value: exponent of the power of 10 used in compact decimal formatting.
 }
 
 // NEqualsAny returns true if o represents an integer equal to any of the arguments.
@@ -74,19 +78,87 @@ func newOperandsInt64(i int64) *Operands {
 	if i < 0 {
 		i = -i
 	}
-	return &Operands{float64(i), i, 0, 0, 0, 0}
+	return &Operands{float64(i), i, 0, 0, 0, 0, 0}
+}
+
+func splitSignificandExponent(s string) (significand, exponent string) {
+	i := strings.IndexAny(s, "eE")
+	if i < 0 {
+		return s, ""
+	}
+	return s[:i], s[i+1:]
+}
+
+func shiftDecimalLeft(s string, n int) string {
+	if n <= 0 {
+		return s
+	}
+	i := strings.IndexRune(s, '.')
+	tilt := 0
+	if i < 0 {
+		i = len(s)
+		tilt = -1
+	}
+	switch {
+	case n == i:
+		return "0." + s[:i] + s[i+1+tilt:]
+	case n > i:
+		return "0." + strings.Repeat("0", n-i) + s[:i] + s[i+1+tilt:]
+	default:
+		return s[:i-n] + "." + s[i-n:i] + s[i+1+tilt:]
+	}
+}
+
+func shiftDecimalRight(s string, n int) string {
+	if n <= 0 {
+		return s
+	}
+	i := strings.IndexRune(s, '.')
+	if i < 0 {
+		return s + strings.Repeat("0", n)
+	}
+	switch rest := len(s) - i - 1; {
+	case n == rest:
+		return s[:i] + s[i+1:]
+	case n > rest:
+		return s[:i] + s[i+1:] + strings.Repeat("0", n-rest)
+	default:
+		return s[:i] + s[i+1:i+1+n] + "." + s[i+1+n:]
+	}
+}
+
+func applyExponent(s string, exponent int) string {
+	switch {
+	case exponent > 0:
+		return shiftDecimalRight(s, exponent)
+	case exponent < 0:
+		return shiftDecimalLeft(s, -exponent)
+	}
+	return s
 }
 
 func newOperandsString(s string) (*Operands, error) {
 	if s[0] == '-' {
 		s = s[1:]
 	}
-	n, err := strconv.ParseFloat(s, 64)
+	ops := &Operands{}
+	var err error
+	ops.N, err = strconv.ParseFloat(s, 64)
 	if err != nil {
 		return nil, err
 	}
-	ops := &Operands{N: n}
-	parts := strings.SplitN(s, ".", 2)
+	significand, exponent := splitSignificandExponent(s)
+	if exponent != "" {
+		// We are storing C as an int64 but only allowing
+		// numbers that fit into the bitsize of an int
+		// so C is safe to cast as a int later.
+		ops.C, err = strconv.ParseInt(exponent, 10, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+	value := applyExponent(significand, int(ops.C))
+	parts := strings.SplitN(value, ".", 2)
 	ops.I, err = strconv.ParseInt(parts[0], 10, 64)
 	if err != nil {
 		return nil, err

v2/internal/plural/operands_test.go

@@ -11,18 +11,47 @@ func TestNewOperands(t *testing.T) {
 		ops   *Operands
 		err   bool
 	}{
-		{int64(0), &Operands{0.0, 0, 0, 0, 0, 0}, false},
-		{int64(1), &Operands{1.0, 1, 0, 0, 0, 0}, false},
-		{"0", &Operands{0.0, 0, 0, 0, 0, 0}, false},
-		{"1", &Operands{1.0, 1, 0, 0, 0, 0}, false},
-		{"1.0", &Operands{1.0, 1, 1, 0, 0, 0}, false},
-		{"1.00", &Operands{1.0, 1, 2, 0, 0, 0}, false},
-		{"1.3", &Operands{1.3, 1, 1, 1, 3, 3}, false},
-		{"1.30", &Operands{1.3, 1, 2, 1, 30, 3}, false},
-		{"1.03", &Operands{1.03, 1, 2, 2, 3, 3}, false},
-		{"1.230", &Operands{1.23, 1, 3, 2, 230, 23}, false},
-		{"20.0230", &Operands{20.023, 20, 4, 3, 230, 23}, false},
+		{int64(0), &Operands{0.0, 0, 0, 0, 0, 0, 0}, false},
+		{int64(1), &Operands{1.0, 1, 0, 0, 0, 0, 0}, false},
+		{"0", &Operands{0.0, 0, 0, 0, 0, 0, 0}, false},
+		{"1", &Operands{1.0, 1, 0, 0, 0, 0, 0}, false},
+		{"1.0", &Operands{1.0, 1, 1, 0, 0, 0, 0}, false},
+		{"1.00", &Operands{1.0, 1, 2, 0, 0, 0, 0}, false},
+		{"1.3", &Operands{1.3, 1, 1, 1, 3, 3, 0}, false},
+		{"1.30", &Operands{1.3, 1, 2, 1, 30, 3, 0}, false},
+		{"1.03", &Operands{1.03, 1, 2, 2, 3, 3, 0}, false},
+		{"1.230", &Operands{1.23, 1, 3, 2, 230, 23, 0}, false},
+		{"20.0230", &Operands{20.023, 20, 4, 3, 230, 23, 0}, false},
 		{20.0230, nil, true},
+
+		{"1200", &Operands{1200, 1200, 0, 0, 0, 0, 0}, false},
+		{"1.2e3", &Operands{1200, 1200, 0, 0, 0, 0, 3}, false},
+		{"1.2E3", &Operands{1200, 1200, 0, 0, 0, 0, 3}, false},
+
+		{"1234", &Operands{1234, 1234, 0, 0, 0, 0, 0}, false},
+		{"1234e0", &Operands{1234, 1234, 0, 0, 0, 0, 0}, false},
+		{"123.4e1", &Operands{1234, 1234, 0, 0, 0, 0, 1}, false},
+		{"12.34e2", &Operands{1234, 1234, 0, 0, 0, 0, 2}, false},
+		{"1.234e3", &Operands{1234, 1234, 0, 0, 0, 0, 3}, false},
+		{"0.1234e4", &Operands{1234, 1234, 0, 0, 0, 0, 4}, false},
+		{"0.01234e5", &Operands{1234, 1234, 0, 0, 0, 0, 5}, false},
+
+		{"1234.0", &Operands{1234, 1234, 1, 0, 0, 0, 0}, false},
+		{"12340e-1", &Operands{1234, 1234, 1, 0, 0, 0, -1}, false},
+
+		{"1200.5", &Operands{1200.5, 1200, 1, 1, 5, 5, 0}, false},
+		{"1.2005e3", &Operands{1200.5, 1200, 1, 1, 5, 5, 3}, false},
+
+		{"1200e3", &Operands{1200000, 1200000, 0, 0, 0, 0, 3}, false},
+
+		{"0.0012340", &Operands{0.001234, 0, 7, 6, 12340, 1234, 0}, false},
+		{"0.012340e-1", &Operands{0.001234, 0, 7, 6, 12340, 1234, -1}, false},
+		{"0.12340e-2", &Operands{0.001234, 0, 7, 6, 12340, 1234, -2}, false},
+		{"1.2340e-3", &Operands{0.001234, 0, 7, 6, 12340, 1234, -3}, false},
+		{"12.340e-4", &Operands{0.001234, 0, 7, 6, 12340, 1234, -4}, false},
+		{"123.40e-5", &Operands{0.001234, 0, 7, 6, 12340, 1234, -5}, false},
+		{"1234.0e-6", &Operands{0.001234, 0, 7, 6, 12340, 1234, -6}, false},
+		{"12340e-7", &Operands{0.001234, 0, 7, 6, 12340, 1234, -7}, false},
 	}
 	for _, test := range tests {
 		ops, err := NewOperands(test.input)