internal/language: bump script types to uint16 and update registry

The IANA language-subtag-registry now contains more than 256 scripts,
causing the uint8 types to overflow during table generation. The
internal script types are bumped to uint16 which should be more than
enough.

Fixes golang/go#45093

Change-Id: I58184902e6652f488521d084fce6e0b424121825
Reviewed-on: https://go-review.googlesource.com/c/text/+/304029
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Than McIntosh <[email protected]>
Run-TryBot: Roland Shoemaker <[email protected]>
Reviewed-by: Damien Neil <[email protected]>
Auto-Submit: Roland Shoemaker <[email protected]>

Author: rolandshoemaker

encoding/htmlindex/tables.go

@@ -1209,12 +1209,12 @@ func (b *builder) writeLikelyData() {
 	type ( // generated types
 		likelyScriptRegion struct {
 			region uint16
-			script uint8
+			script uint16
 			flags  uint8
 		}
 		likelyLangScript struct {
 			lang   uint16
-			script uint8
+			script uint16
 			flags  uint8
 		}
 		likelyLangRegion struct {
@@ -1226,7 +1226,7 @@ func (b *builder) writeLikelyData() {
 		likelyTag struct {
 			lang   uint16
 			region uint16
-			script uint8
+			script uint16
 		}
 	)
 	var ( // generated variables
@@ -1279,7 +1279,7 @@ func (b *builder) writeLikelyData() {
 					log.Fatalf("region changed unexpectedly: %s -> %s", from, to)
 				}
 				likelyRegionGroup[id].lang = uint16(b.langIndex(to[0]))
-				likelyRegionGroup[id].script = uint8(b.script.index(to[1]))
+				likelyRegionGroup[id].script = uint16(b.script.index(to[1]))
 				likelyRegionGroup[id].region = uint16(b.region.index(to[2]))
 			} else {
 				regionToOther[r] = append(regionToOther[r], fromTo{from, to})
@@ -1293,11 +1293,11 @@ func (b *builder) writeLikelyData() {
 		list := langToOther[id]
 		if len(list) == 1 {
 			likelyLang[id].region = uint16(b.region.index(list[0].to[2]))
-			likelyLang[id].script = uint8(b.script.index(list[0].to[1]))
+			likelyLang[id].script = uint16(b.script.index(list[0].to[1]))
 		} else if len(list) > 1 {
 			likelyLang[id].flags = isList
 			likelyLang[id].region = uint16(len(likelyLangList))
-			likelyLang[id].script = uint8(len(list))
+			likelyLang[id].script = uint16(len(list))
 			for _, x := range list {
 				flags := uint8(0)
 				if len(x.from) > 1 {
@@ -1309,7 +1309,7 @@ func (b *builder) writeLikelyData() {
 				}
 				likelyLangList = append(likelyLangList, likelyScriptRegion{
 					region: uint16(b.region.index(x.to[2])),
-					script: uint8(b.script.index(x.to[1])),
+					script: uint16(b.script.index(x.to[1])),
 					flags:  flags,
 				})
 			}
@@ -1324,21 +1324,21 @@ func (b *builder) writeLikelyData() {
 		list := regionToOther[id]
 		if len(list) == 1 {
 			likelyRegion[id].lang = uint16(b.langIndex(list[0].to[0]))
-			likelyRegion[id].script = uint8(b.script.index(list[0].to[1]))
+			likelyRegion[id].script = uint16(b.script.index(list[0].to[1]))
 			if len(list[0].from) > 2 {
 				likelyRegion[id].flags = scriptInFrom
 			}
 		} else if len(list) > 1 {
 			likelyRegion[id].flags = isList
 			likelyRegion[id].lang = uint16(len(likelyRegionList))
-			likelyRegion[id].script = uint8(len(list))
+			likelyRegion[id].script = uint16(len(list))
 			for i, x := range list {
 				if len(x.from) == 2 && i != 0 || i > 0 && len(x.from) != 3 {
 					log.Fatalf("unspecified script must be first in list: %v at %d", x.from, i)
 				}
 				x := likelyLangScript{
 					lang:   uint16(b.langIndex(x.to[0])),
-					script: uint8(b.script.index(x.to[1])),
+					script: uint16(b.script.index(x.to[1])),
 				}
 				if len(list[0].from) > 2 {
 					x.flags = scriptInFrom
@@ -1453,8 +1453,8 @@ func (b *builder) writeRegionInclusionData() {
 
 type parentRel struct {
 	lang       uint16
-	script     uint8
-	maxScript  uint8
+	script     uint16
+	maxScript  uint16
 	toRegion   uint16
 	fromRegion []uint16
 }
@@ -1477,10 +1477,10 @@ func (b *builder) writeParents() {
 		if len(sub) == 2 {
 			// TODO: check that all undefined scripts are indeed Latn in these
 			// cases.
-			parent.maxScript = uint8(b.script.index("Latn"))
+			parent.maxScript = uint16(b.script.index("Latn"))
 			parent.toRegion = uint16(b.region.index(sub[1]))
 		} else {
-			parent.script = uint8(b.script.index(sub[1]))
+			parent.script = uint16(b.script.index(sub[1]))
 			parent.maxScript = parent.script
 			parent.toRegion = uint16(b.region.index(sub[2]))
 		}

encoding/ianaindex/tables.go

@@ -14,8 +14,8 @@ import (
 func TestTagSize(t *testing.T) {
 	id := Tag{}
 	typ := reflect.TypeOf(id)
-	if typ.Size() > 24 {
-		t.Errorf("size of Tag was %d; want 24", typ.Size())
+	if typ.Size() > 32 {
+		t.Errorf("size of Tag was %d; want <= 32", typ.Size())
 	}
 }
 

encoding/internal/identifier/mib.go

@@ -328,7 +328,7 @@ func (r Region) IsPrivateUse() bool {
 	return r.typ()&iso3166UserAssigned != 0
 }
 
-type Script uint8
+type Script uint16
 
 // getScriptID returns the script id for string s. It assumes that s
 // is of the format [A-Z][a-z]{3}.