CSS: unicode-range minification, fixes #321 and closes #344

tdewolff · Mar 15, 2021 · 75cef5c · 75cef5c
1 parent d68dbea
commit 75cef5c
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 4 deletions.
diff --git a/css/css.go b/css/css.go
@@ -6,7 +6,9 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/tdewolff/minify/v2"
 	"github.com/tdewolff/parse/v2"
@@ -1218,6 +1220,100 @@ func (c *cssMinifier) minifyProperty(prop Hash, values []Token) []Token {
 			values[0].Data = oneBytes
 			values[0].Ident = 0
 		}
+	case Unicode_Range:
+		ranges := [][2]int{}
+		for _, value := range values {
+			if value.TokenType == css.CommaToken {
+				continue
+			} else if value.TokenType != css.UnicodeRangeToken {
+				return values
+			}
+
+			i := 2
+			iWildcard := 0
+			start := 0
+			for i < len(value.Data) && value.Data[i] != '-' {
+				start *= 16
+				if '0' <= value.Data[i] && value.Data[i] <= '9' {
+					start += int(value.Data[i] - '0')
+				} else if 'a' <= value.Data[i]|32 && value.Data[i]|32 <= 'f' {
+					start += int(value.Data[i]|32-'a') + 10
+				} else if iWildcard == 0 && value.Data[i] == '?' {
+					iWildcard = i
+				}
+				i++
+			}
+			end := start
+			if iWildcard != 0 {
+				end = start + int(math.Pow(16.0, float64(len(value.Data)-iWildcard))) - 1
+			} else if i < len(value.Data) && value.Data[i] == '-' {
+				i++
+				end = 0
+				for i < len(value.Data) {
+					end *= 16
+					if '0' <= value.Data[i] && value.Data[i] <= '9' {
+						end += int(value.Data[i] - '0')
+					} else if 'a' <= value.Data[i]|32 && value.Data[i]|32 <= 'f' {
+						end += int(value.Data[i]|32-'a') + 10
+					}
+					i++
+				}
+				if end <= start {
+					end = start
+				}
+			}
+			ranges = append(ranges, [2]int{start, end})
+		}
+
+		// sort and remove overlapping ranges
+		sort.Slice(ranges, func(i, j int) bool { return ranges[i][0] < ranges[j][0] })
+		for i := 0; i < len(ranges)-1; i++ {
+			if ranges[i+1][1] <= ranges[i][1] {
+				// next range is fully contained in the current range
+				ranges = append(ranges[:i+1], ranges[i+2:]...)
+			} else if ranges[i+1][0] <= ranges[i][1]+1 {
+				// next range is partially covering the current range
+				ranges[i][1] = ranges[i+1][1]
+				ranges = append(ranges[:i+1], ranges[i+2:]...)
+			}
+		}
+
+		values = values[:0]
+		for i, ran := range ranges {
+			if i != 0 {
+				values = append(values, Token{css.CommaToken, commaBytes, nil, 0, None})
+			}
+			if ran[0] == ran[1] {
+				urange := []byte(fmt.Sprintf("U+%X", ran[0]))
+				values = append(values, Token{css.UnicodeRangeToken, urange, nil, 0, None})
+			} else if ran[0] == 0 && ran[1] == 0x10FFFF {
+				values = append(values, Token{css.IdentToken, initialBytes, nil, 0, None})
+			} else {
+				k := 0
+				for k < 6 && (ran[0]>>(k*4))&0xF == 0 && (ran[1]>>(k*4))&0xF == 0xF {
+					k++
+				}
+				wildcards := k
+				for k < 6 {
+					if (ran[0]>>(k*4))&0xF != (ran[1]>>(k*4))&0xF {
+						wildcards = 0
+						break
+					}
+					k++
+				}
+				var urange []byte
+				if wildcards != 0 {
+					if ran[0]>>(wildcards*4) == 0 {
+						urange = []byte(fmt.Sprintf("U+%s", strings.Repeat("?", wildcards)))
+					} else {
+						urange = []byte(fmt.Sprintf("U+%X%s", ran[0]>>(wildcards*4), strings.Repeat("?", wildcards)))
+					}
+				} else {
+					urange = []byte(fmt.Sprintf("U+%X-%X", ran[0], ran[1]))
+				}
+				values = append(values, Token{css.UnicodeRangeToken, urange, nil, 0, None})
+			}
+		}
 	}
 	return values
 }

diff --git a/css/css_test.go b/css/css_test.go
@@ -304,6 +304,21 @@ func TestCSSInline(t *testing.T) {
 		{"g:url('abc\\\ndef')", "g:url(abcdef)"},
 		{"url:local('abc\\\ndef')", "url:local(abcdef)"},
 		{"url:local('abc def') , url('abc def') format('truetype')", "url:local('abc def'),url('abc def')format('truetype')"},
+		{"unicode-range:U", "unicode-range:U"},
+		{"unicode-range:U+", "unicode-range:U+"},
+		{"unicode-range:U+0-04FF", "unicode-range:U+0-4FF"},
+		{"unicode-range:U+0-7F", "unicode-range:U+0-7F"},
+		{"unicode-range:U+000-FFF", "unicode-range:U+???"},
+		{"unicode-range:U+1000-2FFF", "unicode-range:U+1000-2FFF"},
+		{"unicode-range:U+2300-23FF,U+2A00-2aFf", "unicode-range:U+23??,U+2A??"},
+		{"unicode-range:U+25-FF", "unicode-range:U+25-FF"},
+		{"unicode-range:U+26", "unicode-range:U+26"},
+		{"unicode-range:U+2600-26FF", "unicode-range:U+26??"},
+		{"unicode-range:u+1234-1234", "unicode-range:U+1234"},
+		{"unicode-range:U+26??,U+27??", "unicode-range:U+2600-27FF"},
+		{"unicode-range:U+26??,U+2680-2780", "unicode-range:U+2600-2780"},
+		{"unicode-range:U+26??,U+2680-2690", "unicode-range:U+26??"},
+		{"unicode-range:U+0-10FFFF", "unicode-range:initial"},
 
 		// case
 		{"MARGIN:1EM", "margin:1em"},

diff --git a/css/hash.go b/css/hash.go
@@ -404,7 +404,7 @@ const (
 	Turn                        Hash = 0xd1f04 // turn
 	Turquoise                   Hash = 0xa8209 // turquoise
 	Unicode_Bidi                Hash = 0xcc40c // unicode-bidi
-	UnicodeRange                Hash = 0xd230d // unicode-range
+	Unicode_Range               Hash = 0xd230d // unicode-range
 	Unset                       Hash = 0xd3005 // unset
 	Url                         Hash = 0x3f403 // url
 	Var                         Hash = 0x64503 // var

diff --git a/go.mod b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2
 	github.com/spf13/pflag v1.0.5
-	github.com/tdewolff/parse/v2 v2.5.13
+	github.com/tdewolff/parse/v2 v2.5.14
 	github.com/tdewolff/test v1.0.6
 	golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c // indirect
 )
diff --git a/go.sum b/go.sum
@@ -8,8 +8,8 @@ github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 h1:JAEbJn3j/FrhdWA9jW8
 github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/tdewolff/parse/v2 v2.5.13 h1:9JiNjiyub3Xsqu2pIFuQFhQQtPBDS7Jkqy1htSKuu2M=
-github.com/tdewolff/parse/v2 v2.5.13/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
+github.com/tdewolff/parse/v2 v2.5.14 h1:ftdD54vkOeLZ7VkEZxp+wZrYZyyPi43GGon5GwBTRUI=
+github.com/tdewolff/parse/v2 v2.5.14/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
 github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
 github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=