370 lines
9.9 KiB
Go
370 lines
9.9 KiB
Go
// Copyright 2009 The Go Authors. All rights reserved.
|
||
// Use of this source code is governed by a BSD-style
|
||
// license that can be found in the LICENSE file.
|
||
|
||
// Package unicode provides data and functions to test some properties of
|
||
// Unicode code points.
|
||
package unicode
|
||
|
||
const (
|
||
MaxRune = '\U0010FFFF' // Maximum valid Unicode code point.
|
||
ReplacementChar = '\uFFFD' // Represents invalid code points.
|
||
MaxASCII = '\u007F' // maximum ASCII value.
|
||
MaxLatin1 = '\u00FF' // maximum Latin-1 value.
|
||
)
|
||
|
||
// RangeTable defines a set of Unicode code points by listing the ranges of
|
||
// code points within the set. The ranges are listed in two slices
|
||
// to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
|
||
// The two slices must be in sorted order and non-overlapping.
|
||
// Also, R32 should contain only values >= 0x10000 (1<<16).
|
||
type RangeTable struct {
|
||
R16 []Range16
|
||
R32 []Range32
|
||
LatinOffset int // number of entries in R16 with Hi <= MaxLatin1
|
||
}
|
||
|
||
// Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
|
||
// inclusive and has the specified stride.
|
||
type Range16 struct {
|
||
Lo uint16
|
||
Hi uint16
|
||
Stride uint16
|
||
}
|
||
|
||
// Range32 represents of a range of Unicode code points and is used when one or
|
||
// more of the values will not fit in 16 bits. The range runs from Lo to Hi
|
||
// inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
|
||
type Range32 struct {
|
||
Lo uint32
|
||
Hi uint32
|
||
Stride uint32
|
||
}
|
||
|
||
// CaseRange represents a range of Unicode code points for simple (one
|
||
// code point to one code point) case conversion.
|
||
// The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas
|
||
// are the number to add to the code point to reach the code point for a
|
||
// different case for that character. They may be negative. If zero, it
|
||
// means the character is in the corresponding case. There is a special
|
||
// case representing sequences of alternating corresponding Upper and Lower
|
||
// pairs. It appears with a fixed Delta of
|
||
// {UpperLower, UpperLower, UpperLower}
|
||
// The constant UpperLower has an otherwise impossible delta value.
|
||
type CaseRange struct {
|
||
Lo uint32
|
||
Hi uint32
|
||
Delta d
|
||
}
|
||
|
||
// SpecialCase represents language-specific case mappings such as Turkish.
|
||
// Methods of SpecialCase customize (by overriding) the standard mappings.
|
||
type SpecialCase []CaseRange
|
||
|
||
// BUG(r): There is no mechanism for full case folding, that is, for
|
||
// characters that involve multiple runes in the input or output.
|
||
|
||
// Indices into the Delta arrays inside CaseRanges for case mapping.
|
||
const (
|
||
UpperCase = iota
|
||
LowerCase
|
||
TitleCase
|
||
MaxCase
|
||
)
|
||
|
||
type d [MaxCase]rune // to make the CaseRanges text shorter
|
||
|
||
// If the Delta field of a CaseRange is UpperLower, it means
|
||
// this CaseRange represents a sequence of the form (say)
|
||
// Upper Lower Upper Lower.
|
||
const (
|
||
UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
|
||
)
|
||
|
||
// linearMax is the maximum size table for linear search for non-Latin1 rune.
|
||
// Derived by running 'go test -calibrate'.
|
||
const linearMax = 18
|
||
|
||
// is16 reports whether r is in the sorted slice of 16-bit ranges.
|
||
func is16(ranges []Range16, r uint16) bool {
|
||
if len(ranges) <= linearMax || r <= MaxLatin1 {
|
||
for i := range ranges {
|
||
range_ := &ranges[i]
|
||
if r < range_.Lo {
|
||
return false
|
||
}
|
||
if r <= range_.Hi {
|
||
return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// binary search over ranges
|
||
lo := 0
|
||
hi := len(ranges)
|
||
for lo < hi {
|
||
m := lo + (hi-lo)/2
|
||
range_ := &ranges[m]
|
||
if range_.Lo <= r && r <= range_.Hi {
|
||
return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
|
||
}
|
||
if r < range_.Lo {
|
||
hi = m
|
||
} else {
|
||
lo = m + 1
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// is32 reports whether r is in the sorted slice of 32-bit ranges.
|
||
func is32(ranges []Range32, r uint32) bool {
|
||
if len(ranges) <= linearMax {
|
||
for i := range ranges {
|
||
range_ := &ranges[i]
|
||
if r < range_.Lo {
|
||
return false
|
||
}
|
||
if r <= range_.Hi {
|
||
return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// binary search over ranges
|
||
lo := 0
|
||
hi := len(ranges)
|
||
for lo < hi {
|
||
m := lo + (hi-lo)/2
|
||
range_ := ranges[m]
|
||
if range_.Lo <= r && r <= range_.Hi {
|
||
return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
|
||
}
|
||
if r < range_.Lo {
|
||
hi = m
|
||
} else {
|
||
lo = m + 1
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// Is reports whether the rune is in the specified table of ranges.
|
||
func Is(rangeTab *RangeTable, r rune) bool {
|
||
r16 := rangeTab.R16
|
||
// Compare as uint32 to correctly handle negative runes.
|
||
if len(r16) > 0 && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
|
||
return is16(r16, uint16(r))
|
||
}
|
||
r32 := rangeTab.R32
|
||
if len(r32) > 0 && r >= rune(r32[0].Lo) {
|
||
return is32(r32, uint32(r))
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isExcludingLatin(rangeTab *RangeTable, r rune) bool {
|
||
r16 := rangeTab.R16
|
||
// Compare as uint32 to correctly handle negative runes.
|
||
if off := rangeTab.LatinOffset; len(r16) > off && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
|
||
return is16(r16[off:], uint16(r))
|
||
}
|
||
r32 := rangeTab.R32
|
||
if len(r32) > 0 && r >= rune(r32[0].Lo) {
|
||
return is32(r32, uint32(r))
|
||
}
|
||
return false
|
||
}
|
||
|
||
// IsUpper reports whether the rune is an upper case letter.
|
||
func IsUpper(r rune) bool {
|
||
// See comment in IsGraphic.
|
||
if uint32(r) <= MaxLatin1 {
|
||
return properties[uint8(r)]&pLmask == pLu
|
||
}
|
||
return isExcludingLatin(Upper, r)
|
||
}
|
||
|
||
// IsLower reports whether the rune is a lower case letter.
|
||
func IsLower(r rune) bool {
|
||
// See comment in IsGraphic.
|
||
if uint32(r) <= MaxLatin1 {
|
||
return properties[uint8(r)]&pLmask == pLl
|
||
}
|
||
return isExcludingLatin(Lower, r)
|
||
}
|
||
|
||
// IsTitle reports whether the rune is a title case letter.
|
||
func IsTitle(r rune) bool {
|
||
if r <= MaxLatin1 {
|
||
return false
|
||
}
|
||
return isExcludingLatin(Title, r)
|
||
}
|
||
|
||
// to maps the rune using the specified case mapping.
|
||
// It additionally reports whether caseRange contained a mapping for r.
|
||
func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) {
|
||
if _case < 0 || MaxCase <= _case {
|
||
return ReplacementChar, false // as reasonable an error as any
|
||
}
|
||
// binary search over ranges
|
||
lo := 0
|
||
hi := len(caseRange)
|
||
for lo < hi {
|
||
m := lo + (hi-lo)/2
|
||
cr := caseRange[m]
|
||
if rune(cr.Lo) <= r && r <= rune(cr.Hi) {
|
||
delta := cr.Delta[_case]
|
||
if delta > MaxRune {
|
||
// In an Upper-Lower sequence, which always starts with
|
||
// an UpperCase letter, the real deltas always look like:
|
||
// {0, 1, 0} UpperCase (Lower is next)
|
||
// {-1, 0, -1} LowerCase (Upper, Title are previous)
|
||
// The characters at even offsets from the beginning of the
|
||
// sequence are upper case; the ones at odd offsets are lower.
|
||
// The correct mapping can be done by clearing or setting the low
|
||
// bit in the sequence offset.
|
||
// The constants UpperCase and TitleCase are even while LowerCase
|
||
// is odd so we take the low bit from _case.
|
||
return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)), true
|
||
}
|
||
return r + delta, true
|
||
}
|
||
if r < rune(cr.Lo) {
|
||
hi = m
|
||
} else {
|
||
lo = m + 1
|
||
}
|
||
}
|
||
return r, false
|
||
}
|
||
|
||
// To maps the rune to the specified case: UpperCase, LowerCase, or TitleCase.
|
||
func To(_case int, r rune) rune {
|
||
r, _ = to(_case, r, CaseRanges)
|
||
return r
|
||
}
|
||
|
||
// ToUpper maps the rune to upper case.
|
||
func ToUpper(r rune) rune {
|
||
if r <= MaxASCII {
|
||
if 'a' <= r && r <= 'z' {
|
||
r -= 'a' - 'A'
|
||
}
|
||
return r
|
||
}
|
||
return To(UpperCase, r)
|
||
}
|
||
|
||
// ToLower maps the rune to lower case.
|
||
func ToLower(r rune) rune {
|
||
if r <= MaxASCII {
|
||
if 'A' <= r && r <= 'Z' {
|
||
r += 'a' - 'A'
|
||
}
|
||
return r
|
||
}
|
||
return To(LowerCase, r)
|
||
}
|
||
|
||
// ToTitle maps the rune to title case.
|
||
func ToTitle(r rune) rune {
|
||
if r <= MaxASCII {
|
||
if 'a' <= r && r <= 'z' { // title case is upper case for ASCII
|
||
r -= 'a' - 'A'
|
||
}
|
||
return r
|
||
}
|
||
return To(TitleCase, r)
|
||
}
|
||
|
||
// ToUpper maps the rune to upper case giving priority to the special mapping.
|
||
func (special SpecialCase) ToUpper(r rune) rune {
|
||
r1, hadMapping := to(UpperCase, r, []CaseRange(special))
|
||
if r1 == r && !hadMapping {
|
||
r1 = ToUpper(r)
|
||
}
|
||
return r1
|
||
}
|
||
|
||
// ToTitle maps the rune to title case giving priority to the special mapping.
|
||
func (special SpecialCase) ToTitle(r rune) rune {
|
||
r1, hadMapping := to(TitleCase, r, []CaseRange(special))
|
||
if r1 == r && !hadMapping {
|
||
r1 = ToTitle(r)
|
||
}
|
||
return r1
|
||
}
|
||
|
||
// ToLower maps the rune to lower case giving priority to the special mapping.
|
||
func (special SpecialCase) ToLower(r rune) rune {
|
||
r1, hadMapping := to(LowerCase, r, []CaseRange(special))
|
||
if r1 == r && !hadMapping {
|
||
r1 = ToLower(r)
|
||
}
|
||
return r1
|
||
}
|
||
|
||
// caseOrbit is defined in tables.go as []foldPair. Right now all the
|
||
// entries fit in uint16, so use uint16. If that changes, compilation
|
||
// will fail (the constants in the composite literal will not fit in uint16)
|
||
// and the types here can change to uint32.
|
||
type foldPair struct {
|
||
From uint16
|
||
To uint16
|
||
}
|
||
|
||
// SimpleFold iterates over Unicode code points equivalent under
|
||
// the Unicode-defined simple case folding. Among the code points
|
||
// equivalent to rune (including rune itself), SimpleFold returns the
|
||
// smallest rune > r if one exists, or else the smallest rune >= 0.
|
||
// If r is not a valid Unicode code point, SimpleFold(r) returns r.
|
||
//
|
||
// For example:
|
||
// SimpleFold('A') = 'a'
|
||
// SimpleFold('a') = 'A'
|
||
//
|
||
// SimpleFold('K') = 'k'
|
||
// SimpleFold('k') = '\u212A' (Kelvin symbol, K)
|
||
// SimpleFold('\u212A') = 'K'
|
||
//
|
||
// SimpleFold('1') = '1'
|
||
//
|
||
// SimpleFold(-2) = -2
|
||
//
|
||
func SimpleFold(r rune) rune {
|
||
if r < 0 || r > MaxRune {
|
||
return r
|
||
}
|
||
|
||
if int(r) < len(asciiFold) {
|
||
return rune(asciiFold[r])
|
||
}
|
||
|
||
// Consult caseOrbit table for special cases.
|
||
lo := 0
|
||
hi := len(caseOrbit)
|
||
for lo < hi {
|
||
m := lo + (hi-lo)/2
|
||
if rune(caseOrbit[m].From) < r {
|
||
lo = m + 1
|
||
} else {
|
||
hi = m
|
||
}
|
||
}
|
||
if lo < len(caseOrbit) && rune(caseOrbit[lo].From) == r {
|
||
return rune(caseOrbit[lo].To)
|
||
}
|
||
|
||
// No folding specified. This is a one- or two-element
|
||
// equivalence class containing rune and ToLower(rune)
|
||
// and ToUpper(rune) if they are different from rune.
|
||
if l := ToLower(r); l != r {
|
||
return l
|
||
}
|
||
return ToUpper(r)
|
||
}
|