auto merge of #20395 : huonw/rust/char-stab-2, r=aturon

cc #19260 

The casing transformations are left unstable (it is highly likely to be better to adopt the proper non-1-to-1 case mappings, per #20333) as are `is_xid_*`.

I've got a little todo list in the last commit of things I thought about/was told about that I haven't yet handled (I'd also like some feedback).
This commit is contained in:
bors 2015-01-05 06:45:39 +00:00
commit 1f732ef53d
16 changed files with 181 additions and 64 deletions

View File

@ -103,7 +103,6 @@ mod std {
mod prelude {
// from core.
pub use core::borrow::IntoCow;
pub use core::char::Char;
pub use core::clone::Clone;
pub use core::cmp::{PartialEq, Eq, PartialOrd, Ord};
pub use core::cmp::Ordering::{Less, Equal, Greater};
@ -127,7 +126,7 @@ mod prelude {
// from other crates.
pub use alloc::boxed::Box;
pub use unicode::char::UnicodeChar;
pub use unicode::char::CharExt;
// from collections.
pub use slice::SliceConcatExt;

View File

@ -55,7 +55,7 @@ use self::RecompositionState::*;
use self::DecompositionType::*;
use core::borrow::{BorrowFrom, ToOwned};
use core::char::Char;
use core::char::CharExt;
use core::clone::Clone;
use core::iter::AdditiveIterator;
use core::iter::{range, Iterator, IteratorExt};

View File

@ -69,7 +69,7 @@ pub const MAX: char = '\u{10ffff}';
/// Converts from `u32` to a `char`
#[inline]
#[unstable = "pending decisions about costructors for primitives"]
#[stable]
pub fn from_u32(i: u32) -> Option<char> {
// catch out-of-bounds and surrogates
if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) {
@ -92,7 +92,7 @@ pub fn from_u32(i: u32) -> Option<char> {
/// Panics if given an `radix` > 36.
///
#[inline]
#[unstable = "pending decisions about costructors for primitives"]
#[unstable = "pending integer conventions"]
pub fn from_digit(num: uint, radix: uint) -> Option<char> {
if radix > 36 {
panic!("from_digit: radix is too high (maximum 36)");
@ -111,8 +111,8 @@ pub fn from_digit(num: uint, radix: uint) -> Option<char> {
}
/// Basic `char` manipulations.
#[experimental = "trait organization may change"]
pub trait Char {
#[stable]
pub trait CharExt {
/// Checks if a `char` parses as a numeric digit in the given radix.
///
/// Compared to `is_numeric()`, this function only recognizes the characters
@ -126,7 +126,7 @@ pub trait Char {
/// # Panics
///
/// Panics if given a radix > 36.
#[unstable = "pending error conventions"]
#[unstable = "pending integer conventions"]
fn is_digit(self, radix: uint) -> bool;
/// Converts a character to the corresponding digit.
@ -140,7 +140,7 @@ pub trait Char {
/// # Panics
///
/// Panics if given a radix outside the range [0..36].
#[unstable = "pending error conventions, trait organization"]
#[unstable = "pending integer conventions"]
fn to_digit(self, radix: uint) -> Option<uint>;
/// Returns an iterator that yields the hexadecimal Unicode escape
@ -149,7 +149,7 @@ pub trait Char {
/// All characters are escaped with Rust syntax of the form `\\u{NNNN}`
/// where `NNNN` is the shortest hexadecimal representation of the code
/// point.
#[unstable = "pending error conventions, trait organization"]
#[stable]
fn escape_unicode(self) -> EscapeUnicode;
/// Returns an iterator that yields the 'default' ASCII and
@ -164,17 +164,17 @@ pub trait Char {
/// escaped.
/// * Any other chars in the range [0x20,0x7e] are not escaped.
/// * Any other chars are given hex Unicode escapes; see `escape_unicode`.
#[unstable = "pending error conventions, trait organization"]
#[stable]
fn escape_default(self) -> EscapeDefault;
/// Returns the amount of bytes this character would need if encoded in
/// UTF-8.
#[unstable = "pending trait organization"]
#[stable]
fn len_utf8(self) -> uint;
/// Returns the amount of bytes this character would need if encoded in
/// UTF-16.
#[unstable = "pending trait organization"]
#[stable]
fn len_utf16(self) -> uint;
/// Encodes this character as UTF-8 into the provided byte buffer,
@ -182,29 +182,26 @@ pub trait Char {
///
/// If the buffer is not large enough, nothing will be written into it
/// and a `None` will be returned.
#[unstable = "pending trait organization"]
fn encode_utf8(&self, dst: &mut [u8]) -> Option<uint>;
#[stable]
fn encode_utf8(self, dst: &mut [u8]) -> Option<uint>;
/// Encodes this character as UTF-16 into the provided `u16` buffer,
/// and then returns the number of `u16`s written.
///
/// If the buffer is not large enough, nothing will be written into it
/// and a `None` will be returned.
#[unstable = "pending trait organization"]
fn encode_utf16(&self, dst: &mut [u16]) -> Option<uint>;
#[stable]
fn encode_utf16(self, dst: &mut [u16]) -> Option<uint>;
}
#[experimental = "trait is experimental"]
impl Char for char {
#[unstable = "pending trait organization"]
#[stable]
impl CharExt for char {
#[unstable = "pending integer conventions"]
fn is_digit(self, radix: uint) -> bool {
match self.to_digit(radix) {
Some(_) => true,
None => false,
}
self.to_digit(radix).is_some()
}
#[unstable = "pending trait organization"]
#[unstable = "pending integer conventions"]
fn to_digit(self, radix: uint) -> Option<uint> {
if radix > 36 {
panic!("to_digit: radix is too high (maximum 36)");
@ -219,12 +216,12 @@ impl Char for char {
else { None }
}
#[unstable = "pending error conventions, trait organization"]
#[stable]
fn escape_unicode(self) -> EscapeUnicode {
EscapeUnicode { c: self, state: EscapeUnicodeState::Backslash }
}
#[unstable = "pending error conventions, trait organization"]
#[stable]
fn escape_default(self) -> EscapeDefault {
let init_state = match self {
'\t' => EscapeDefaultState::Backslash('t'),
@ -240,7 +237,7 @@ impl Char for char {
}
#[inline]
#[unstable = "pending trait organization"]
#[stable]
fn len_utf8(self) -> uint {
let code = self as u32;
match () {
@ -252,17 +249,17 @@ impl Char for char {
}
#[inline]
#[unstable = "pending trait organization"]
#[stable]
fn len_utf16(self) -> uint {
let ch = self as u32;
if (ch & 0xFFFF_u32) == ch { 1 } else { 2 }
}
#[inline]
#[unstable = "pending error conventions, trait organization"]
fn encode_utf8<'a>(&self, dst: &'a mut [u8]) -> Option<uint> {
#[unstable = "pending decision about Iterator/Writer/Reader"]
fn encode_utf8(self, dst: &mut [u8]) -> Option<uint> {
// Marked #[inline] to allow llvm optimizing it away
let code = *self as u32;
let code = self as u32;
if code < MAX_ONE_B && dst.len() >= 1 {
dst[0] = code as u8;
Some(1)
@ -287,10 +284,10 @@ impl Char for char {
}
#[inline]
#[unstable = "pending error conventions, trait organization"]
fn encode_utf16(&self, dst: &mut [u16]) -> Option<uint> {
#[unstable = "pending decision about Iterator/Writer/Reader"]
fn encode_utf16(self, dst: &mut [u16]) -> Option<uint> {
// Marked #[inline] to allow llvm optimizing it away
let mut ch = *self as u32;
let mut ch = self as u32;
if (ch & 0xFFFF_u32) == ch && dst.len() >= 1 {
// The BMP falls through (assuming non-surrogate, as it should)
dst[0] = ch as u16;
@ -310,6 +307,7 @@ impl Char for char {
/// An iterator over the characters that represent a `char`, as escaped by
/// Rust's unicode escaping rules.
#[derive(Clone)]
#[stable]
pub struct EscapeUnicode {
c: char,
state: EscapeUnicodeState
@ -325,6 +323,7 @@ enum EscapeUnicodeState {
Done,
}
#[stable]
impl Iterator for EscapeUnicode {
type Item = char;
@ -370,6 +369,7 @@ impl Iterator for EscapeUnicode {
/// An iterator over the characters that represent a `char`, escaped
/// for maximum portability.
#[derive(Clone)]
#[stable]
pub struct EscapeDefault {
state: EscapeDefaultState
}
@ -382,6 +382,7 @@ enum EscapeDefaultState {
Unicode(EscapeUnicode),
}
#[stable]
impl Iterator for EscapeDefault {
type Item = char;

View File

@ -15,7 +15,7 @@ pub use self::SignificantDigits::*;
pub use self::SignFormat::*;
use char;
use char::Char;
use char::CharExt;
use fmt;
use iter::{IteratorExt, range};
use num::{cast, Float, ToPrimitive};

View File

@ -388,7 +388,7 @@ impl<'a> Formatter<'a> {
prefix: &str,
buf: &str)
-> Result {
use char::Char;
use char::CharExt;
use fmt::rt::{FlagAlternate, FlagSignPlus, FlagSignAwareZeroPad};
let mut width = buf.len();
@ -504,7 +504,7 @@ impl<'a> Formatter<'a> {
fn with_padding<F>(&mut self, padding: uint, default: rt::Alignment, f: F) -> Result where
F: FnOnce(&mut Formatter) -> Result,
{
use char::Char;
use char::CharExt;
let align = match self.align {
rt::AlignUnknown => default,
_ => self.align
@ -613,7 +613,7 @@ impl Show for str {
impl Show for char {
fn fmt(&self, f: &mut Formatter) -> Result {
use char::Char;
use char::CharExt;
let mut utf8 = [0u8; 4];
let amt = self.encode_utf8(&mut utf8).unwrap_or(0);

View File

@ -15,7 +15,7 @@
#![stable]
#![allow(missing_docs)]
use char::Char;
use char::CharExt;
use clone::Clone;
use cmp::{PartialEq, Eq};
use cmp::{PartialOrd, Ord};

View File

@ -38,7 +38,7 @@ pub use mem::drop;
// Reexported types and traits
pub use char::Char;
pub use char::CharExt;
pub use clone::Clone;
pub use cmp::{PartialEq, PartialOrd, Eq, Ord};
pub use iter::{Extend, IteratorExt};

View File

@ -50,7 +50,6 @@ use rustc::session::config;
use std::rc::Rc;
use std::u32;
use std::str::Str as StrTrait; // Conflicts with Str variant
use std::char::Char as CharTrait; // Conflicts with Char variant
use std::path::Path as FsPath; // Conflicts with Path struct
use core::DocContext;

View File

@ -225,7 +225,7 @@ pub use self::FileMode::*;
pub use self::FileAccess::*;
pub use self::IoErrorKind::*;
use char::Char;
use char::CharExt;
use clone::Clone;
use default::Default;
use error::{FromError, Error};
@ -248,7 +248,6 @@ use str;
use string::String;
use uint;
use unicode;
use unicode::char::UnicodeChar;
use vec::Vec;
// Reexports

View File

@ -16,7 +16,7 @@ use self::ExponentFormat::*;
use self::SignificantDigits::*;
use self::SignFormat::*;
use char::{self, Char};
use char::{self, CharExt};
use num::{self, Int, Float, ToPrimitive};
use num::FpCategory as Fp;
use ops::FnMut;

View File

@ -16,6 +16,7 @@ use self::PathPrefix::*;
use ascii::AsciiExt;
use c_str::{CString, ToCStr};
use char::CharExt;
use clone::Clone;
use cmp::{PartialEq, Eq, PartialOrd, Ord, Ordering};
use hash;
@ -28,7 +29,6 @@ use option::Option::{Some, None};
use slice::{SliceExt, SliceConcatExt};
use str::{SplitTerminator, FromStr, StrExt};
use string::{String, ToString};
use unicode::char::UnicodeChar;
use vec::Vec;
use super::{contains_nul, BytesContainer, GenericPath, GenericPathUnsafe};

View File

@ -22,7 +22,7 @@
// Reexported types and traits
#[stable] #[doc(no_inline)] pub use boxed::Box;
#[stable] #[doc(no_inline)] pub use char::{Char, UnicodeChar};
#[stable] #[doc(no_inline)] pub use char::CharExt;
#[stable] #[doc(no_inline)] pub use clone::Clone;
#[stable] #[doc(no_inline)] pub use cmp::{PartialEq, PartialOrd, Eq, Ord};
#[stable] #[doc(no_inline)] pub use iter::CloneIteratorExt;

View File

@ -44,9 +44,9 @@ mod u_str;
// re-export char so that std et al see it correctly
/// Character manipulation (`char` type, Unicode Scalar Value)
///
/// This module provides the `Char` and `UnicodeChar` traits, as well as their
/// implementation for the primitive `char` type, in order to allow basic character
/// manipulation.
/// This module provides the `CharExt` trait, as well as its
/// implementation for the primitive `char` type, in order to allow
/// basic character manipulation.
///
/// A `char` actually represents a
/// *[Unicode Scalar Value](http://www.unicode.org/glossary/#unicode_scalar_value)*,
@ -58,16 +58,14 @@ mod u_str;
/// however the converse is not always true due to the above range limits
/// and, as such, should be performed via the `from_u32` function..
pub mod char {
pub use core::char::{MAX, from_u32};
pub use core::char::{from_digit};
pub use core::char::Char;
pub use core::char::{MAX, from_u32, from_digit};
pub use normalize::{decompose_canonical, decompose_compatible, compose};
pub use tables::normalization::canonical_combining_class;
pub use tables::UNICODE_VERSION;
pub use u_char::UnicodeChar;
pub use u_char::CharExt;
}
pub mod str {

View File

@ -13,7 +13,7 @@
#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
/// The version of [Unicode](http://www.unicode.org/)
/// that the `UnicodeChar` and `UnicodeStrPrelude` traits are based on.
/// that the unicode parts of `CharExt` and `UnicodeStrPrelude` traits are based on.
pub const UNICODE_VERSION: (uint, uint, uint) = (7, 0, 0);
fn bsearch_range_table(c: char, r: &'static [(char,char)]) -> bool {

View File

@ -8,19 +8,102 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Unicode-intensive `char` methods.
//! Unicode-intensive `char` methods along with the `core` methods.
//!
//! These methods implement functionality for `char` that requires knowledge of
//! Unicode definitions, including normalization, categorization, and display information.
use core::char;
use core::char::CharExt as C;
use core::option::Option;
use tables::{derived_property, property, general_category, conversions, charwidth};
/// Useful functions for Unicode characters.
#[experimental = "pending prelude organization"]
pub trait UnicodeChar {
/// Functionality for manipulating `char`.
#[stable]
pub trait CharExt {
/// Checks if a `char` parses as a numeric digit in the given radix.
///
/// Compared to `is_numeric()`, this function only recognizes the characters
/// `0-9`, `a-z` and `A-Z`.
///
/// # Return value
///
/// Returns `true` if `c` is a valid digit under `radix`, and `false`
/// otherwise.
///
/// # Panics
///
/// Panics if given a radix > 36.
#[unstable = "pending integer conventions"]
fn is_digit(self, radix: uint) -> bool;
/// Converts a character to the corresponding digit.
///
/// # Return value
///
/// If `c` is between '0' and '9', the corresponding value between 0 and
/// 9. If `c` is 'a' or 'A', 10. If `c` is 'b' or 'B', 11, etc. Returns
/// none if the character does not refer to a digit in the given radix.
///
/// # Panics
///
/// Panics if given a radix outside the range [0..36].
#[unstable = "pending integer conventions"]
fn to_digit(self, radix: uint) -> Option<uint>;
/// Returns an iterator that yields the hexadecimal Unicode escape
/// of a character, as `char`s.
///
/// All characters are escaped with Rust syntax of the form `\\u{NNNN}`
/// where `NNNN` is the shortest hexadecimal representation of the code
/// point.
#[stable]
fn escape_unicode(self) -> char::EscapeUnicode;
/// Returns an iterator that yields the 'default' ASCII and
/// C++11-like literal escape of a character, as `char`s.
///
/// The default is chosen with a bias toward producing literals that are
/// legal in a variety of languages, including C++11 and similar C-family
/// languages. The exact rules are:
///
/// * Tab, CR and LF are escaped as '\t', '\r' and '\n' respectively.
/// * Single-quote, double-quote and backslash chars are backslash-
/// escaped.
/// * Any other chars in the range [0x20,0x7e] are not escaped.
/// * Any other chars are given hex Unicode escapes; see `escape_unicode`.
#[stable]
fn escape_default(self) -> char::EscapeDefault;
/// Returns the amount of bytes this character would need if encoded in
/// UTF-8.
#[stable]
fn len_utf8(self) -> uint;
/// Returns the amount of bytes this character would need if encoded in
/// UTF-16.
#[stable]
fn len_utf16(self) -> uint;
/// Encodes this character as UTF-8 into the provided byte buffer,
/// and then returns the number of bytes written.
///
/// If the buffer is not large enough, nothing will be written into it
/// and a `None` will be returned.
#[unstable = "pending decision about Iterator/Writer/Reader"]
fn encode_utf8(self, dst: &mut [u8]) -> Option<uint>;
/// Encodes this character as UTF-16 into the provided `u16` buffer,
/// and then returns the number of `u16`s written.
///
/// If the buffer is not large enough, nothing will be written into it
/// and a `None` will be returned.
#[unstable = "pending decision about Iterator/Writer/Reader"]
fn encode_utf16(self, dst: &mut [u16]) -> Option<uint>;
/// Returns whether the specified character is considered a Unicode
/// alphabetic code point.
#[stable]
fn is_alphabetic(self) -> bool;
/// Returns whether the specified character satisfies the 'XID_Start'
@ -29,6 +112,7 @@ pub trait UnicodeChar {
/// 'XID_Start' is a Unicode Derived Property specified in
/// [UAX #31](http://unicode.org/reports/tr31/#NFKC_Modifications),
/// mostly similar to ID_Start but modified for closure under NFKx.
#[experimental = "mainly needed for compiler internals"]
fn is_xid_start(self) -> bool;
/// Returns whether the specified `char` satisfies the 'XID_Continue'
@ -37,38 +121,45 @@ pub trait UnicodeChar {
/// 'XID_Continue' is a Unicode Derived Property specified in
/// [UAX #31](http://unicode.org/reports/tr31/#NFKC_Modifications),
/// mostly similar to 'ID_Continue' but modified for closure under NFKx.
#[experimental = "mainly needed for compiler internals"]
fn is_xid_continue(self) -> bool;
/// Indicates whether a character is in lowercase.
///
/// This is defined according to the terms of the Unicode Derived Core
/// Property `Lowercase`.
#[stable]
fn is_lowercase(self) -> bool;
/// Indicates whether a character is in uppercase.
///
/// This is defined according to the terms of the Unicode Derived Core
/// Property `Uppercase`.
#[stable]
fn is_uppercase(self) -> bool;
/// Indicates whether a character is whitespace.
///
/// Whitespace is defined in terms of the Unicode Property `White_Space`.
#[stable]
fn is_whitespace(self) -> bool;
/// Indicates whether a character is alphanumeric.
///
/// Alphanumericness is defined in terms of the Unicode General Categories
/// 'Nd', 'Nl', 'No' and the Derived Core Property 'Alphabetic'.
#[stable]
fn is_alphanumeric(self) -> bool;
/// Indicates whether a character is a control code point.
///
/// Control code points are defined in terms of the Unicode General
/// Category `Cc`.
#[stable]
fn is_control(self) -> bool;
/// Indicates whether the character is numeric (Nd, Nl, or No).
#[stable]
fn is_numeric(self) -> bool;
/// Converts a character to its lowercase equivalent.
@ -80,6 +171,7 @@ pub trait UnicodeChar {
///
/// Returns the lowercase equivalent of the character, or the character
/// itself if no conversion is possible.
#[experimental = "pending case transformation decisions"]
fn to_lowercase(self) -> char;
/// Converts a character to its uppercase equivalent.
@ -102,6 +194,7 @@ pub trait UnicodeChar {
/// [`SpecialCasing`.txt`]: ftp://ftp.unicode.org/Public/UNIDATA/SpecialCasing.txt
///
/// [2]: http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf#G33992
#[experimental = "pending case transformation decisions"]
fn to_uppercase(self) -> char;
/// Returns this character's displayed width in columns, or `None` if it is a
@ -117,8 +210,26 @@ pub trait UnicodeChar {
fn width(self, is_cjk: bool) -> Option<uint>;
}
#[experimental = "pending prelude organization"]
impl UnicodeChar for char {
#[stable]
impl CharExt for char {
#[unstable = "pending integer conventions"]
fn is_digit(self, radix: uint) -> bool { C::is_digit(self, radix) }
#[unstable = "pending integer conventions"]
fn to_digit(self, radix: uint) -> Option<uint> { C::to_digit(self, radix) }
#[stable]
fn escape_unicode(self) -> char::EscapeUnicode { C::escape_unicode(self) }
#[stable]
fn escape_default(self) -> char::EscapeDefault { C::escape_default(self) }
#[stable]
fn len_utf8(self) -> uint { C::len_utf8(self) }
#[stable]
fn len_utf16(self) -> uint { C::len_utf16(self) }
#[unstable = "pending decision about Iterator/Writer/Reader"]
fn encode_utf8(self, dst: &mut [u8]) -> Option<uint> { C::encode_utf8(self, dst) }
#[unstable = "pending decision about Iterator/Writer/Reader"]
fn encode_utf16(self, dst: &mut [u16]) -> Option<uint> { C::encode_utf16(self, dst) }
#[stable]
fn is_alphabetic(self) -> bool {
match self {
'a' ... 'z' | 'A' ... 'Z' => true,
@ -127,10 +238,13 @@ impl UnicodeChar for char {
}
}
#[experimental = "mainly needed for compiler internals"]
fn is_xid_start(self) -> bool { derived_property::XID_Start(self) }
#[experimental = "mainly needed for compiler internals"]
fn is_xid_continue(self) -> bool { derived_property::XID_Continue(self) }
#[stable]
fn is_lowercase(self) -> bool {
match self {
'a' ... 'z' => true,
@ -139,6 +253,7 @@ impl UnicodeChar for char {
}
}
#[stable]
fn is_uppercase(self) -> bool {
match self {
'A' ... 'Z' => true,
@ -147,6 +262,7 @@ impl UnicodeChar for char {
}
}
#[stable]
fn is_whitespace(self) -> bool {
match self {
' ' | '\x09' ... '\x0d' => true,
@ -155,12 +271,15 @@ impl UnicodeChar for char {
}
}
#[stable]
fn is_alphanumeric(self) -> bool {
self.is_alphabetic() || self.is_numeric()
}
#[stable]
fn is_control(self) -> bool { general_category::Cc(self) }
#[stable]
fn is_numeric(self) -> bool {
match self {
'0' ... '9' => true,
@ -169,8 +288,10 @@ impl UnicodeChar for char {
}
}
#[experimental = "pending case transformation decisions"]
fn to_lowercase(self) -> char { conversions::to_lower(self) }
#[experimental = "pending case transformation decisions"]
fn to_uppercase(self) -> char { conversions::to_upper(self) }
#[experimental = "needs expert opinion. is_cjk flag stands out as ugly"]

View File

@ -13,7 +13,7 @@
//! Unicode-intensive string manipulations.
//!
//! This module provides functionality to `str` that requires the Unicode methods provided by the
//! UnicodeChar trait.
//! unicode parts of the CharExt trait.
use self::GraphemeState::*;
use core::prelude::*;
@ -26,7 +26,7 @@ use core::num::Int;
use core::slice;
use core::str::Split;
use u_char::UnicodeChar;
use u_char::CharExt as UCharExt; // conflicts with core::prelude::CharExt
use tables::grapheme::GraphemeCat;
/// An iterator over the words of a string, separated by a sequence of whitespace
@ -529,7 +529,7 @@ impl<I> Iterator for Utf16Encoder<I> where I: Iterator<Item=char> {
let mut buf = [0u16; 2];
self.chars.next().map(|ch| {
let n = ch.encode_utf16(buf.as_mut_slice()).unwrap_or(0);
let n = CharExt::encode_utf16(ch, buf.as_mut_slice()).unwrap_or(0);
if n == 2 { self.extra = buf[1]; }
buf[0]
})