407 lines
9.7 KiB
Go
407 lines
9.7 KiB
Go
// Copyright 2013 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package bufio_test
|
|
|
|
import (
|
|
. "bufio"
|
|
"bytes"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// Test white space table matches the Unicode definition.
|
|
func TestSpace(t *testing.T) {
|
|
for r := rune(0); r <= utf8.MaxRune; r++ {
|
|
if IsSpace(r) != unicode.IsSpace(r) {
|
|
t.Fatalf("white space property disagrees: %#U should be %t", r, unicode.IsSpace(r))
|
|
}
|
|
}
|
|
}
|
|
|
|
var scanTests = []string{
|
|
"",
|
|
"a",
|
|
"¼",
|
|
"☹",
|
|
"\x81", // UTF-8 error
|
|
"\uFFFD", // correctly encoded RuneError
|
|
"abcdefgh",
|
|
"abc def\n\t\tgh ",
|
|
"abc¼☹\x81\uFFFD日本語\x82abc",
|
|
}
|
|
|
|
func TestScanByte(t *testing.T) {
|
|
for n, test := range scanTests {
|
|
buf := strings.NewReader(test)
|
|
s := NewScanner(buf)
|
|
s.Split(ScanBytes)
|
|
var i int
|
|
for i = 0; s.Scan(); i++ {
|
|
if b := s.Bytes(); len(b) != 1 || b[0] != test[i] {
|
|
t.Errorf("#%d: %d: expected %q got %q", n, i, test, b)
|
|
}
|
|
}
|
|
if i != len(test) {
|
|
t.Errorf("#%d: termination expected at %d; got %d", n, len(test), i)
|
|
}
|
|
err := s.Err()
|
|
if err != nil {
|
|
t.Errorf("#%d: %v", n, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test that the rune splitter returns same sequence of runes (not bytes) as for range string.
|
|
func TestScanRune(t *testing.T) {
|
|
for n, test := range scanTests {
|
|
buf := strings.NewReader(test)
|
|
s := NewScanner(buf)
|
|
s.Split(ScanRunes)
|
|
var i, runeCount int
|
|
var expect rune
|
|
// Use a string range loop to validate the sequence of runes.
|
|
for i, expect = range string(test) {
|
|
if !s.Scan() {
|
|
break
|
|
}
|
|
runeCount++
|
|
got, _ := utf8.DecodeRune(s.Bytes())
|
|
if got != expect {
|
|
t.Errorf("#%d: %d: expected %q got %q", n, i, expect, got)
|
|
}
|
|
}
|
|
if s.Scan() {
|
|
t.Errorf("#%d: scan ran too long, got %q", n, s.Text())
|
|
}
|
|
testRuneCount := utf8.RuneCountInString(test)
|
|
if runeCount != testRuneCount {
|
|
t.Errorf("#%d: termination expected at %d; got %d", n, testRuneCount, runeCount)
|
|
}
|
|
err := s.Err()
|
|
if err != nil {
|
|
t.Errorf("#%d: %v", n, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
var wordScanTests = []string{
|
|
"",
|
|
" ",
|
|
"\n",
|
|
"a",
|
|
" a ",
|
|
"abc def",
|
|
" abc def ",
|
|
" abc\tdef\nghi\rjkl\fmno\vpqr\u0085stu\u00a0\n",
|
|
}
|
|
|
|
// Test that the word splitter returns the same data as strings.Fields.
|
|
func TestScanWords(t *testing.T) {
|
|
for n, test := range wordScanTests {
|
|
buf := strings.NewReader(test)
|
|
s := NewScanner(buf)
|
|
s.Split(ScanWords)
|
|
words := strings.Fields(test)
|
|
var wordCount int
|
|
for wordCount = 0; wordCount < len(words); wordCount++ {
|
|
if !s.Scan() {
|
|
break
|
|
}
|
|
got := s.Text()
|
|
if got != words[wordCount] {
|
|
t.Errorf("#%d: %d: expected %q got %q", n, wordCount, words[wordCount], got)
|
|
}
|
|
}
|
|
if s.Scan() {
|
|
t.Errorf("#%d: scan ran too long, got %q", n, s.Text())
|
|
}
|
|
if wordCount != len(words) {
|
|
t.Errorf("#%d: termination expected at %d; got %d", n, len(words), wordCount)
|
|
}
|
|
err := s.Err()
|
|
if err != nil {
|
|
t.Errorf("#%d: %v", n, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// slowReader is a reader that returns only a few bytes at a time, to test the incremental
|
|
// reads in Scanner.Scan.
|
|
type slowReader struct {
|
|
max int
|
|
buf io.Reader
|
|
}
|
|
|
|
func (sr *slowReader) Read(p []byte) (n int, err error) {
|
|
if len(p) > sr.max {
|
|
p = p[0:sr.max]
|
|
}
|
|
return sr.buf.Read(p)
|
|
}
|
|
|
|
// genLine writes to buf a predictable but non-trivial line of text of length
|
|
// n, including the terminal newline and an occasional carriage return.
|
|
// If addNewline is false, the \r and \n are not emitted.
|
|
func genLine(buf *bytes.Buffer, lineNum, n int, addNewline bool) {
|
|
buf.Reset()
|
|
doCR := lineNum%5 == 0
|
|
if doCR {
|
|
n--
|
|
}
|
|
for i := 0; i < n-1; i++ { // Stop early for \n.
|
|
c := 'a' + byte(lineNum+i)
|
|
if c == '\n' || c == '\r' { // Don't confuse us.
|
|
c = 'N'
|
|
}
|
|
buf.WriteByte(c)
|
|
}
|
|
if addNewline {
|
|
if doCR {
|
|
buf.WriteByte('\r')
|
|
}
|
|
buf.WriteByte('\n')
|
|
}
|
|
return
|
|
}
|
|
|
|
// Test the line splitter, including some carriage returns but no long lines.
|
|
func TestScanLongLines(t *testing.T) {
|
|
const smallMaxTokenSize = 256 // Much smaller for more efficient testing.
|
|
// Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
|
|
tmp := new(bytes.Buffer)
|
|
buf := new(bytes.Buffer)
|
|
lineNum := 0
|
|
j := 0
|
|
for i := 0; i < 2*smallMaxTokenSize; i++ {
|
|
genLine(tmp, lineNum, j, true)
|
|
if j < smallMaxTokenSize {
|
|
j++
|
|
} else {
|
|
j--
|
|
}
|
|
buf.Write(tmp.Bytes())
|
|
lineNum++
|
|
}
|
|
s := NewScanner(&slowReader{1, buf})
|
|
s.Split(ScanLines)
|
|
s.MaxTokenSize(smallMaxTokenSize)
|
|
j = 0
|
|
for lineNum := 0; s.Scan(); lineNum++ {
|
|
genLine(tmp, lineNum, j, false)
|
|
if j < smallMaxTokenSize {
|
|
j++
|
|
} else {
|
|
j--
|
|
}
|
|
line := tmp.String() // We use the string-valued token here, for variety.
|
|
if s.Text() != line {
|
|
t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Text(), line)
|
|
}
|
|
}
|
|
err := s.Err()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// Test that the line splitter errors out on a long line.
|
|
func TestScanLineTooLong(t *testing.T) {
|
|
const smallMaxTokenSize = 256 // Much smaller for more efficient testing.
|
|
// Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize.
|
|
tmp := new(bytes.Buffer)
|
|
buf := new(bytes.Buffer)
|
|
lineNum := 0
|
|
j := 0
|
|
for i := 0; i < 2*smallMaxTokenSize; i++ {
|
|
genLine(tmp, lineNum, j, true)
|
|
j++
|
|
buf.Write(tmp.Bytes())
|
|
lineNum++
|
|
}
|
|
s := NewScanner(&slowReader{3, buf})
|
|
s.Split(ScanLines)
|
|
s.MaxTokenSize(smallMaxTokenSize)
|
|
j = 0
|
|
for lineNum := 0; s.Scan(); lineNum++ {
|
|
genLine(tmp, lineNum, j, false)
|
|
if j < smallMaxTokenSize {
|
|
j++
|
|
} else {
|
|
j--
|
|
}
|
|
line := tmp.Bytes()
|
|
if !bytes.Equal(s.Bytes(), line) {
|
|
t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
|
|
}
|
|
}
|
|
err := s.Err()
|
|
if err != ErrTooLong {
|
|
t.Fatalf("expected ErrTooLong; got %s", err)
|
|
}
|
|
}
|
|
|
|
// Test that the line splitter handles a final line without a newline.
|
|
func testNoNewline(text string, lines []string, t *testing.T) {
|
|
buf := strings.NewReader(text)
|
|
s := NewScanner(&slowReader{7, buf})
|
|
s.Split(ScanLines)
|
|
for lineNum := 0; s.Scan(); lineNum++ {
|
|
line := lines[lineNum]
|
|
if s.Text() != line {
|
|
t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
|
|
}
|
|
}
|
|
err := s.Err()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
var noNewlineLines = []string{
|
|
"abcdefghijklmn\nopqrstuvwxyz",
|
|
}
|
|
|
|
// Test that the line splitter handles a final line without a newline.
|
|
func TestScanLineNoNewline(t *testing.T) {
|
|
const text = "abcdefghijklmn\nopqrstuvwxyz"
|
|
lines := []string{
|
|
"abcdefghijklmn",
|
|
"opqrstuvwxyz",
|
|
}
|
|
testNoNewline(text, lines, t)
|
|
}
|
|
|
|
// Test that the line splitter handles a final line with a carriage return but no newline.
|
|
func TestScanLineReturnButNoNewline(t *testing.T) {
|
|
const text = "abcdefghijklmn\nopqrstuvwxyz\r"
|
|
lines := []string{
|
|
"abcdefghijklmn",
|
|
"opqrstuvwxyz",
|
|
}
|
|
testNoNewline(text, lines, t)
|
|
}
|
|
|
|
// Test that the line splitter handles a final empty line.
|
|
func TestScanLineEmptyFinalLine(t *testing.T) {
|
|
const text = "abcdefghijklmn\nopqrstuvwxyz\n\n"
|
|
lines := []string{
|
|
"abcdefghijklmn",
|
|
"opqrstuvwxyz",
|
|
"",
|
|
}
|
|
testNoNewline(text, lines, t)
|
|
}
|
|
|
|
// Test that the line splitter handles a final empty line with a carriage return but no newline.
|
|
func TestScanLineEmptyFinalLineWithCR(t *testing.T) {
|
|
const text = "abcdefghijklmn\nopqrstuvwxyz\n\r"
|
|
lines := []string{
|
|
"abcdefghijklmn",
|
|
"opqrstuvwxyz",
|
|
"",
|
|
}
|
|
testNoNewline(text, lines, t)
|
|
}
|
|
|
|
var testError = errors.New("testError")
|
|
|
|
// Test the correct error is returned when the split function errors out.
|
|
func TestSplitError(t *testing.T) {
|
|
// Create a split function that delivers a little data, then a predictable error.
|
|
numSplits := 0
|
|
const okCount = 7
|
|
errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
|
if atEOF {
|
|
panic("didn't get enough data")
|
|
}
|
|
if numSplits >= okCount {
|
|
return 0, nil, testError
|
|
}
|
|
numSplits++
|
|
return 1, data[0:1], nil
|
|
}
|
|
// Read the data.
|
|
const text = "abcdefghijklmnopqrstuvwxyz"
|
|
buf := strings.NewReader(text)
|
|
s := NewScanner(&slowReader{1, buf})
|
|
s.Split(errorSplit)
|
|
var i int
|
|
for i = 0; s.Scan(); i++ {
|
|
if len(s.Bytes()) != 1 || text[i] != s.Bytes()[0] {
|
|
t.Errorf("#%d: expected %q got %q", i, text[i], s.Bytes()[0])
|
|
}
|
|
}
|
|
// Check correct termination location and error.
|
|
if i != okCount {
|
|
t.Errorf("unexpected termination; expected %d tokens got %d", okCount, i)
|
|
}
|
|
err := s.Err()
|
|
if err != testError {
|
|
t.Fatalf("expected %q got %v", testError, err)
|
|
}
|
|
}
|
|
|
|
// Test that an EOF is overridden by a user-generated scan error.
|
|
func TestErrAtEOF(t *testing.T) {
|
|
s := NewScanner(strings.NewReader("1 2 33"))
|
|
// This spitter will fail on last entry, after s.err==EOF.
|
|
split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
|
advance, token, err = ScanWords(data, atEOF)
|
|
if len(token) > 1 {
|
|
if s.ErrOrEOF() != io.EOF {
|
|
t.Fatal("not testing EOF")
|
|
}
|
|
err = testError
|
|
}
|
|
return
|
|
}
|
|
s.Split(split)
|
|
for s.Scan() {
|
|
}
|
|
if s.Err() != testError {
|
|
t.Fatal("wrong error:", s.Err())
|
|
}
|
|
}
|
|
|
|
// Test for issue 5268.
|
|
type alwaysError struct{}
|
|
|
|
func (alwaysError) Read(p []byte) (int, error) {
|
|
return 0, io.ErrUnexpectedEOF
|
|
}
|
|
|
|
func TestNonEOFWithEmptyRead(t *testing.T) {
|
|
scanner := NewScanner(alwaysError{})
|
|
for scanner.Scan() {
|
|
t.Fatal("read should fail")
|
|
}
|
|
err := scanner.Err()
|
|
if err != io.ErrUnexpectedEOF {
|
|
t.Errorf("unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
// Test that Scan finishes if we have endless empty reads.
|
|
type endlessZeros struct{}
|
|
|
|
func (endlessZeros) Read(p []byte) (int, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
func TestBadReader(t *testing.T) {
|
|
scanner := NewScanner(endlessZeros{})
|
|
for scanner.Scan() {
|
|
t.Fatal("read should fail")
|
|
}
|
|
err := scanner.Err()
|
|
if err != io.ErrNoProgress {
|
|
t.Errorf("unexpected error: %v", err)
|
|
}
|
|
}
|