Editando: scan.go
// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package bufio import ( "bytes" "errors" "io" "unicode/utf8" ) // Scanner provides a convenient interface for reading data such as // a file of newline-delimited lines of text. Successive calls to // the [Scanner.Scan] method will step through the 'tokens' of a file, skipping // the bytes between the tokens. The specification of a token is // defined by a split function of type [SplitFunc]; the default split // function breaks the input into lines with line termination stripped. [Scanner.Split] // functions are defined in this package for scanning a file into // lines, bytes, UTF-8-encoded runes, and space-delimited words. The // client may instead provide a custom split function. // // Scanning stops unrecoverably at EOF, the first I/O error, or a token too // large to fit in the [Scanner.Buffer]. When a scan stops, the reader may have // advanced arbitrarily far past the last token. Programs that need more // control over error handling or large tokens, or must run sequential scans // on a reader, should use [bufio.Reader] instead. type Scanner struct { r io.Reader // The reader provided by the client. split SplitFunc // The function to split the tokens. maxTokenSize int // Maximum size of a token; modified by tests. token []byte // Last token returned by split. buf []byte // Buffer used as argument to split. start int // First non-processed byte in buf. end int // End of data in buf. err error // Sticky error. empties int // Count of successive empty tokens. scanCalled bool // Scan has been called; buffer is in use. done bool // Scan has finished. } // SplitFunc is the signature of the split function used to tokenize the // input. The arguments are an initial substring of the remaining unprocessed // data and a flag, atEOF, that reports whether the [Reader] has no more data // to give. The return values are the number of bytes to advance the input // and the next token to return to the user, if any, plus an error, if any. // // Scanning stops if the function returns an error, in which case some of // the input may be discarded. If that error is [ErrFinalToken], scanning // stops with no error. A non-nil token delivered with [ErrFinalToken] // will be the last token, and a nil token with [ErrFinalToken] // immediately stops the scanning. // // Otherwise, the [Scanner] advances the input. If the token is not nil, // the [Scanner] returns it to the user. If the token is nil, the // Scanner reads more data and continues scanning; if there is no more // data--if atEOF was true--the [Scanner] returns. If the data does not // yet hold a complete token, for instance if it has no newline while // scanning lines, a [SplitFunc] can return (0, nil, nil) to signal the // [Scanner] to read more data into the slice and try again with a // longer slice starting at the same point in the input. // // The function is never called with an empty data slice unless atEOF // is true. If atEOF is true, however, data may be non-empty and, // as always, holds unprocessed text. type SplitFunc func(data []byte, atEOF bool) (advance int, token []byte, err error) // Errors returned by Scanner. var ( ErrTooLong = errors.New("bufio.Scanner: token too long") ErrNegativeAdvance = errors.New("bufio.Scanner: SplitFunc returns negative advance count") ErrAdvanceTooFar = errors.New("bufio.Scanner: SplitFunc returns advance count beyond input") ErrBadReadCount = errors.New("bufio.Scanner: Read returned impossible count") ) const ( // MaxScanTokenSize is the maximum size used to buffer a token // unless the user provides an explicit buffer with [Scanner.Buffer]. // The actual maximum token size may be smaller as the buffer // may need to include, for instance, a newline. MaxScanTokenSize = 64 * 1024 startBufSize = 4096 // Size of initial allocation for buffer. ) // NewScanner returns a new [Scanner] to read from r. // The split function defaults to [ScanLines]. func NewScanner(r io.Reader) *Scanner { return &Scanner{ r: r, split: ScanLines, maxTokenSize: MaxScanTokenSize, } } // Err returns the first non-EOF error that was encountered by the [Scanner]. func (s *Scanner) Err() error { if s.err == io.EOF { return nil } return s.err } // Bytes returns the most recent token generated by a call to [Scanner.Scan]. // The underlying array may point to data that will be overwritten // by a subsequent call to Scan. It does no allocation. func (s *Scanner) Bytes() []byte { return s.token } // Text returns the most recent token generated by a call to [Scanner.Scan] // as a newly allocated string holding its bytes. func (s *Scanner) Text() string { return string(s.token) } // ErrFinalToken is a special sentinel error value. It is intended to be // returned by a Split function to indicate that the scanning should stop // with no error. If the token being delivered with this error is not nil, // the token is the last token. // // The value is useful to stop processing early or when it is necessary to // deliver a final empty token (which is different from a nil token). // One could achieve the same behavior with a custom error value but // providing one here is tidier. // See the emptyFinalToken example for a use of this value. var ErrFinalToken = errors.New("final token") // Scan advances the [Scanner] to the next token, which will then be // available through the [Scanner.Bytes] or [Scanner.Text] method. It returns false when // there are no more tokens, either by reaching the end of the input or an error. // After Scan returns false, the [Scanner.Err] method will return any error that // occurred during scanning, except that if it was [io.EOF], [Scanner.Err] // will return nil. // Scan panics if the split function returns too many empty // tokens without advancing the input. This is a common error mode for // scanners. func (s *Scanner) Scan() bool { if s.done { return false } s.scanCalled = true // Loop until we have a token. for { // See if we can get a token with what we already have. // If we've run out of data but have an error, give the split function // a chance to recover any remaining, possibly empty token. if s.end > s.start || s.err != nil { advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil) if err != nil { if err == ErrFinalToken { s.token = token s.done = true // When token is not nil, it means the scanning stops // with a trailing token, and thus the return value // should be true to indicate the existence of the token. return token != nil } s.setErr(err) return false } if !s.advance(advance) { return false } s.token = token if token != nil { if s.err == nil || advance > 0 { s.empties = 0 } else { // Returning tokens not advancing input at EOF. s.empties++ if s.empties > maxConsecutiveEmptyReads { panic("bufio.Scan: too many empty tokens without progressing") } } return true } } // We cannot generate a token with what we are holding. // If we've already hit EOF or an I/O error, we are done. if s.err != nil { // Shut it down. s.start = 0 s.end = 0 return false } // Must read more data. // First, shift data to beginning of buffer if there's lots of empty space // or space is needed. if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) { copy(s.buf, s.buf[s.start:s.end]) s.end -= s.start s.start = 0 } // Is the buffer full? If so, resize. if s.end == len(s.buf) { // Guarantee no overflow in the multiplication below. const maxInt = int(^uint(0) >> 1) if len(s.buf) >= s.maxTokenSize || len(s.buf) > maxInt/2 { s.setErr(ErrTooLong) return false } newSize := len(s.buf) * 2 if newSize == 0 { newSize = startBufSize } newSize = min(newSize, s.maxTokenSize) newBuf := make([]byte, newSize) copy(newBuf, s.buf[s.start:s.end]) s.buf = newBuf s.end -= s.start s.start = 0 } // Finally we can read some input. Make sure we don't get stuck with // a misbehaving Reader. Officially we don't need to do this, but let's // be extra careful: Scanner is for safe, simple jobs. for loop := 0; ; { n, err := s.r.Read(s.buf[s.end:len(s.buf)]) if n < 0 || len(s.buf)-s.end < n { s.setErr(ErrBadReadCount) break } s.end += n if err != nil { s.setErr(err) break } if n > 0 { s.empties = 0 break } loop++ if loop > maxConsecutiveEmptyReads { s.setErr(io.ErrNoProgress) break } } } } // advance consumes n bytes of the buffer. It reports whether the advance was legal. func (s *Scanner) advance(n int) bool { if n < 0 { s.setErr(ErrNegativeAdvance) return false } if n > s.end-s.start { s.setErr(ErrAdvanceTooFar) return false } s.start += n return true } // setErr records the first error encountered. func (s *Scanner) setErr(err error) { if s.err == nil || s.err == io.EOF { s.err = err } } // Buffer sets the initial buffer to use when scanning // and the maximum size of buffer that may be allocated during scanning. // The maximum token size must be less than the larger of max and cap(buf). // If max <= cap(buf), [Scanner.Scan] will use this buffer only and do no allocation. // // By default, [Scanner.Scan] uses an internal buffer and sets the // maximum token size to [MaxScanTokenSize]. // // Buffer panics if it is called after scanning has started. func (s *Scanner) Buffer(buf []byte, max int) { if s.scanCalled { panic("Buffer called after Scan") } s.buf = buf[0:cap(buf)] s.maxTokenSize = max } // Split sets the split function for the [Scanner]. // The default split function is [ScanLines]. // // Split panics if it is called after scanning has started. func (s *Scanner) Split(split SplitFunc) { if s.scanCalled { panic("Split called after Scan") } s.split = split } // Split functions // ScanBytes is a split function for a [Scanner] that returns each byte as a token. func ScanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { return 0, nil, nil } return 1, data[0:1], nil } var errorRune = []byte(string(utf8.RuneError)) // ScanRunes is a split function for a [Scanner] that returns each // UTF-8-encoded rune as a token. The sequence of runes returned is // equivalent to that from a range loop over the input as a string, which // means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd". // Because of the Scan interface, this makes it impossible for the client to // distinguish correctly encoded replacement runes from encoding errors. func ScanRunes(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { return 0, nil, nil } // Fast path 1: ASCII. if data[0] < utf8.RuneSelf { return 1, data[0:1], nil } // Fast path 2: Correct UTF-8 decode without error. _, width := utf8.DecodeRune(data) if width > 1 { // It's a valid encoding. Width cannot be one for a correctly encoded // non-ASCII rune. return width, data[0:width], nil } // We know it's an error: we have width==1 and implicitly r==utf8.RuneError. // Is the error because there wasn't a full rune to be decoded? // FullRune distinguishes correctly between erroneous and incomplete encodings. if !atEOF && !utf8.FullRune(data) { // Incomplete; get more bytes. return 0, nil, nil } // We have a real UTF-8 encoding error. Return a properly encoded error rune // but advance only one byte. This matches the behavior of a range loop over // an incorrectly encoded string. return 1, errorRune, nil } // dropCR drops a terminal \r from the data. func dropCR(data []byte) []byte { if len(data) > 0 && data[len(data)-1] == '\r' { return data[0 : len(data)-1] } return data } // ScanLines is a split function for a [Scanner] that returns each line of // text, stripped of any trailing end-of-line marker. The returned line may // be empty. The end-of-line marker is one optional carriage return followed // by one mandatory newline. In regular expression notation, it is `\r?\n`. // The last non-empty line of input will be returned even if it has no // newline. func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { return 0, nil, nil } if i := bytes.IndexByte(data, '\n'); i >= 0 { // We have a full newline-terminated line. return i + 1, dropCR(data[0:i]), nil } // If we're at EOF, we have a final, non-terminated line. Return it. if atEOF { return len(data), dropCR(data), nil } // Request more data. return 0, nil, nil } // isSpace reports whether the character is a Unicode white space character. // We avoid dependency on the unicode package, but check validity of the implementation // in the tests. func isSpace(r rune) bool { if r <= '\u00FF' { // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. switch r { case ' ', '\t', '\n', '\v', '\f', '\r': return true case '\u0085', '\u00A0': return true } return false } // High-valued ones. if '\u2000' <= r && r <= '\u200a' { return true } switch r { case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': return true } return false } // ScanWords is a split function for a [Scanner] that returns each // space-separated word of text, with surrounding spaces deleted. It will // never return an empty string. The definition of space is set by // unicode.IsSpace. func ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) { // Skip leading spaces. start := 0 for width := 0; start < len(data); start += width { var r rune r, width = utf8.DecodeRune(data[start:]) if !isSpace(r) { break } } // Scan until space, marking end of word. for width, i := 0, start; i < len(data); i += width { var r rune r, width = utf8.DecodeRune(data[i:]) if isSpace(r) { return i + width, data[start:i], nil } } // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. if atEOF && len(data) > start { return len(data), data[start:], nil } // Request more data. return start, nil, nil }
Cancelar
Kerym Chaeceran