bufreader.rs - source
Description: Source of the Rust file `library/std/src/io/buffered/bufreader.rs`.

#
std/io/buffered/
bufreader.rs

    1mod buffer;
    2
    3use buffer::Buffer;
    4
    5use crate::fmt;
    6use crate::io::{
    7    self, BorrowedCursor, BufRead, DEFAULT_BUF_SIZE, IoSliceMut, Read, Seek, SeekFrom, SizeHint,
    8    SpecReadByte, uninlined_slow_read_byte,
    9};
    10
    11/// The `BufReader<R>` struct adds buffering to any reader.
    12///
    13/// It can be excessively inefficient to work directly with a [`Read`] instance.
    14/// For example, every call to [`read`][`TcpStream::read`] on [`TcpStream`]
    15/// results in a system call. A `BufReader<R>` performs large, infrequent reads on
    16/// the underlying [`Read`] and maintains an in-memory buffer of the results.
    17///
    18/// `BufReader<R>` can improve the speed of programs that make *small* and
    19/// *repeated* read calls to the same file or network socket. It does not
    20/// help when reading very large amounts at once, or reading just one or a few
    21/// times. It also provides no advantage when reading from a source that is
    22/// already in memory, like a <code>[Vec]\<u8></code>.
    23///
    24/// When the `BufReader<R>` is dropped, the contents of its buffer will be
    25/// discarded. Creating multiple instances of a `BufReader<R>` on the same
    26/// stream can cause data loss. Reading from the underlying reader after
    27/// unwrapping the `BufReader<R>` with [`BufReader::into_inner`] can also cause
    28/// data loss.
    29///
    30/// [`TcpStream::read`]: crate::net::TcpStream::read
    31/// [`TcpStream`]: crate::net::TcpStream
    32///
    33/// # Examples
    34///
    35/// ```no_run
    36/// use std::io::prelude::*;
    37/// use std::io::BufReader;
    38/// use std::fs::File;
    39///
    40/// fn main() -> std::io::Result<()> {
    41///     let f = File::open("log.txt")?;
    42///     let mut reader = BufReader::new(f);
    43///
    44///     let mut line = String::new();
    45///     let len = reader.read_line(&mut line)?;
    46///     println!("First line is {len} bytes long");
    47///     Ok(())
    48/// }
    49/// ```
    50#[stable(feature = "rust1", since = "1.0.0")]
    51pub struct BufReader<R: ?Sized> {
    52    buf: Buffer,
    53    inner: R,
    54}
    55
    56impl<R: Read> BufReader<R> {
    57    /// Creates a new `BufReader<R>` with a default buffer capacity. The default is currently 8 KiB,
    58    /// but may change in the future.
    59    ///
    60    /// # Examples
    61    ///
    62    /// ```no_run
    63    /// use std::io::BufReader;
    64    /// use std::fs::File;
    65    ///
    66    /// fn main() -> std::io::Result<()> {
    67    ///     let f = File::open("log.txt")?;
    68    ///     let reader = BufReader::new(f);
    69    ///     Ok(())
    70    /// }
    71    /// ```
    72    #[stable(feature = "rust1", since = "1.0.0")]
    73    pub fn new(inner: R) -> BufReader<R> {
    74        BufReader::with_capacity(DEFAULT_BUF_SIZE, inner)
    75    }
    76
    77    pub(crate) fn try_new_buffer() -> io::Result<Buffer> {
    78        Buffer::try_with_capacity(DEFAULT_BUF_SIZE)
    79    }
    80
    81    pub(crate) fn with_buffer(inner: R, buf: Buffer) -> Self {
    82        Self { inner, buf }
    83    }
    84
    85    /// Creates a new `BufReader<R>` with the specified buffer capacity.
    86    ///
    87    /// # Examples
    88    ///
    89    /// Creating a buffer with ten bytes of capacity:
    90    ///
    91    /// ```no_run
    92    /// use std::io::BufReader;
    93    /// use std::fs::File;
    94    ///
    95    /// fn main() -> std::io::Result<()> {
    96    ///     let f = File::open("log.txt")?;
    97    ///     let reader = BufReader::with_capacity(10, f);
    98    ///     Ok(())
    99    /// }
    100    /// ```
    101    #[stable(feature = "rust1", since = "1.0.0")]
    102    pub fn with_capacity(capacity: usize, inner: R) -> BufReader<R> {
    103        BufReader { inner, buf: Buffer::with_capacity(capacity) }
    104    }
    105}
    106
    107impl<R: Read + ?Sized> BufReader<R> {
    108    /// Attempt to look ahead `n` bytes.
    109    ///
    110    /// `n` must be less than or equal to `capacity`.
    111    ///
    112    /// The returned slice may be less than `n` bytes long if
    113    /// end of file is reached.
    114    ///
    115    /// After calling this method, you may call [`consume`](BufRead::consume)
    116    /// with a value less than or equal to `n` to advance over some or all of
    117    /// the returned bytes.
    118    ///
    119    /// ## Examples
    120    ///
    121    /// ```rust
    122    /// #![feature(bufreader_peek)]
    123    /// use std::io::{Read, BufReader};
    124    ///
    125    /// let mut bytes = &b"oh, hello there"[..];
    126    /// let mut rdr = BufReader::with_capacity(6, &mut bytes);
    127    /// assert_eq!(rdr.peek(2).unwrap(), b"oh");
    128    /// let mut buf = [0; 4];
    129    /// rdr.read(&mut buf[..]).unwrap();
    130    /// assert_eq!(&buf, b"oh, ");
    131    /// assert_eq!(rdr.peek(5).unwrap(), b"hello");
    132    /// let mut s = String::new();
    133    /// rdr.read_to_string(&mut s).unwrap();
    134    /// assert_eq!(&s, "hello there");
    135    /// assert_eq!(rdr.peek(1).unwrap().len(), 0);
    136    /// ```
    137    #[unstable(feature = "bufreader_peek", issue = "128405")]
    138    pub fn peek(&mut self, n: usize) -> io::Result<&[u8]> {
    139        assert!(n <= self.capacity());
    140        while n > self.buf.buffer().len() {
    141            if self.buf.pos() > 0 {
    142                self.buf.backshift();
    143            }
    144            let new = self.buf.read_more(&mut self.inner)?;
    145            if new == 0 {
    146                // end of file, no more bytes to read
    147                return Ok(&self.buf.buffer()[..]);
    148            }
    149            debug_assert_eq!(self.buf.pos(), 0);
    150        }
    151        Ok(&self.buf.buffer()[..n])
    152    }
    153}
    154
    155impl<R: ?Sized> BufReader<R> {
    156    /// Gets a reference to the underlying reader.
    157    ///
    158    /// It is inadvisable to directly read from the underlying reader.
    159    ///
    160    /// # Examples
    161    ///
    162    /// ```no_run
    163    /// use std::io::BufReader;
    164    /// use std::fs::File;
    165    ///
    166    /// fn main() -> std::io::Result<()> {
    167    ///     let f1 = File::open("log.txt")?;
    168    ///     let reader = BufReader::new(f1);
    169    ///
    170    ///     let f2 = reader.get_ref();
    171    ///     Ok(())
    172    /// }
    173    /// ```
    174    #[stable(feature = "rust1", since = "1.0.0")]
    175    pub fn get_ref(&self) -> &R {
    176        &self.inner
    177    }
    178
    179    /// Gets a mutable reference to the underlying reader.
    180    ///
    181    /// It is inadvisable to directly read from the underlying reader.
    182    ///
    183    /// # Examples
    184    ///
    185    /// ```no_run
    186    /// use std::io::BufReader;
    187    /// use std::fs::File;
    188    ///
    189    /// fn main() -> std::io::Result<()> {
    190    ///     let f1 = File::open("log.txt")?;
    191    ///     let mut reader = BufReader::new(f1);
    192    ///
    193    ///     let f2 = reader.get_mut();
    194    ///     Ok(())
    195    /// }
    196    /// ```
    197    #[stable(feature = "rust1", since = "1.0.0")]
    198    pub fn get_mut(&mut self) -> &mut R {
    199        &mut self.inner
    200    }
    201
    202    /// Returns a reference to the internally buffered data.
    203    ///
    204    /// Unlike [`fill_buf`], this will not attempt to fill the buffer if it is empty.
    205    ///
    206    /// [`fill_buf`]: BufRead::fill_buf
    207    ///
    208    /// # Examples
    209    ///
    210    /// ```no_run
    211    /// use std::io::{BufReader, BufRead};
    212    /// use std::fs::File;
    213    ///
    214    /// fn main() -> std::io::Result<()> {
    215    ///     let f = File::open("log.txt")?;
    216    ///     let mut reader = BufReader::new(f);
    217    ///     assert!(reader.buffer().is_empty());
    218    ///
    219    ///     if reader.fill_buf()?.len() > 0 {
    220    ///         assert!(!reader.buffer().is_empty());
    221    ///     }
    222    ///     Ok(())
    223    /// }
    224    /// ```
    225    #[stable(feature = "bufreader_buffer", since = "1.37.0")]
    226    pub fn buffer(&self) -> &[u8] {
    227        self.buf.buffer()
    228    }
    229
    230    /// Returns the number of bytes the internal buffer can hold at once.
    231    ///
    232    /// # Examples
    233    ///
    234    /// ```no_run
    235    /// use std::io::{BufReader, BufRead};
    236    /// use std::fs::File;
    237    ///
    238    /// fn main() -> std::io::Result<()> {
    239    ///     let f = File::open("log.txt")?;
    240    ///     let mut reader = BufReader::new(f);
    241    ///
    242    ///     let capacity = reader.capacity();
    243    ///     let buffer = reader.fill_buf()?;
    244    ///     assert!(buffer.len() <= capacity);
    245    ///     Ok(())
    246    /// }
    247    /// ```
    248    #[stable(feature = "buffered_io_capacity", since = "1.46.0")]
    249    pub fn capacity(&self) -> usize {
    250        self.buf.capacity()
    251    }
    252
    253    /// Unwraps this `BufReader<R>`, returning the underlying reader.
    254    ///
    255    /// Note that any leftover data in the internal buffer is lost. Therefore,
    256    /// a following read from the underlying reader may lead to data loss.
    257    ///
    258    /// # Examples
    259    ///
    260    /// ```no_run
    261    /// use std::io::BufReader;
    262    /// use std::fs::File;
    263    ///
    264    /// fn main() -> std::io::Result<()> {
    265    ///     let f1 = File::open("log.txt")?;
    266    ///     let reader = BufReader::new(f1);
    267    ///
    268    ///     let f2 = reader.into_inner();
    269    ///     Ok(())
    270    /// }
    271    /// ```
    272    #[stable(feature = "rust1", since = "1.0.0")]
    273    pub fn into_inner(self) -> R
    274    where
    275        R: Sized,
    276    {
    277        self.inner
    278    }
    279
    280    /// Invalidates all data in the internal buffer.
    281    #[inline]
    282    pub(in crate::io) fn discard_buffer(&mut self) {
    283        self.buf.discard_buffer()
    284    }
    285}
    286
    287// This is only used by a test which asserts that the initialization-tracking is correct.
    288#[cfg(test)]
    289impl<R: ?Sized> BufReader<R> {
    290    #[allow(missing_docs)]
    291    pub fn initialized(&self) -> usize {
    292        self.buf.initialized()
    293    }
    294}
    295
    296impl<R: ?Sized + Seek> BufReader<R> {
    297    /// Seeks relative to the current position. If the new position lies within the buffer,
    298    /// the buffer will not be flushed, allowing for more efficient seeks.
    299    /// This method does not return the location of the underlying reader, so the caller
    300    /// must track this information themselves if it is required.
    301    #[stable(feature = "bufreader_seek_relative", since = "1.53.0")]
    302    pub fn seek_relative(&mut self, offset: i64) -> io::Result<()> {
    303        let pos = self.buf.pos() as u64;
    304        if offset < 0 {
    305            if let Some(_) = pos.checked_sub((-offset) as u64) {
    306                self.buf.unconsume((-offset) as usize);
    307                return Ok(());
    308            }
    309        } else if let Some(new_pos) = pos.checked_add(offset as u64) {
    310            if new_pos <= self.buf.filled() as u64 {
    311                self.buf.consume(offset as usize);
    312                return Ok(());
    313            }
    314        }
    315
    316        self.seek(SeekFrom::Current(offset)).map(drop)
    317    }
    318}
    319
    320impl<R> SpecReadByte for BufReader<R>
    321where
    322    Self: Read,
    323{
    324    #[inline]
    325    fn spec_read_byte(&mut self) -> Option<io::Result<u8>> {
    326        let mut byte = 0;
    327        if self.buf.consume_with(1, |claimed| byte = claimed[0]) {
    328            return Some(Ok(byte));
    329        }
    330
    331        // Fallback case, only reached once per buffer refill.
    332        uninlined_slow_read_byte(self)
    333    }
    334}
    335
    336#[stable(feature = "rust1", since = "1.0.0")]
    337impl<R: ?Sized + Read> Read for BufReader<R> {
    338    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
    339        // If we don't have any buffered data and we're doing a massive read
    340        // (larger than our internal buffer), bypass our internal buffer
    341        // entirely.
    342        if self.buf.pos() == self.buf.filled() && buf.len() >= self.capacity() {
    343            self.discard_buffer();
    344            return self.inner.read(buf);
    345        }
    346        let mut rem = self.fill_buf()?;
    347        let nread = rem.read(buf)?;
    348        self.consume(nread);
    349        Ok(nread)
    350    }
    351
    352    fn read_buf(&mut self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> {
    353        // If we don't have any buffered data and we're doing a massive read
    354        // (larger than our internal buffer), bypass our internal buffer
    355        // entirely.
    356        if self.buf.pos() == self.buf.filled() && cursor.capacity() >= self.capacity() {
    357            self.discard_buffer();
    358            return self.inner.read_buf(cursor);
    359        }
    360
    361        let prev = cursor.written();
    362
    363        let mut rem = self.fill_buf()?;
    364        rem.read_buf(cursor.reborrow())?; // actually never fails
    365
    366        self.consume(cursor.written() - prev); //slice impl of read_buf known to never unfill buf
    367
    368        Ok(())
    369    }
    370
    371    // Small read_exacts from a BufReader are extremely common when used with a deserializer.
    372    // The default implementation calls read in a loop, which results in surprisingly poor code
    373    // generation for the common path where the buffer has enough bytes to fill the passed-in
    374    // buffer.
    375    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
    376        if self.buf.consume_with(buf.len(), |claimed| buf.copy_from_slice(claimed)) {
    377            return Ok(());
    378        }
    379
    380        crate::io::default_read_exact(self, buf)
    381    }
    382
    383    fn read_buf_exact(&mut self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> {
    384        if self.buf.consume_with(cursor.capacity(), |claimed| cursor.append(claimed)) {
    385            return Ok(());
    386        }
    387
    388        crate::io::default_read_buf_exact(self, cursor)
    389    }
    390
    391    fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result<usize> {
    392        let total_len = bufs.iter().map(|b| b.len()).sum::<usize>();
    393        if self.buf.pos() == self.buf.filled() && total_len >= self.capacity() {
    394            self.discard_buffer();
    395            return self.inner.read_vectored(bufs);
    396        }
    397        let mut rem = self.fill_buf()?;
    398        let nread = rem.read_vectored(bufs)?;
    399
    400        self.consume(nread);
    401        Ok(nread)
    402    }
    403
    404    fn is_read_vectored(&self) -> bool {
    405        self.inner.is_read_vectored()
    406    }
    407
    408    // The inner reader might have an optimized `read_to_end`. Drain our buffer and then
    409    // delegate to the inner implementation.
    410    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
    411        let inner_buf = self.buffer();
    412        buf.try_reserve(inner_buf.len())?;
    413        buf.extend_from_slice(inner_buf);
    414        let nread = inner_buf.len();
    415        self.discard_buffer();
    416        Ok(nread + self.inner.read_to_end(buf)?)
    417    }
    418
    419    // The inner reader might have an optimized `read_to_end`. Drain our buffer and then
    420    // delegate to the inner implementation.
    421    fn read_to_string(&mut self, buf: &mut String) -> io::Result<usize> {
    422        // In the general `else` case below we must read bytes into a side buffer, check
    423        // that they are valid UTF-8, and then append them to `buf`. This requires a
    424        // potentially large memcpy.
    425        //
    426        // If `buf` is empty--the most common case--we can leverage `append_to_string`
    427        // to read directly into `buf`'s internal byte buffer, saving an allocation and
    428        // a memcpy.
    429        if buf.is_empty() {
    430            // `append_to_string`'s safety relies on the buffer only being appended to since
    431            // it only checks the UTF-8 validity of new data. If there were existing content in
    432            // `buf` then an untrustworthy reader (i.e. `self.inner`) could not only append
    433            // bytes but also modify existing bytes and render them invalid. On the other hand,
    434            // if `buf` is empty then by definition any writes must be appends and
    435            // `append_to_string` will validate all of the new bytes.
    436            unsafe { crate::io::append_to_string(buf, |b| self.read_to_end(b)) }
    437        } else {
    438            // We cannot append our byte buffer directly onto the `buf` String as there could
    439            // be an incomplete UTF-8 sequence that has only been partially read. We must read
    440            // everything into a side buffer first and then call `from_utf8` on the complete
    441            // buffer.
    442            let mut bytes = Vec::new();
    443            self.read_to_end(&mut bytes)?;
    444            let string = crate::str::from_utf8(&bytes).map_err(|_| io::Error::INVALID_UTF8)?;
    445            *buf += string;
    446            Ok(string.len())
    447        }
    448    }
    449}
    450
    451#[stable(feature = "rust1", since = "1.0.0")]
    452impl<R: ?Sized + Read> BufRead for BufReader<R> {
    453    fn fill_buf(&mut self) -> io::Result<&[u8]> {
    454        self.buf.fill_buf(&mut self.inner)
    455    }
    456
    457    fn consume(&mut self, amt: usize) {
    458        self.buf.consume(amt)
    459    }
    460}
    461
    462#[stable(feature = "rust1", since = "1.0.0")]
    463impl<R> fmt::Debug for BufReader<R>
    464where
    465    R: ?Sized + fmt::Debug,
    466{
    467    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
    468        fmt.debug_struct("BufReader")
    469            .field("reader", &&self.inner)
    470            .field(
    471                "buffer",
    472                &format_args!("{}/{}", self.buf.filled() - self.buf.pos(), self.capacity()),
    473            )
    474            .finish()
    475    }
    476}
    477
    478#[stable(feature = "rust1", since = "1.0.0")]
    479impl<R: ?Sized + Seek> Seek for BufReader<R> {
    480    /// Seek to an offset, in bytes, in the underlying reader.
    481    ///
    482    /// The position used for seeking with <code>[SeekFrom::Current]\(_)</code> is the
    483    /// position the underlying reader would be at if the `BufReader<R>` had no
    484    /// internal buffer.
    485    ///
    486    /// Seeking always discards the internal buffer, even if the seek position
    487    /// would otherwise fall within it. This guarantees that calling
    488    /// [`BufReader::into_inner()`] immediately after a seek yields the underlying reader
    489    /// at the same position.
    490    ///
    491    /// To seek without discarding the internal buffer, use [`BufReader::seek_relative`].
    492    ///
    493    /// See [`std::io::Seek`] for more details.
    494    ///
    495    /// Note: In the edge case where you're seeking with <code>[SeekFrom::Current]\(n)</code>
    496    /// where `n` minus the internal buffer length overflows an `i64`, two
    497    /// seeks will be performed instead of one. If the second seek returns
    498    /// [`Err`], the underlying reader will be left at the same position it would
    499    /// have if you called `seek` with <code>[SeekFrom::Current]\(0)</code>.
    500    ///
    501    /// [`std::io::Seek`]: Seek
    502    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
    503        let result: u64;
    504        if let SeekFrom::Current(n) = pos {
    505            let remainder = (self.buf.filled() - self.buf.pos()) as i64;
    506            // it should be safe to assume that remainder fits within an i64 as the alternative
    507            // means we managed to allocate 8 exbibytes and that's absurd.
    508            // But it's not out of the realm of possibility for some weird underlying reader to
    509            // support seeking by i64::MIN so we need to handle underflow when subtracting
    510            // remainder.
    511            if let Some(offset) = n.checked_sub(remainder) {
    512                result = self.inner.seek(SeekFrom::Current(offset))?;
    513            } else {
    514                // seek backwards by our remainder, and then by the offset
    515                self.inner.seek(SeekFrom::Current(-remainder))?;
    516                self.discard_buffer();
    517                result = self.inner.seek(SeekFrom::Current(n))?;
    518            }
    519        } else {
    520            // Seeking with Start/End doesn't care about our buffer length.
    521            result = self.inner.seek(pos)?;
    522        }
    523        self.discard_buffer();
    524        Ok(result)
    525    }
    526
    527    /// Returns the current seek position from the start of the stream.
    528    ///
    529    /// The value returned is equivalent to `self.seek(SeekFrom::Current(0))`
    530    /// but does not flush the internal buffer. Due to this optimization the
    531    /// function does not guarantee that calling `.into_inner()` immediately
    532    /// afterwards will yield the underlying reader at the same position. Use
    533    /// [`BufReader::seek`] instead if you require that guarantee.
    534    ///
    535    /// # Panics
    536    ///
    537    /// This function will panic if the position of the inner reader is smaller
    538    /// than the amount of buffered data. That can happen if the inner reader
    539    /// has an incorrect implementation of [`Seek::stream_position`], or if the
    540    /// position has gone out of sync due to calling [`Seek::seek`] directly on
    541    /// the underlying reader.
    542    ///
    543    /// # Example
    544    ///
    545    /// ```no_run
    546    /// use std::{
    547    ///     io::{self, BufRead, BufReader, Seek},
    548    ///     fs::File,
    549    /// };
    550    ///
    551    /// fn main() -> io::Result<()> {
    552    ///     let mut f = BufReader::new(File::open("foo.txt")?);
    553    ///
    554    ///     let before = f.stream_position()?;
    555    ///     f.read_line(&mut String::new())?;
    556    ///     let after = f.stream_position()?;
    557    ///
    558    ///     println!("The first line was {} bytes long", after - before);
    559    ///     Ok(())
    560    /// }
    561    /// ```
    562    fn stream_position(&mut self) -> io::Result<u64> {
    563        let remainder = (self.buf.filled() - self.buf.pos()) as u64;
    564        self.inner.stream_position().map(|pos| {
    565            pos.checked_sub(remainder).expect(
    566                "overflow when subtracting remaining buffer size from inner stream position",
    567            )
    568        })
    569    }
    570
    571    /// Seeks relative to the current position.
    572    ///
    573    /// If the new position lies within the buffer, the buffer will not be
    574    /// flushed, allowing for more efficient seeks. This method does not return
    575    /// the location of the underlying reader, so the caller must track this
    576    /// information themselves if it is required.
    577    fn seek_relative(&mut self, offset: i64) -> io::Result<()> {
    578        self.seek_relative(offset)
    579    }
    580}
    581
    582impl<T: ?Sized> SizeHint for BufReader<T> {
    583    #[inline]
    584    fn lower_bound(&self) -> usize {
    585        SizeHint::lower_bound(self.get_ref()) + self.buffer().len()
    586    }
    587
    588    #[inline]
    589    fn upper_bound(&self) -> Option<usize> {
    590        SizeHint::upper_bound(self.get_ref()).and_then(|up| self.buffer().len().checked_add(up))
    591    }
    592}