temp

Author: dralley

src/encoding.rs

@@ -1,9 +1,10 @@
 //! A module for wrappers that encode / decode data.
 
 use std::borrow::Cow;
+use std::io::{self, BufRead, Read};
 
 #[cfg(feature = "encoding")]
-use encoding_rs::{Encoding, UTF_16BE, UTF_16LE, UTF_8};
+use encoding_rs::{Decoder as ExtDecoder, Encoding, UTF_16BE, UTF_16LE, UTF_8, CoderResult};
 
 use crate::{Error, Result};
 
@@ -184,4 +185,166 @@ pub fn detect_encoding(bytes: &[u8]) -> Option<&'static Encoding> {
     }
 }
 
+/// A reference to an encoding together with information about how it was retrieved.
+///
+/// The state transition diagram:
+///
+/// ```mermaid
+/// flowchart LR
+///   Implicit    -- from_str       --> Explicit
+///   Implicit    -- BOM            --> BomDetected
+///   Implicit    -- "encoding=..." --> XmlDetected
+///   BomDetected -- "encoding=..." --> XmlDetected
+/// ```
+#[cfg(feature = "encoding")]
+#[derive(Clone, Copy)]
+pub(crate) enum EncodingRef {
+    /// Encoding was implicitly assumed to have a specified value. It can be refined
+    /// using BOM or by the XML declaration event (`<?xml encoding=... ?>`)
+    Implicit(&'static Encoding),
+    /// Encoding was explicitly set to the desired value. It cannot be changed
+    /// nor by BOM, nor by parsing XML declaration (`<?xml encoding=... ?>`)
+    Explicit(&'static Encoding),
+    /// Encoding was detected from a byte order mark (BOM) or by the first bytes
+    /// of the content. It can be refined by the XML declaration event (`<?xml encoding=... ?>`)
+    BomDetected(&'static Encoding),
+    /// Encoding was detected using XML declaration event (`<?xml encoding=... ?>`).
+    /// It can no longer change
+    XmlDetected(&'static Encoding),
+}
+#[cfg(feature = "encoding")]
+impl EncodingRef {
+    #[inline]
+    pub(crate) fn encoding(&self) -> &'static Encoding {
+        match self {
+            Self::Implicit(e) => e,
+            Self::Explicit(e) => e,
+            Self::BomDetected(e) => e,
+            Self::XmlDetected(e) => e,
+        }
+    }
+    #[inline]
+    pub(crate) fn can_be_refined(&self) -> bool {
+        match self {
+            Self::Implicit(_) | Self::BomDetected(_) => true,
+            Self::Explicit(_) | Self::XmlDetected(_) => false,
+        }
+    }
+}
+
+#[cfg(feature = "encoding")]
+
+struct DecodingBufReader<R> {
+    // // The buffer
+    // buffer: String,
+    // // How many bytes in the buffer currently hold significant data.
+    // current_position: usize,
+
+    // /// Track whether we see errors.
+    // encoding: Option<Encoding>,
+
+    inner: R,
+    decoded_buffer: Vec<u8>,
+    current_pos: usize,
+
+    decoder: ExtDecoder,
+    encoding: EncodingRef,
+}
+
+#[cfg(feature = "encoding")]
+impl<R: BufRead> BufRead for DecodingBufReader<R> {
+    fn fill_buf(&mut self) -> io::Result<&[u8]> {
+        self.shuffle();
+        let data = self.inner.fill_buf()?;
+
+        let amount_read_from_inner = self.feed(data)?;
+        self.inner.consume(amount_read_from_inner);
+
+        Ok(data)
+    }
+
+    fn consume(&mut self, amt: usize) {
+        self.current_pos = std::cmp::min(self.current_pos + amt, self.decoded_buffer.capacity());
+    }
+}
+
+
+#[cfg(feature = "encoding")]
+impl<R: Read> Read for DecodingBufReader<R> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.inner.read(buf)
+    }
+}
+
+#[cfg(feature = "encoding")]
+impl<R: BufRead> DecodingBufReader<R> {
+    fn new(inner: R) -> Self {
+        DecodingBufReader {
+            inner: inner,
+            decoded_buffer: Vec::new(),
+            current_pos: 0,
+
+            decoder: UTF_8.new_decoder(),
+            encoding: EncodingRef::Implicit(UTF_8),
+        }
+    }
+
+    fn get_raw_buffer(&mut self) -> io::Result<&[u8]> {
+        self.inner.fill_buf()
+    }
+
+    /// Move unconsumed data to the front of the buffer and reset the length
+    fn shuffle(&mut self) {
+        if self.current_pos == 0 {
+            return;
+        }
+
+        // Copy all unconsumed bytes to the beginning of the buffer
+        self.decoded_buffer.as_mut_slice().copy_within(self.current_pos.., 0);
+        // Truncate the buffer
+        self.decoded_buffer.truncate(self.decoded_buffer.len() - self.current_pos);
+        self.current_pos = 0;
+    }
+
+    /// Reallocate a smaller buffer with the provided size
+    fn shrink_buffer(&mut self, size: usize) {
+        self.shuffle();
+        self.decoded_buffer.shrink_to(size);
+    }
+
+    fn set_encoding(&mut self, encoding: &'static Encoding) {
+        self.encoding = EncodingRef::Explicit(encoding);
+    }
+
+    fn feed(&mut self, data: &[u8]) -> io::Result<usize> {
+        // reserve (at least) enough space in our buffer to hold the decoded data
+        // encoding::max_utf8_buffer_length(data.len())
+        self.decoded_buffer.reserve(data.len());
+
+        // The number of bytes already read from current `input` in total.
+        let (result, read, written, had_errors) =
+            self.decoder.decode_to_utf8(&data[..],
+                                &mut self.decoded_buffer[self.current_pos..],
+                                data.is_empty());
+        self.current_pos += written;
+        match result {
+            CoderResult::InputEmpty => {
+                // We have consumed the current input buffer.
+                match had_errors {
+                    true => Err(io::Error::new(io::ErrorKind::Other, "Errors decoding")),
+                    false => Ok(read),
+                }
+            },
+            CoderResult::OutputFull => unreachable!("This shouldn't happen, we reserved space"),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+}
+
+
+
 // TODO: add some tests for functions

src/reader/mod.rs

@@ -3,10 +3,10 @@
 use std::str::from_utf8;
 
 #[cfg(feature = "encoding")]
-use encoding_rs::{Encoding, UTF_8};
+use encoding_rs::UTF_8;
 
 #[cfg(feature = "encoding")]
-use crate::encoding::detect_encoding;
+use crate::encoding::{detect_encoding, EncodingRef};
 use crate::encoding::Decoder;
 use crate::errors::{Error, Result};
 use crate::events::{BytesCData, BytesDecl, BytesEnd, BytesStart, BytesText, Event};
@@ -179,53 +179,6 @@ enum TagState {
     Exit,
 }
 
-/// A reference to an encoding together with information about how it was retrieved.
-///
-/// The state transition diagram:
-///
-/// ```mermaid
-/// flowchart LR
-///   Implicit    -- from_str       --> Explicit
-///   Implicit    -- BOM            --> BomDetected
-///   Implicit    -- "encoding=..." --> XmlDetected
-///   BomDetected -- "encoding=..." --> XmlDetected
-/// ```
-#[cfg(feature = "encoding")]
-#[derive(Clone, Copy)]
-enum EncodingRef {
-    /// Encoding was implicitly assumed to have a specified value. It can be refined
-    /// using BOM or by the XML declaration event (`<?xml encoding=... ?>`)
-    Implicit(&'static Encoding),
-    /// Encoding was explicitly set to the desired value. It cannot be changed
-    /// nor by BOM, nor by parsing XML declaration (`<?xml encoding=... ?>`)
-    Explicit(&'static Encoding),
-    /// Encoding was detected from a byte order mark (BOM) or by the first bytes
-    /// of the content. It can be refined by the XML declaration event (`<?xml encoding=... ?>`)
-    BomDetected(&'static Encoding),
-    /// Encoding was detected using XML declaration event (`<?xml encoding=... ?>`).
-    /// It can no longer change
-    XmlDetected(&'static Encoding),
-}
-#[cfg(feature = "encoding")]
-impl EncodingRef {
-    #[inline]
-    fn encoding(&self) -> &'static Encoding {
-        match self {
-            Self::Implicit(e) => e,
-            Self::Explicit(e) => e,
-            Self::BomDetected(e) => e,
-            Self::XmlDetected(e) => e,
-        }
-    }
-    #[inline]
-    fn can_be_refined(&self) -> bool {
-        match self {
-            Self::Implicit(_) | Self::BomDetected(_) => true,
-            Self::Explicit(_) | Self::XmlDetected(_) => false,
-        }
-    }
-}
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /// A low level encoding-agnostic XML event reader.