diff --git a/Cargo.lock b/Cargo.lock index 53e7c36..c09305a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,6 +122,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chardet" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a48563284b67c003ba0fb7243c87fab68885e1532c605704228a80238512e31" + [[package]] name = "colored_json" version = "3.0.1" @@ -989,7 +995,9 @@ name = "reqsnaked" version = "0.1.0-beta0" dependencies = [ "bytes", + "chardet", "colored_json", + "encoding_rs", "futures", "http", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 9902145..231b4a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,9 @@ crate-type = ["cdylib"] [dependencies] bytes = "1.4.0" +chardet = "0.2" colored_json = "3.0.1" +encoding_rs = "0.8.32" futures = "0.3.26" http = "0.2.9" pyo3 = { version = "0.16.3", features = ["extension-module"] } diff --git a/reqsnaked.pyi b/reqsnaked.pyi index 868e4ee..e14fab5 100644 --- a/reqsnaked.pyi +++ b/reqsnaked.pyi @@ -14,7 +14,10 @@ QueryValue = typing.Union[ class Bytes: - def to_bytes(self) -> bytes: + def as_bytes(self) -> bytes: + pass + + def decode(self) -> str: pass diff --git a/src/rs2py/bytes.rs b/src/rs2py/bytes.rs index e560c2f..e0ba454 100644 --- a/src/rs2py/bytes.rs +++ b/src/rs2py/bytes.rs @@ -1,4 +1,6 @@ -use pyo3::{prelude::*, types::PyBytes}; +use chardet::detect; +use encoding_rs::{Encoding, UTF_8}; +use pyo3::{prelude::*, types::{PyBytes, PyString}}; #[derive(Clone)] @@ -10,6 +12,12 @@ impl Bytes { pub fn as_bytes<'rt>(&'rt self, py: Python<'rt>) -> &'rt PyBytes { PyBytes::new(py, self.0.as_ref()) } + pub fn decode<'rt>(&'rt self, py: Python<'rt>) -> &'rt PyString { + let encoding = detect(self.0.as_ref()).0; + + let encoding = Encoding::for_label(encoding.as_bytes()).unwrap_or(UTF_8); + PyString::new(py, &encoding.decode(self.0.as_ref()).0) + } } pub fn init_module(py: Python, parent_module: &PyModule, library: &PyModule) -> PyResult<()> {