diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 89a91cd..cfb56db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,8 @@ jobs: image: clickhouse/clickhouse-server ports: - 9000:9000 + env: + CLICKHOUSE_SKIP_USER_SETUP: 1 steps: - uses: actions/checkout@v3 - name: Build @@ -27,57 +29,47 @@ jobs: - name: Run tests run: cargo test --verbose - build-native-tls: + build-tls: + strategy: + fail-fast: false + matrix: + feature: + - tls-native-tls + - tls-rustls + database_url: + # for TLS we need skip_verify for self-signed certificate + - tcp://localhost:9440?skip_verify=true + # we don't need skip_verify when we pass CA cert + - tcp://localhost:9440?ca_certificate=tls/ca.pem + # mTLS + - tcp://tls@localhost:9440?ca_certificate=tls/ca.pem&client_certificate=tls/client.crt&client_private_key=tls/client.key runs-on: ubuntu-latest env: # NOTE: not all tests "secure" aware, so let's define DATABASE_URL explicitly # NOTE: sometimes for native-tls default connection_timeout (500ms) is not enough, interestingly that for rustls it is OK. - DATABASE_URL: "tcp://localhost:9440?compression=lz4&ping_timeout=2s&retry_timeout=3s&secure=true&skip_verify=true&connection_timeout=5s" + DATABASE_URL: ${{ matrix.database_url }}&compression=lz4&ping_timeout=2s&retry_timeout=3s&secure=true&connection_timeout=5s steps: - uses: actions/checkout@v3 + - name: Generate TLS certificates + run: | + extras/ci/generate_certs.sh tls # NOTE: # - we cannot use "services" because they are executed before the steps, i.e. repository checkout. # - "job.container.network" is empty, hence "host" # - github actions does not support YAML anchors (sigh) - name: Run clickhouse-server run: docker run - -v ./extras/ci/generate_certs.sh:/docker-entrypoint-initdb.d/generate_certs.sh -v ./extras/ci/overrides.xml:/etc/clickhouse-server/config.d/overrides.xml - -e CH_SSL_CERTIFICATE=/etc/clickhouse-server/config.d/server.crt - -e CH_SSL_PRIVATE_KEY=/etc/clickhouse-server/config.d/server.key + -v ./extras/ci/users-overrides.yaml:/etc/clickhouse-server/users.d/overrides.yaml + -v ./tls:/etc/clickhouse-server/tls + -e CLICKHOUSE_SKIP_USER_SETUP=1 --network host + --name clickhouse --rm --detach --publish 9440:9440 clickhouse/clickhouse-server - name: Build - run: cargo build --features tls-native-tls --verbose + run: cargo build --features ${{ matrix.feature }} --verbose - name: Run tests - run: cargo test --features tls-native-tls --verbose - - build-rustls: - runs-on: ubuntu-latest - env: - # NOTE: not all tests "secure" aware, so let's define DATABASE_URL explicitly - DATABASE_URL: "tcp://localhost:9440?compression=lz4&ping_timeout=2s&retry_timeout=3s&secure=true&skip_verify=true" - steps: - - uses: actions/checkout@v3 - # NOTE: - # - we cannot use "services" because they are executed before the steps, i.e. repository checkout. - # - "job.container.network" is empty, hence "host" - # - github actions does not support YAML anchors (sigh) - - name: Run clickhouse-server - run: docker run - -v ./extras/ci/generate_certs.sh:/docker-entrypoint-initdb.d/generate_certs.sh - -v ./extras/ci/overrides.xml:/etc/clickhouse-server/config.d/overrides.xml - -e CH_SSL_CERTIFICATE=/etc/clickhouse-server/config.d/server.crt - -e CH_SSL_PRIVATE_KEY=/etc/clickhouse-server/config.d/server.key - --network host - --rm - --detach - --publish 9440:9440 - clickhouse/clickhouse-server - - name: Build - run: cargo build --features tls-rustls --verbose - - name: Run tests - run: cargo test --features tls-rustls --verbose + run: cargo test --features ${{ matrix.feature }} --verbose diff --git a/README.md b/README.md index eedb4e4..badbdfb 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,13 @@ for the most common use cases. The following features are available. - `tokio_io` *(enabled by default)* — I/O based on [Tokio](https://tokio.rs/). - `async_std` — I/O based on [async-std](https://async.rs/) (doesn't work together with `tokio_io`). -- `tls` — TLS support (allowed only with `tokio_io`). +- `tls` — TLS support (allowed only with `tokio_io` and one of TLS libraries, under `tls-rustls` or `tls-native-tls` features). + +### TLS + +- `skip_verify` - do not verify the server certificate (**insecure**) +- `ca_certificate` - instead of `skip_verify` it is better to pass CA certificate explicitly (in case of self-signed certificates). +- `client_certificate`/`client_private_key` - authentication using TLS certificates (mTLS) (see [ClickHouse documentation](https://clickhouse.com/docs/operations/external-authenticators/ssl-x509) for more info) ## Example diff --git a/extras/ci/generate_certs.sh b/extras/ci/generate_certs.sh index 3e2caa1..ed41da3 100755 --- a/extras/ci/generate_certs.sh +++ b/extras/ci/generate_certs.sh @@ -1,7 +1,52 @@ #!/usr/bin/env bash -crt=$CH_SSL_CERTIFICATE -key=$CH_SSL_PRIVATE_KEY +out=$1 && shift +mkdir -p "$out" +cd "$out" -openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout "$key" -out "$crt" -chown clickhouse:clickhouse "$crt" "$key" +# +# CA +# +openssl genrsa -out ca.key 4096 +openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 -out ca.pem -subj "/C=US/ST=DevState/O=DevOrg/CN=MyDevCA" + +# +# server +# +openssl genrsa -out server.key 2048 +openssl req -new -key server.key -out server.csr -subj "/C=US/ST=DevState/O=DevOrg/CN=localhost" + +cat > server.ext < client.ext < - - - none + /etc/clickhouse-server/tls/server.crt + /etc/clickhouse-server/tls/server.key + /etc/clickhouse-server/tls/ca.pem + relaxed true true sslv2,sslv3 diff --git a/extras/ci/users-overrides.yaml b/extras/ci/users-overrides.yaml new file mode 100644 index 0000000..16f3e4d --- /dev/null +++ b/extras/ci/users-overrides.yaml @@ -0,0 +1,6 @@ +--- +users: + tls: + ssl_certificates: + subject_alt_name: + - DNS:localhost diff --git a/src/connecting_stream.rs b/src/connecting_stream.rs index 05b69e5..de06b0b 100644 --- a/src/connecting_stream.rs +++ b/src/connecting_stream.rs @@ -35,6 +35,8 @@ use crate::{errors::ConnectionError, io::Stream as InnerStream, Options}; use tokio_native_tls::TlsStream; #[cfg(feature = "tls-rustls")] use tokio_rustls::client::TlsStream; +#[cfg(feature = "_tls")] +use crate::types::ClientTlsIdentity; type Result = std::result::Result; @@ -104,6 +106,11 @@ impl State { State::Tcp(TcpState::Fail(Some(conn_error))) } + #[cfg(feature = "tls-rustls")] + fn tls_err(e: TlsError) -> Self { + State::Tls(TlsState::Fail(Some(ConnectionError::TlsError(e)))) + } + #[cfg(feature = "_tls")] fn tls_host_err() -> Self { State::Tls(TlsState::Fail(Some(ConnectionError::TlsHostNotProvided))) @@ -125,6 +132,7 @@ pub(crate) struct ConnectingStream { state: State, } +#[cfg(feature = "tls-rustls")] #[derive(Debug)] struct DummyTlsVerifier; @@ -231,10 +239,14 @@ impl ConnectingStream { Some(host) => { let mut builder = TlsConnector::builder(); builder.danger_accept_invalid_certs(options.skip_verify); - if let Some(certificate) = options.certificate.clone() { + if let Some(certificate) = options.ca_certificate.clone() { let native_cert = native_tls::Certificate::from(certificate); builder.add_root_certificate(native_cert); } + if let Some(identity) = &options.client_tls_identity { + let ClientTlsIdentity::Pkcs(pkcs) = identity; + builder.identity(pkcs.clone()); + } Self { state: State::tls_wait(Box::pin(async move { @@ -261,11 +273,10 @@ impl ConnectingStream { state: State::tls_host_err(), }, Some(host) => { - let config = if options.skip_verify { + let builder = if options.skip_verify { ClientConfig::builder() .dangerous() .with_custom_certificate_verifier(Arc::new(DummyTlsVerifier)) - .with_no_client_auth() } else { let mut cert_store = RootCertStore::empty(); cert_store.extend( @@ -273,7 +284,7 @@ impl ConnectingStream { .iter() .cloned() ); - if let Some(certificates) = options.certificate.clone() { + if let Some(certificates) = options.ca_certificate.clone() { for certificate in Into::>>::into( certificates, @@ -293,7 +304,18 @@ impl ConnectingStream { } ClientConfig::builder() .with_root_certificates(cert_store) - .with_no_client_auth() + }; + let config = if let Some(identity) = &options.client_tls_identity { + let ClientTlsIdentity::Pem { key, certs } = identity; + builder.with_client_auth_cert(certs.clone().into(), key.clone_key()) + } else { + Ok(builder.with_no_client_auth()) + }; + let config = match config { + Ok(config) => config, + Err(err) => { + return Self { state: State::tls_err(err) }; + }, }; Self { state: State::tls_wait(Box::pin(async move { diff --git a/src/io/transport.rs b/src/io/transport.rs index 7ecfdfd..e4eff04 100644 --- a/src/io/transport.rs +++ b/src/io/transport.rs @@ -287,6 +287,15 @@ impl Stream for ClickhouseTransport { } if *this.done { + // We still have may have something in buffer, since the client may read something + // first, and only after the server will close the connection, likely in case of + // exception, let's try to parse it here + if !this.rd.is_empty() { + if let Poll::Ready(ret) = this.try_parse_msg()? { + return Poll::Ready(ret.map(Ok)); + } + } + return Poll::Ready(None); } diff --git a/src/lib.rs b/src/lib.rs index 1ecd896..385fea5 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -328,7 +328,7 @@ impl ClientHandle { } self.inner = h; - self.context.server_info = info.unwrap(); + self.context.server_info = info.ok_or(Error::Other("Missing Hello/Exception packet".into()))?; Ok(()) } diff --git a/src/types/mod.rs b/src/types/mod.rs index 83ff796..aaba44c 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -21,6 +21,8 @@ pub use self::{ value::Value, value_ref::ValueRef, }; +#[cfg(feature = "_tls")] +pub use self::options::ClientTlsIdentity; pub(crate) use self::{ cmd::Cmd, diff --git a/src/types/options.rs b/src/types/options.rs index dd3d810..94979c3 100644 --- a/src/types/options.rs +++ b/src/types/options.rs @@ -6,11 +6,16 @@ use std::{ sync::{Arc, Mutex}, time::Duration, }; +#[cfg(feature = "_tls")] +use std::fs; use crate::errors::{Error, Result, UrlError}; use percent_encoding::percent_decode; use url::Url; +#[cfg(feature = "tls-rustls")] +use rustls::pki_types::pem::PemObject; + const DEFAULT_MIN_CONNS: usize = 10; const DEFAULT_MAX_CONNS: usize = 20; @@ -107,8 +112,8 @@ impl Certificate { } /// Parses a PEM-formatted X509 certificate. - pub fn from_pem(der: &[u8]) -> Result { - let inner = match native_tls::Certificate::from_pem(der) { + pub fn from_pem(pem: &[u8]) -> Result { + let inner = match native_tls::Certificate::from_pem(pem) { Ok(certificate) => certificate, Err(err) => return Err(Error::Other(err.to_string().into())), }; @@ -139,8 +144,8 @@ impl Certificate { } /// Parses a PEM-formatted X509 certificate. - pub fn from_pem(der: &[u8]) -> Result { - let certs = rustls_pemfile::certs(&mut der.as_ref()) + pub fn from_pem(pem: &[u8]) -> Result { + let certs = rustls_pemfile::certs(&mut pem.as_ref()) .map(|result| result.unwrap()) .collect(); Ok(Certificate(Arc::new(certs))) @@ -166,6 +171,62 @@ impl PartialEq for Certificate { } } +#[cfg(feature = "_tls")] +pub fn load_certificate(file: &str) -> Result { + let data = fs::read(file)?; + if file.ends_with(".der") || file.ends_with(".cer") { + Certificate::from_der(&data) + } else { + Certificate::from_pem(&data) + } +} + +#[cfg(feature = "_tls")] +#[derive(Clone)] +pub enum ClientTlsIdentity { + #[cfg(feature = "tls-rustls")] + Pem { + key: Arc>, + certs: Certificate, + }, + #[cfg(feature = "tls-native-tls")] + Pkcs(native_tls::Identity), +} + +#[cfg(feature = "_tls")] +impl ClientTlsIdentity { + #[cfg(feature = "tls-rustls")] + pub fn load(cert_path: &str, key_path: &str) -> Result { + let key = rustls::pki_types::PrivateKeyDer::from_pem_slice(fs::read(key_path)?.as_ref()) + .map_err(|e| format!("Cannot read private key from {}: {}", key_path, e))?; + let key = Arc::new(key); + let certs = load_certificate(cert_path)?; + return Ok(Self::Pem{ key, certs }); + } + + #[cfg(feature = "tls-native-tls")] + pub fn load(cert_path: &str, key_path: &str) -> Result { + let identity = native_tls::Identity::from_pkcs8( + fs::read(cert_path)?.as_ref(), + fs::read(key_path)?.as_ref(), + ).map_err(|e| format!("Cannot load identity from {} and {}: {}", cert_path, key_path, e))?; + return Ok(Self::Pkcs(identity)); + } +} + +#[cfg(feature = "_tls")] +impl fmt::Debug for ClientTlsIdentity { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "[Client Certificate]") + } +} +#[cfg(feature = "_tls")] +impl PartialEq for ClientTlsIdentity { + fn eq(&self, _other: &Self) -> bool { + true + } +} + #[derive(Clone, PartialEq, Debug)] pub enum SettingType { String(String), @@ -287,9 +348,13 @@ pub struct Options { #[cfg(feature = "_tls")] pub(crate) skip_verify: bool, - /// An X509 certificate. + /// CA certificate. #[cfg(feature = "_tls")] - pub(crate) certificate: Option, + pub(crate) ca_certificate: Option, + + /// Authorization with certificate (mTLS). + #[cfg(feature = "_tls")] + pub(crate) client_tls_identity: Option, /// Query settings pub(crate) settings: HashMap, @@ -300,7 +365,8 @@ pub struct Options { impl fmt::Debug for Options { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("Options") + let mut debug = f.debug_struct("Options"); + let res = debug .field("addr", &self.addr) .field("database", &self.database) .field("compression", &self.compression) @@ -314,8 +380,15 @@ impl fmt::Debug for Options { .field("ping_timeout", &self.ping_timeout) .field("connection_timeout", &self.connection_timeout) .field("settings", &self.settings) - .field("alt_hosts", &self.alt_hosts) - .finish() + .field("alt_hosts", &self.alt_hosts); + + #[cfg(feature = "_tls")] + res + .field("secure", &self.secure) + .field("ca_certificate", &self.ca_certificate) + .field("client_tls_identity", &self.client_tls_identity); + + res.finish() } } @@ -344,7 +417,9 @@ impl Default for Options { #[cfg(feature = "_tls")] skip_verify: false, #[cfg(feature = "_tls")] - certificate: None, + ca_certificate: None, + #[cfg(feature = "_tls")] + client_tls_identity: None, settings: HashMap::new(), alt_hosts: Vec::new(), } @@ -495,8 +570,14 @@ impl Options { #[cfg(feature = "_tls")] property! { - /// An X509 certificate. - => certificate: Option + /// CA certificate. + => ca_certificate: Option + } + + #[cfg(feature = "_tls")] + property! { + /// Authorization with certificate (mTLS). + => client_tls_identity: Option } property! { @@ -563,6 +644,11 @@ fn set_params<'a, I>(options: &mut Options, iter: I) -> std::result::Result<(), where I: Iterator, Cow<'a, str>)>, { + #[cfg(feature = "_tls")] + let mut client_certificate = None; + #[cfg(feature = "_tls")] + let mut client_private_key = None; + for (key, value) in iter { match key.as_ref() { "pool_min" => options.pool_min = parse_param(key, value, usize::from_str)?, @@ -590,6 +676,12 @@ where "secure" => options.secure = parse_param(key, value, bool::from_str)?, #[cfg(feature = "_tls")] "skip_verify" => options.skip_verify = parse_param(key, value, bool::from_str)?, + #[cfg(feature = "_tls")] + "ca_certificate" => options.ca_certificate = Some(parse_param(key, value, load_certificate)?), + #[cfg(feature = "_tls")] + "client_certificate" => client_certificate = Some(value), + #[cfg(feature = "_tls")] + "client_private_key" => client_private_key = Some(value), "alt_hosts" => options.alt_hosts = parse_param(key, value, parse_hosts)?, _ => { let value = SettingType::String(value.to_string()); @@ -604,6 +696,17 @@ where }; } + #[cfg(feature = "_tls")] + match (client_certificate, client_private_key) { + (Some(cert), Some(key)) => { + options.client_tls_identity = Some(ClientTlsIdentity::load(&cert, &key).map_err(|_| UrlError::Invalid)?); + } + (None, None) => {} + _ => { + return Err(UrlError::Invalid); + }, + } + Ok(()) } diff --git a/src/types/value.rs b/src/types/value.rs index 9b38b41..e86ef9c 100644 --- a/src/types/value.rs +++ b/src/types/value.rs @@ -806,7 +806,7 @@ mod test { #[test] fn test_size_of() { use std::mem; - assert_eq!(56, mem::size_of::<[Value; 1]>()); + assert_eq!(64, mem::size_of::<[Value; 1]>()); } #[test]