diff --git a/docs/_snippets/_users-and-roles-common.md b/docs/_snippets/_users-and-roles-common.md index 29726229be9..964f9a8b40d 100644 --- a/docs/_snippets/_users-and-roles-common.md +++ b/docs/_snippets/_users-and-roles-common.md @@ -42,64 +42,73 @@ Create these tables and users to be used in the examples. #### Creating a sample database, table, and rows {#creating-a-sample-database-table-and-rows} -1. Create a test database + - ```sql - CREATE DATABASE db1; - ``` +##### Create a test database {#create-a-test-database} -2. Create a table +```sql +CREATE DATABASE db1; +``` - ```sql - CREATE TABLE db1.table1 ( - id UInt64, - column1 String, - column2 String - ) - ENGINE MergeTree - ORDER BY id; - ``` +##### Create a table {#create-a-table} -3. Populate the table with sample rows +```sql +CREATE TABLE db1.table1 ( + id UInt64, + column1 String, + column2 String +) +ENGINE MergeTree +ORDER BY id; +``` - ```sql - INSERT INTO db1.table1 - (id, column1, column2) - VALUES - (1, 'A', 'abc'), - (2, 'A', 'def'), - (3, 'B', 'abc'), - (4, 'B', 'def'); - ``` +##### Populate the table with sample rows {#populate} -4. Verify the table: +```sql +INSERT INTO db1.table1 + (id, column1, column2) +VALUES + (1, 'A', 'abc'), + (2, 'A', 'def'), + (3, 'B', 'abc'), + (4, 'B', 'def'); +``` - ```sql - SELECT * - FROM db1.table1 - ``` +##### Verify the table {#verify} - ```response - Query id: 475015cc-6f51-4b20-bda2-3c9c41404e49 +```sql title="Query" +SELECT * +FROM db1.table1 +``` - ┌─id─┬─column1─┬─column2─┐ - │ 1 │ A │ abc │ - │ 2 │ A │ def │ - │ 3 │ B │ abc │ - │ 4 │ B │ def │ - └────┴─────────┴─────────┘ - ``` +```response title="Response" +Query id: 475015cc-6f51-4b20-bda2-3c9c41404e49 -5. Create a regular user that will be used to demonstrate restrict access to certain columns: +┌─id─┬─column1─┬─column2─┐ +│ 1 │ A │ abc │ +│ 2 │ A │ def │ +│ 3 │ B │ abc │ +│ 4 │ B │ def │ +└────┴─────────┴─────────┘ +``` - ```sql - CREATE USER column_user IDENTIFIED BY 'password'; - ``` +##### Create `column_user` {#create-a-user-with-restricted-access-to-columns} -6. Create a regular user that will be used to demonstrate restricting access to rows with certain values: - ```sql - CREATE USER row_user IDENTIFIED BY 'password'; - ``` +Create a regular user that will be used to demonstrate restrict access to certain columns: + +```sql +CREATE USER column_user IDENTIFIED BY 'password'; +``` + +##### Create `row_user` {#create-a-user-with-restricted-access-to-rows-with-certain-values} + +Create a regular user that will be used to demonstrate restricting access to rows with certain values: + +```sql +CREATE USER row_user IDENTIFIED BY 'password'; +``` + + #### Creating roles {#creating-roles} diff --git a/docs/best-practices/_snippets/_table_of_contents.md b/docs/best-practices/_snippets/_table_of_contents.md new file mode 100644 index 00000000000..9e0d34ef2d1 --- /dev/null +++ b/docs/best-practices/_snippets/_table_of_contents.md @@ -0,0 +1,12 @@ +| Page | Description | +|--------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| [Choosing a Primary Key](/best-practices/choosing-a-primary-key) | How to select primary keys that maximize query performance and minimize storage overhead. | +| [Select Data Types](/best-practices/select-data-types) | Choose optimal data types to reduce memory usage, improve compression, and accelerate queries. | +| [Use Materialized Views](/best-practices/use-materialized-views) | Leverage materialized views to pre-aggregate data and dramatically speed up analytical queries. | +| [Minimize and Optimize JOINs](/best-practices/minimize-optimize-joins) | Best practices for using ClickHouse's `JOIN` capabilities efficiently. | +| [Choosing a Partitioning Key](/best-practices/choosing-a-partitioning-key) | Select partitioning strategies that enable efficient data pruning and faster query execution. | +| [Selecting an Insert Strategy](/best-practices/selecting-an-insert-strategy) | Optimize data ingestion throughput and reduce resource consumption with proper insert patterns. | +| [Data Skipping Indices](/best-practices/use-data-skipping-indices-where-appropriate) | Apply secondary indices strategically to skip irrelevant data blocks and accelerate filtered queries. | +| [Avoid Mutations](/best-practices/avoid-mutations) | Design schemas and workflows that eliminate costly `UPDATE`/`DELETE` operations for better performance. | +| [Avoid OPTIMIZE FINAL](/best-practices/avoid-optimize-final) | Prevent performance bottlenecks by understanding when `OPTIMIZE FINAL` hurts more than it helps. | +| [Use JSON where appropriate](/best-practices/use-json-where-appropriate) | Balance flexibility and performance when working with semi-structured JSON data in ClickHouse. | \ No newline at end of file diff --git a/docs/best-practices/index.md b/docs/best-practices/index.md index 5a3ae78ab5f..b4721106510 100644 --- a/docs/best-practices/index.md +++ b/docs/best-practices/index.md @@ -6,19 +6,10 @@ hide_title: true description: 'Landing page for Best Practices section in ClickHouse' --- +import TableOfContents from '@site/docs/best-practices/_snippets/_table_of_contents.md'; + # Best Practices in ClickHouse {#best-practices-in-clickhouse} This section provides the best practices you will want to follow to get the most out of ClickHouse. -| Page | Description | -|----------------------------------------------------------------------|--------------------------------------------------------------------------| -| [Choosing a Primary Key](/best-practices/choosing-a-primary-key) | Guidance on selecting an effective Primary Key in ClickHouse. | -| [Select Data Types](/best-practices/select-data-types) | Recommendations for choosing appropriate data types. | -| [Use Materialized Views](/best-practices/use-materialized-views) | When and how to benefit from materialized views. | -| [Minimize and Optimize JOINs](/best-practices/minimize-optimize-joins)| Best practices for minimizing and optimizing JOIN operations. | -| [Choosing a Partitioning Key](/best-practices/choosing-a-partitioning-key) | How to choose and apply partitioning keys effectively. | -| [Selecting an Insert Strategy](/best-practices/selecting-an-insert-strategy) | Strategies for efficient data insertion in ClickHouse. | -| [Data Skipping Indices](/best-practices/use-data-skipping-indices-where-appropriate) | When to apply data skipping indices for performance gains. | -| [Avoid Mutations](/best-practices/avoid-mutations) | Reasons to avoid mutations and how to design without them. | -| [Avoid OPTIMIZE FINAL](/best-practices/avoid-optimize-final) | Why `OPTIMIZE FINAL` can be costly and how to work around it. | -| [Use JSON where appropriate](/best-practices/use-json-where-appropriate) | Considerations for using JSON columns in ClickHouse. | + \ No newline at end of file diff --git a/docs/chdb/guides/index.md b/docs/chdb/guides/index.md index f276e625a2e..2fb11eba418 100644 --- a/docs/chdb/guides/index.md +++ b/docs/chdb/guides/index.md @@ -14,4 +14,13 @@ in the table of contents, please edit the frontmatter of the files directly. --> +| Page | Description | +|-----|-----| +| [How to query a remote ClickHouse server](/chdb/guides/query-remote-clickhouse) | In this guide, we will learn how to query a remote ClickHouse server from chDB. | +| [How to query Apache Arrow with chDB](/chdb/guides/apache-arrow) | In this guide, we will learn how to query Apache Arrow tables with chDB | +| [How to query data in an S3 bucket](/chdb/guides/querying-s3) | Learn how to query data in an S3 bucket with chDB. | +| [How to query Pandas DataFrames with chDB](/chdb/guides/pandas) | Learn how to query Pandas DataFrames with chDB | +| [How to query Parquet files](/chdb/guides/querying-parquet) | Learn how to query Parquet files with chDB. | +| [JupySQL and chDB](/chdb/guides/jupysql) | How to install chDB for Bun | +| [Using a clickhouse-local database](/chdb/guides/clickhouse-local) | Learn how to use a clickhouse-local database with chDB | diff --git a/docs/cloud-index.md b/docs/cloud-index.md deleted file mode 100644 index 911b6d139ff..00000000000 --- a/docs/cloud-index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -slug: /cloud/overview -keywords: ['AWS', 'Cloud', 'serverless'] -title: 'Overview' -hide_title: true -description: 'Overview page for Cloud' ---- - -import Content from '@site/docs/about-us/cloud.md'; - - diff --git a/docs/cloud/_snippets/_clickpipes_faq.md b/docs/cloud/_snippets/_clickpipes_faq.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/cloud/_snippets/_security_table_of_contents.md b/docs/cloud/_snippets/_security_table_of_contents.md new file mode 100644 index 00000000000..9ff837bb8a9 --- /dev/null +++ b/docs/cloud/_snippets/_security_table_of_contents.md @@ -0,0 +1,8 @@ +| Page | Description | +|---------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| [Shared Responsibility Model](/cloud/security/shared-responsibility-model) | Understand how security responsibilities are divided between ClickHouse Cloud and your organization for different service types. | +| [Cloud Access Management](/cloud/security/cloud-access-management) | Manage user access with authentication, single sign-on (SSO), role-based permissions, and team invitations. | +| [Connectivity](/cloud/security/connectivity) | Configure secure network access including IP allow-lists, private networking, S3 data access, and Cloud IP address management. | +| [Enhanced Encryption](/cloud/security/cmek) | Learn about default AES 256 encryption and how to enable Transparent Data Encryption (TDE) for additional data protection at rest. | +| [Audit Logging](/cloud/security/audit-logging) | Set up and use audit logging to track and monitor activities in your ClickHouse Cloud environment. | +| [Privacy and Compliance](/cloud/security/privacy-compliance-overview) | Review security certifications, compliance standards, and learn how to manage your personal information and data rights. | \ No newline at end of file diff --git a/docs/cloud/bestpractices/index.md b/docs/cloud/bestpractices/index.md deleted file mode 100644 index 550f2901bc4..00000000000 --- a/docs/cloud/bestpractices/index.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -slug: /cloud/bestpractices -keywords: ['Cloud', 'Best Practices', 'Bulk Inserts', 'Asynchronous Inserts', 'Avoid mutations', 'Avoid nullable columns', 'Avoid Optimize Final', 'Low Cardinality Partitioning Key', 'Multi Tenancy', 'Usage Limits'] -title: 'Overview' -hide_title: true -description: 'Landing page for Best Practices section in ClickHouse Cloud' ---- - -# Best Practices in ClickHouse Cloud {#best-practices-in-clickhouse-cloud} - -This section provides best practices you will want to follow to get the most out of ClickHouse Cloud. - -| Page | Description | -|----------------------------------------------------------|----------------------------------------------------------------------------| -| [Usage Limits](/cloud/bestpractices/usage-limits)| Explore the limits of ClickHouse. | -| [Multi tenancy](/cloud/bestpractices/multi-tenancy)| Learn about different strategies to implement multi-tenancy. | - -These are in addition to the standard best practices which apply to all deployments of ClickHouse. - -| Page | Description | -|----------------------------------------------------------------------|--------------------------------------------------------------------------| -| [Choosing a Primary Key](/best-practices/choosing-a-primary-key) | Guidance on selecting an effective Primary Key in ClickHouse. | -| [Select Data Types](/best-practices/select-data-types) | Recommendations for choosing appropriate data types. | -| [Use Materialized Views](/best-practices/use-materialized-views) | When and how to benefit from materialized views. | -| [Minimize and Optimize JOINs](/best-practices/minimize-optimize-joins)| Best practices for minimizing and optimizing JOIN operations. | -| [Choosing a Partitioning Key](/best-practices/choosing-a-partitioning-key) | How to choose and apply partitioning keys effectively. | -| [Selecting an Insert Strategy](/best-practices/selecting-an-insert-strategy) | Strategies for efficient data insertion in ClickHouse. | -| [Data Skipping Indices](/best-practices/use-data-skipping-indices-where-appropriate) | When to apply data skipping indices for performance gains. | -| [Avoid Mutations](/best-practices/avoid-mutations) | Reasons to avoid mutations and how to design without them. | -| [Avoid `OPTIMIZE FINAL`](/best-practices/avoid-optimize-final) | Why `OPTIMIZE FINAL` can be costly and how to work around it. | -| [Use JSON where appropriate](/best-practices/use-json-where-appropriate) | Considerations for using JSON columns in ClickHouse. | diff --git a/docs/cloud/bestpractices/usagelimits.md b/docs/cloud/bestpractices/usagelimits.md deleted file mode 100644 index 37ab67b542c..00000000000 --- a/docs/cloud/bestpractices/usagelimits.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -slug: /cloud/bestpractices/usage-limits -sidebar_label: 'Usage Limits' -title: 'Usage limits' -description: 'Describes the recommended usage limits in ClickHouse Cloud' ---- - -While ClickHouse is known for its speed and reliability, optimal performance is achieved within certain operating parameters. For example, having too many tables, databases or parts could negatively impact performance. To avoid this, Clickhouse Cloud has guardrails set up for several types of items. You can find details of these guardrails below. - -:::tip -If you've run up against one of these guardrails, it's possible that you are implementing your use case in an unoptimized way. Contact our support team and we will gladly help you refine your use case to avoid exceeding the guardrails or look together at how we can increase them in a controlled manner. -::: - -| Dimension | Limit | -|-----------|-------| -|**Databases**| 1000| -|**Tables**| 5000| -|**Columns**| ∼1000 (wide format is preferred to compact)| -|**Partitions**| 50k| -|**Parts**| 100k across the entire instance| -|**Part size**| 150gb| -|**Services per organization**| 20 (soft)| -|**Services per warehouse**| 5 (soft)| -|**Low cardinality**| 10k or less| -|**Primary keys in a table**| 4-5 that sufficiently filter down the data| -|**Query concurrency**| 1000| -|**Batch ingest**| anything > 1M will be split by the system in 1M row blocks| - -:::note -For Single Replica Services, the maximum number of databases is restricted to 100, and the maximum number of tables is restricted to 500. In addition, storage for Basic Tier Services is limited to 1 TB. -::: diff --git a/docs/cloud/manage/cloud-tiers.md b/docs/cloud/features/01_cloud_tiers.md similarity index 90% rename from docs/cloud/manage/cloud-tiers.md rename to docs/cloud/features/01_cloud_tiers.md index 244b453f8ab..68a47cd365d 100644 --- a/docs/cloud/manage/cloud-tiers.md +++ b/docs/cloud/features/01_cloud_tiers.md @@ -164,7 +164,7 @@ This page discusses which tiers are right for your specific use case. :::note Services in the basic tier are meant to be fixed in size and do not allow scaling, both automatic and manual. -Users can upgrade to the Scale or Enterprise tier to scale their services. +You can upgrade to the Scale or Enterprise tier to scale their services. ::: ## Scale {#scale} @@ -172,9 +172,10 @@ Users can upgrade to the Scale or Enterprise tier to scale their services. Designed for workloads requiring enhanced SLAs (2+ replica deployments), scalability, and advanced security. - Offers support for features such as: - - [Private networking support](../security/private-link-overview.md). + - [Private networking support](/cloud/security/private-link-overview). - [Compute-compute separation](../reference/warehouses#what-is-compute-compute-separation). - - [Flexible scaling](../manage/scaling.md) options (scale up/down, in/out). + - [Flexible scaling](/manage/scaling) options (scale up/down, in/out). + - [Configurable backups](/cloud/manage/backups/configurable-backups) ## Enterprise {#enterprise} @@ -186,8 +187,8 @@ Caters to large-scale, mission critical deployments that have stringent security - Supports enterprise-grade security: - Single Sign On (SSO) - Enhanced Encryption: For AWS and GCP services. Services are encrypted by our key by default and can be rotated to their key to enable Customer Managed Encryption Keys (CMEK). -- Allows Scheduled upgrades: Users can select the day of the week/time window for upgrades, both database and cloud releases. -- Offers [HIPAA](../security/compliance-overview.md/#hipaa-since-2024) Compliance. +- Allows Scheduled upgrades: you can select the day of the week/time window for upgrades, both database and cloud releases. +- Offers [HIPAA](/cloud/security/compliance-overview#hipaa-since-2024) and PCI compliance. - Exports Backups to the user's account. :::note diff --git a/docs/cloud/features/02_integrations.md b/docs/cloud/features/02_integrations.md new file mode 100644 index 00000000000..d30fb71e7e1 --- /dev/null +++ b/docs/cloud/features/02_integrations.md @@ -0,0 +1,68 @@ +--- +sidebar_label: 'Integrations' +slug: /manage/integrations +title: 'Integrations' +description: 'Integrations for ClickHouse' +--- + +import Kafkasvg from '@site/static/images/integrations/logos/kafka.svg'; +import Confluentsvg from '@site/static/images/integrations/logos/confluent.svg'; +import Msksvg from '@site/static/images/integrations/logos/msk.svg'; +import Azureeventhubssvg from '@site/static/images/integrations/logos/azure_event_hubs.svg'; +import Warpstreamsvg from '@site/static/images/integrations/logos/warpstream.svg'; +import S3svg from '@site/static/images/integrations/logos/amazon_s3_logo.svg'; +import AmazonKinesis from '@site/static/images/integrations/logos/amazon_kinesis_logo.svg'; +import Gcssvg from '@site/static/images/integrations/logos/gcs.svg'; +import DOsvg from '@site/static/images/integrations/logos/digitalocean.svg'; +import ABSsvg from '@site/static/images/integrations/logos/azureblobstorage.svg'; +import Postgressvg from '@site/static/images/integrations/logos/postgresql.svg'; +import Mysqlsvg from '@site/static/images/integrations/logos/mysql.svg'; +import Mongodbsvg from '@site/static/images/integrations/logos/mongodb.svg'; +import redpanda_logo from '@site/static/images/integrations/logos/logo_redpanda.png'; +import clickpipes_stack from '@site/static/images/integrations/data-ingestion/clickpipes/clickpipes_stack.png'; +import cp_custom_role from '@site/static/images/integrations/data-ingestion/clickpipes/cp_custom_role.png'; +import Image from '@theme/IdealImage'; + +ClickHouse Cloud allows you to connect the tools and services that you love. + +## Managed integration pipelines for ClickHouse Cloud {#clickpipes} + +ClickPipes is a managed integration platform that makes ingesting data from a diverse set of sources as simple as clicking a few buttons. +Designed for the most demanding workloads, ClickPipes's robust and scalable architecture ensures consistent performance and reliability. +ClickPipes can be used for long-term streaming needs or one-time data loading job. + +| Name | Logo |Type| Status | Description | +|----------------------------------------------------|--------------------------------------------------------------------------------------------------|----|------------------|------------------------------------------------------------------------------------------------------| +| [Apache Kafka](/integrations/clickpipes/kafka) | |Streaming| Stable | Configure ClickPipes and start ingesting streaming data from Apache Kafka into ClickHouse Cloud. | +| Confluent Cloud | |Streaming| Stable | Unlock the combined power of Confluent and ClickHouse Cloud through our direct integration. | +| Redpanda | Redpanda logo |Streaming| Stable | Configure ClickPipes and start ingesting streaming data from Redpanda into ClickHouse Cloud. | +| AWS MSK | |Streaming| Stable | Configure ClickPipes and start ingesting streaming data from AWS MSK into ClickHouse Cloud. | +| Azure Event Hubs | |Streaming| Stable | Configure ClickPipes and start ingesting streaming data from Azure Event Hubs into ClickHouse Cloud. | +| WarpStream | |Streaming| Stable | Configure ClickPipes and start ingesting streaming data from WarpStream into ClickHouse Cloud. | +| Amazon S3 | |Object Storage| Stable | Configure ClickPipes to ingest large volumes of data from object storage. | +| Google Cloud Storage | |Object Storage| Stable | Configure ClickPipes to ingest large volumes of data from object storage. | +| DigitalOcean Spaces | | Object Storage | Stable | Configure ClickPipes to ingest large volumes of data from object storage. +| Azure Blob Storage | | Object Storage | Private Beta | Configure ClickPipes to ingest large volumes of data from object storage. +| [Amazon Kinesis](/integrations/clickpipes/kinesis) | |Streaming| Stable | Configure ClickPipes and start ingesting streaming data from Amazon Kinesis into ClickHouse cloud. | +| [Postgres](/integrations/clickpipes/postgres) | |DBMS| Stable | Configure ClickPipes and start ingesting data from Postgres into ClickHouse Cloud. | +| [MySQL](/integrations/clickpipes/mysql) | |DBMS| Private Beta | Configure ClickPipes and start ingesting data from MySQL into ClickHouse Cloud. | +| [MongoDB](/integrations/clickpipes/mongodb) | |DBMS| Private Preview | Configure ClickPipes and start ingesting data from MongoDB into ClickHouse Cloud. | + +## Language client integrations {#language-client-integrations} + +ClickHouse offers a number of language client integrations, for which the documentation for each is linked below. + +| Page | Description | +|-------------------------------------------------------------------------|----------------------------------------------------------------------------------| +| [C++](/interfaces/cpp) | C++ Client Library and userver Asynchronous Framework | +| [C#](/integrations/csharp) | Learn how to connect your C# projects to ClickHouse. | +| [Go](/integrations/go) | Learn how to connect your Go projects to ClickHouse. | +| [JavaScript](/integrations/javascript) | Learn how to connect your JS projects to ClickHouse with the official JS client. | +| [Java](/integrations/java) | Learn more about several integrations for Java and ClickHouse. | +| [Python](/integrations/python) | Learn how to connect your Python projects to ClickHouse. | +| [Rust](/integrations/rust) | Learn how to connect your Rust projects to ClickHouse. | +| [Third-party clients](/interfaces/third-party/client-libraries) | Learn more about client libraries from third party developers. | + +In addition to ClickPipes, language clients, ClickHouse supports a host of other integrations, spanning core integrations, +partner integrations and community integrations. +For a complete list see the ["Integrations"](/integrations) section of the docs. \ No newline at end of file diff --git a/docs/cloud/get-started/sql-console.md b/docs/cloud/features/03_sql_console_features/01_sql-console.md similarity index 100% rename from docs/cloud/get-started/sql-console.md rename to docs/cloud/features/03_sql_console_features/01_sql-console.md diff --git a/docs/cloud/get-started/query-insights.md b/docs/cloud/features/03_sql_console_features/02_query-insights.md similarity index 100% rename from docs/cloud/get-started/query-insights.md rename to docs/cloud/features/03_sql_console_features/02_query-insights.md diff --git a/docs/cloud/get-started/query-endpoints.md b/docs/cloud/features/03_sql_console_features/03_query-endpoints.md similarity index 100% rename from docs/cloud/get-started/query-endpoints.md rename to docs/cloud/features/03_sql_console_features/03_query-endpoints.md diff --git a/docs/cloud/manage/dashboards.md b/docs/cloud/features/03_sql_console_features/04_dashboards.md similarity index 100% rename from docs/cloud/manage/dashboards.md rename to docs/cloud/features/03_sql_console_features/04_dashboards.md diff --git a/docs/cloud/features/03_sql_console_features/_category_.json b/docs/cloud/features/03_sql_console_features/_category_.json new file mode 100644 index 00000000000..07e636bd5ed --- /dev/null +++ b/docs/cloud/features/03_sql_console_features/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "SQL console", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/manage/hyperdx.md b/docs/cloud/features/03_sql_console_features/hyperdx.md similarity index 98% rename from docs/cloud/manage/hyperdx.md rename to docs/cloud/features/03_sql_console_features/hyperdx.md index 7e56e90d279..71e5cee6102 100644 --- a/docs/cloud/manage/hyperdx.md +++ b/docs/cloud/features/03_sql_console_features/hyperdx.md @@ -15,7 +15,7 @@ HyperDX is the user interface for [**ClickStack**](/use-cases/observability/clic HyperDX is a purpose-built frontend for exploring and visualizing observability data, supporting both Lucene-style and SQL queries, interactive dashboards, alerting, trace exploration, and more—all optimized for ClickHouse as the backend. -HyperDX in ClickHouse Cloud allows users to enjoy a more turnkey ClickStack experience - no infrastructure to manage, no separate authentication to configure. +HyperDX in ClickHouse Cloud allows users to enjoy a more turnkey ClickStack experience - no infrastructure to manage, no separate authentication to configure. HyperDX can be launched with a single click and connected to your data - fully integrated into the ClickHouse Cloud authentication system for seamless, secure access to your observability insights. ## Deployment {#main-concepts} diff --git a/docs/cloud/features/04_infrastructure/_category_.json b/docs/cloud/features/04_infrastructure/_category_.json new file mode 100644 index 00000000000..41f211b5456 --- /dev/null +++ b/docs/cloud/features/04_infrastructure/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Infrastructure", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/manage/scaling.md b/docs/cloud/features/04_infrastructure/automatic_scaling.md similarity index 100% rename from docs/cloud/manage/scaling.md rename to docs/cloud/features/04_infrastructure/automatic_scaling.md diff --git a/docs/cloud/reference/byoc.md b/docs/cloud/features/04_infrastructure/byoc.md similarity index 99% rename from docs/cloud/reference/byoc.md rename to docs/cloud/features/04_infrastructure/byoc.md index 536a63e7d59..6cf661f3c3f 100644 --- a/docs/cloud/reference/byoc.md +++ b/docs/cloud/features/04_infrastructure/byoc.md @@ -138,7 +138,7 @@ Contact ClickHouse Support to enable Private Load Balancer. 2. Select Peering Connections. 3. Click Create Peering Connection 4. Set the VPC Requester to the ClickHouse VPC ID. -5. Set the VPC Acceptor to the target VPC ID. (Select another account if applicable) +5. Set the VPC Accepter to the target VPC ID. (Select another account if applicable) 6. Click Create Peering Connection.
diff --git a/docs/cloud/manage/replica-aware-routing.md b/docs/cloud/features/04_infrastructure/replica-aware-routing.md similarity index 95% rename from docs/cloud/manage/replica-aware-routing.md rename to docs/cloud/features/04_infrastructure/replica-aware-routing.md index 8b8376b8667..370e46737cd 100644 --- a/docs/cloud/manage/replica-aware-routing.md +++ b/docs/cloud/features/04_infrastructure/replica-aware-routing.md @@ -5,7 +5,11 @@ description: 'How to use Replica-aware routing to increase cache re-use' keywords: ['cloud', 'sticky endpoints', 'sticky', 'endpoints', 'sticky routing', 'routing', 'replica aware routing'] --- -# Replica-aware routing (private preview) +import PrivatePreviewBadge from '@theme/badges/PrivatePreviewBadge'; + +# Replica-aware routing + + Replica-aware routing (also known as sticky sessions, sticky routing, or session affinity) utilizes [Envoy proxy's ring hash load balancing](https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/load_balancers#ring-hash). The main purpose of replica-aware routing is to increase the chance of cache reuse. It does not guarantee isolation. diff --git a/docs/cloud/reference/shared-catalog.md b/docs/cloud/features/04_infrastructure/shared-catalog.md similarity index 100% rename from docs/cloud/reference/shared-catalog.md rename to docs/cloud/features/04_infrastructure/shared-catalog.md diff --git a/docs/cloud/reference/shared-merge-tree.md b/docs/cloud/features/04_infrastructure/shared-merge-tree.md similarity index 100% rename from docs/cloud/reference/shared-merge-tree.md rename to docs/cloud/features/04_infrastructure/shared-merge-tree.md diff --git a/docs/cloud/reference/warehouses.md b/docs/cloud/features/04_infrastructure/warehouses.md similarity index 100% rename from docs/cloud/reference/warehouses.md rename to docs/cloud/features/04_infrastructure/warehouses.md diff --git a/docs/cloud/features/05_admin_features/_category_.json b/docs/cloud/features/05_admin_features/_category_.json new file mode 100644 index 00000000000..72016fcfba5 --- /dev/null +++ b/docs/cloud/features/05_admin_features/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Admin", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/manage/api/api-overview.md b/docs/cloud/features/05_admin_features/api/api-overview.md similarity index 97% rename from docs/cloud/manage/api/api-overview.md rename to docs/cloud/features/05_admin_features/api/api-overview.md index ab0484d0c5c..95a75a1c886 100644 --- a/docs/cloud/manage/api/api-overview.md +++ b/docs/cloud/features/05_admin_features/api/api-overview.md @@ -15,7 +15,7 @@ organizations and services on ClickHouse Cloud. Using our Cloud API, you can create and manage services, provision API keys, add or remove members in your organization, and more. -[Learn how to create your first API key and start using the ClickHouse Cloud API.](/cloud/manage/openapi.md) +[Learn how to create your first API key and start using the ClickHouse Cloud API.](/cloud/manage/openapi) ## Swagger (OpenAPI) Endpoint and UI {#swagger-openapi-endpoint-and-ui} @@ -56,7 +56,8 @@ If your organization has been migrated to one of the [new pricing plans](https:/ You will now also be able to specify the `num_replicas` field as a property of the service resource. ::: -## Terraform and OpenAPI New Pricing: Replica Settings Explained +## Terraform and OpenAPI New Pricing: Replica Settings Explained {#terraform-and-openapi-new-pricing---replica-settings-explained} + The number of replicas each service will be created with defaults to 3 for the Scale and Enterprise tiers, while it defaults to 1 for the Basic tier. For the Scale and the Enterprise tiers it is possible to adjust it by passing a `numReplicas` field in the service creation request. The value of the `numReplicas` field must be between 2 and 20 for the first service in a warehouse. Services that are created in an existing warehouse can have a number of replicas as low as 1. diff --git a/docs/cloud/manage/api/index.md b/docs/cloud/features/05_admin_features/api/index.md similarity index 100% rename from docs/cloud/manage/api/index.md rename to docs/cloud/features/05_admin_features/api/index.md diff --git a/docs/cloud/manage/openapi.md b/docs/cloud/features/05_admin_features/api/openapi.md similarity index 99% rename from docs/cloud/manage/openapi.md rename to docs/cloud/features/05_admin_features/api/openapi.md index 919cb38cc48..6e9b0d4fad3 100644 --- a/docs/cloud/manage/openapi.md +++ b/docs/cloud/features/05_admin_features/api/openapi.md @@ -17,7 +17,7 @@ import Image from '@theme/IdealImage'; ClickHouse Cloud provides an API utilizing OpenAPI that allows you to programmatically manage your account and aspects of your services. :::note -This document covers the ClickHouse Cloud API. For database API endpoints, please see [Cloud Endpoints API](/cloud/get-started/query-endpoints.md) +This document covers the ClickHouse Cloud API. For database API endpoints, please see [Cloud Endpoints API](/cloud/get-started/query-endpoints) ::: 1. You can use the **API Keys** tab on the left menu to create and manage your API keys. diff --git a/docs/cloud/manage/postman.md b/docs/cloud/features/05_admin_features/api/postman.md similarity index 100% rename from docs/cloud/manage/postman.md rename to docs/cloud/features/05_admin_features/api/postman.md diff --git a/docs/cloud/manage/upgrades.md b/docs/cloud/features/05_admin_features/upgrades.md similarity index 99% rename from docs/cloud/manage/upgrades.md rename to docs/cloud/features/05_admin_features/upgrades.md index f52cb9d8bbd..ad28955ffe6 100644 --- a/docs/cloud/manage/upgrades.md +++ b/docs/cloud/features/05_admin_features/upgrades.md @@ -15,7 +15,7 @@ import scheduled_upgrade_window from '@site/static/images/cloud/manage/scheduled # Upgrades -With ClickHouse Cloud you never have to worry about patching and upgrades. We roll out upgrades that include fixes, new features and performance improvements on a periodic basis. For the full list of what is new in ClickHouse refer to our [Cloud changelog](/cloud/reference/changelog.md). +With ClickHouse Cloud you never have to worry about patching and upgrades. We roll out upgrades that include fixes, new features and performance improvements on a periodic basis. For the full list of what is new in ClickHouse refer to our [Cloud changelog](/whats-new/cloud). :::note We are introducing a new upgrade mechanism, a concept we call "make before break" (or MBB). With this new approach, we add updated replica(s) before removing the old one(s) during the upgrade operation. This results in more seamless upgrades that are less disruptive to running workloads. diff --git a/docs/cloud/security/shared-responsibility-model.md b/docs/cloud/features/06_security/01_shared-responsibility-model.md similarity index 98% rename from docs/cloud/security/shared-responsibility-model.md rename to docs/cloud/features/06_security/01_shared-responsibility-model.md index dbb828332eb..08c6a3e073c 100644 --- a/docs/cloud/security/shared-responsibility-model.md +++ b/docs/cloud/features/06_security/01_shared-responsibility-model.md @@ -1,7 +1,7 @@ --- -sidebar_label: 'Shared Responsibility Model' +sidebar_label: 'Shared responsibility model' slug: /cloud/security/shared-responsibility-model -title: 'Security Shared Responsibility Model' +title: 'Shared responsibility model' description: 'Learn more about the security model of ClickHouse Cloud' --- @@ -104,4 +104,4 @@ The model below generally addresses ClickHouse responsibilities and shows respon | HIPAA compliance | Available | AWS, GCP | Enterprise | | PCI compliance | Available | AWS | Enterprise | - For more information on supported compliance frameworks, please review our [Security and Compliance](/cloud/security/security-and-compliance) page. + For more information on supported compliance frameworks, please review our [Security and Compliance](/cloud/security/compliance-overview) page. diff --git a/docs/cloud/features/06_security/02_cloud-access-management/_category_.json b/docs/cloud/features/06_security/02_cloud-access-management/_category_.json new file mode 100644 index 00000000000..784ea5ce006 --- /dev/null +++ b/docs/cloud/features/06_security/02_cloud-access-management/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Cloud access management", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/security/cloud-access-management/cloud-access-management.md b/docs/cloud/features/06_security/02_cloud-access-management/cloud-access-management.md similarity index 76% rename from docs/cloud/security/cloud-access-management/cloud-access-management.md rename to docs/cloud/features/06_security/02_cloud-access-management/cloud-access-management.md index b0794fccf84..cfab1faad61 100644 --- a/docs/cloud/security/cloud-access-management/cloud-access-management.md +++ b/docs/cloud/features/06_security/02_cloud-access-management/cloud-access-management.md @@ -32,23 +32,31 @@ Users must be assigned an organization level role and may optionally be assigned | SQL console | Custom | Configure using SQL [`GRANT`](/sql-reference/statements/grant) statement; assign the role to a SQL console user by naming the role after the user | To create a custom role for a SQL console user and grant it a general role, run the following commands. The email address must match the user's email address in the console. + + + +#### Create `database_developer` and grant permissions {#create-database_developer-and-grant-permissions} + +Create the `database_developer` role and grant `SHOW`, `CREATE`, `ALTER`, and `DELETE` permissions. -1. Create the database_developer role and grant `SHOW`, `CREATE`, `ALTER`, and `DELETE` permissions. - - ```sql - CREATE ROLE OR REPLACE database_developer; - GRANT SHOW ON * TO database_developer; - GRANT CREATE ON * TO database_developer; - GRANT ALTER ON * TO database_developer; - GRANT DELETE ON * TO database_developer; - ``` - -2. Create a role for the SQL console user my.user@domain.com and assign it the database_developer role. +```sql +CREATE ROLE OR REPLACE database_developer; +GRANT SHOW ON * TO database_developer; +GRANT CREATE ON * TO database_developer; +GRANT ALTER ON * TO database_developer; +GRANT DELETE ON * TO database_developer; +``` + +#### Create SQL console user role {#create-sql-console-user-role} + +Create a role for the SQL console user my.user@domain.com and assign it the database_developer role. - ```sql - CREATE ROLE OR REPLACE `sql-console-role:my.user@domain.com`; - GRANT database_developer TO `sql-console-role:my.user@domain.com`; - ``` +```sql +CREATE ROLE OR REPLACE `sql-console-role:my.user@domain.com`; +GRANT database_developer TO `sql-console-role:my.user@domain.com`; +``` + + ### SQL console passwordless authentication {#sql-console-passwordless-authentication} SQL console users are created for each session and authenticated using X.509 certificates that are automatically rotated. The user is removed when the session is terminated. When generating access lists for audits, please navigate to the Settings tab for the service in the console and note the SQL console access in addition to the database users that exist in the database. If custom roles are configured, the user's access is listed in the role ending with the user's username. @@ -88,38 +96,46 @@ Users can use a SHA256 hash generator or code function such as `hashlib` in Pyth ### Database access listings with SQL console users {#database-access-listings-with-sql-console-users} The following process can be used to generate a complete access listing across the SQL console and databases in your organization. -1. Run the following queries to get a list of all grants in the database. - - ```sql - SELECT grants.user_name, - grants.role_name, - users.name AS role_member, - grants.access_type, - grants.database, - grants.table - FROM system.grants LEFT OUTER JOIN system.role_grants ON grants.role_name = role_grants.granted_role_name - LEFT OUTER JOIN system.users ON role_grants.user_name = users.name - - UNION ALL - - SELECT grants.user_name, - grants.role_name, - role_grants.role_name AS role_member, - grants.access_type, - grants.database, - grants.table - FROM system.role_grants LEFT OUTER JOIN system.grants ON role_grants.granted_role_name = grants.role_name - WHERE role_grants.user_name is null; - ``` - -2. Associate this list to Console users with access to SQL console. + + +#### Get a list of all database grants {#get-a-list-of-all-database-grants} + +Run the following queries to get a list of all grants in the database. + +```sql +SELECT grants.user_name, +grants.role_name, +users.name AS role_member, +grants.access_type, +grants.database, +grants.table +FROM system.grants LEFT OUTER JOIN system.role_grants ON grants.role_name = role_grants.granted_role_name +LEFT OUTER JOIN system.users ON role_grants.user_name = users.name + +UNION ALL + +SELECT grants.user_name, +grants.role_name, +role_grants.role_name AS role_member, +grants.access_type, +grants.database, +grants.table +FROM system.role_grants LEFT OUTER JOIN system.grants ON role_grants.granted_role_name = grants.role_name +WHERE role_grants.user_name is null; +``` + +#### Associate grant list to Console users with access to SQL console {#associate-grant-list-to-console-users-with-access-to-sql-console} + +Associate this list with Console users that have access to SQL console. - a. Go to the Console. +a. Go to the Console. + +b. Select the relevant service. - b. Select the relevant service. +c. Select Settings on the left. - c. Select Settings on the left. +d. Scroll to the SQL console access section. - d. Scroll to the SQL console access section. +e. Click the link for the number of users with access to the database `There are # users with access to this service.` to see the user listing. - e. Click the link for the number of users with access to the database `There are # users with access to this service.` to see the user listing. + \ No newline at end of file diff --git a/docs/cloud/security/cloud-access-management/cloud-authentication.md b/docs/cloud/features/06_security/02_cloud-access-management/cloud-authentication.md similarity index 100% rename from docs/cloud/security/cloud-access-management/cloud-authentication.md rename to docs/cloud/features/06_security/02_cloud-access-management/cloud-authentication.md diff --git a/docs/cloud/security/cloud-access-management/index.md b/docs/cloud/features/06_security/02_cloud-access-management/index.md similarity index 100% rename from docs/cloud/security/cloud-access-management/index.md rename to docs/cloud/features/06_security/02_cloud-access-management/index.md diff --git a/docs/cloud/security/inviting-new-users.md b/docs/cloud/features/06_security/02_cloud-access-management/inviting-new-users.md similarity index 100% rename from docs/cloud/security/inviting-new-users.md rename to docs/cloud/features/06_security/02_cloud-access-management/inviting-new-users.md diff --git a/docs/cloud/features/06_security/03_connectivity/_category_.json b/docs/cloud/features/06_security/03_connectivity/_category_.json new file mode 100644 index 00000000000..6e137e0592d --- /dev/null +++ b/docs/cloud/features/06_security/03_connectivity/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Connectivity", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/security/cloud-endpoints-api.md b/docs/cloud/features/06_security/03_connectivity/cloud-endpoints-api.md similarity index 100% rename from docs/cloud/security/cloud-endpoints-api.md rename to docs/cloud/features/06_security/03_connectivity/cloud-endpoints-api.md diff --git a/docs/cloud/security/connectivity-overview.md b/docs/cloud/features/06_security/03_connectivity/connectivity-overview.md similarity index 100% rename from docs/cloud/security/connectivity-overview.md rename to docs/cloud/features/06_security/03_connectivity/connectivity-overview.md diff --git a/docs/cloud/security/aws-privatelink.md b/docs/cloud/features/06_security/03_connectivity/private_networking/aws-privatelink.md similarity index 96% rename from docs/cloud/security/aws-privatelink.md rename to docs/cloud/features/06_security/03_connectivity/private_networking/aws-privatelink.md index 4a7c9644eb7..720a3a88852 100644 --- a/docs/cloud/security/aws-privatelink.md +++ b/docs/cloud/features/06_security/03_connectivity/private_networking/aws-privatelink.md @@ -375,7 +375,7 @@ jq .result.privateEndpointIds ### Connecting to a remote database {#connecting-to-a-remote-database} -Let's say you are trying to use [MySQL](../../sql-reference/table-functions/mysql.md) or [PostgreSQL](../../sql-reference/table-functions/postgresql.md) table functions in ClickHouse Cloud and connect to your database hosted in an Amazon Web Services (AWS) VPC. AWS PrivateLink cannot be used to enable this connection securely. PrivateLink is a one-way, unidirectional connection. It allows your internal network or Amazon VPC to connect securely to ClickHouse Cloud, but it does not allow ClickHouse Cloud to connect to your internal network. +Let's say you are trying to use [MySQL](/sql-reference/table-functions/mysql) or [PostgreSQL](/sql-reference/table-functions/postgresql) table functions in ClickHouse Cloud and connect to your database hosted in an Amazon Web Services (AWS) VPC. AWS PrivateLink cannot be used to enable this connection securely. PrivateLink is a one-way, unidirectional connection. It allows your internal network or Amazon VPC to connect securely to ClickHouse Cloud, but it does not allow ClickHouse Cloud to connect to your internal network. According to the [AWS PrivateLink documentation](https://docs.aws.amazon.com/whitepapers/latest/building-scalable-secure-multi-vpc-network-infrastructure/aws-privatelink.html): diff --git a/docs/cloud/security/azure-privatelink.md b/docs/cloud/features/06_security/03_connectivity/private_networking/azure-privatelink.md similarity index 100% rename from docs/cloud/security/azure-privatelink.md rename to docs/cloud/features/06_security/03_connectivity/private_networking/azure-privatelink.md diff --git a/docs/cloud/security/gcp-private-service-connect.md b/docs/cloud/features/06_security/03_connectivity/private_networking/gcp-private-service-connect.md similarity index 97% rename from docs/cloud/security/gcp-private-service-connect.md rename to docs/cloud/features/06_security/03_connectivity/private_networking/gcp-private-service-connect.md index 8fcd99e06fd..cc573e09b6c 100644 --- a/docs/cloud/security/gcp-private-service-connect.md +++ b/docs/cloud/features/06_security/03_connectivity/private_networking/gcp-private-service-connect.md @@ -421,7 +421,7 @@ curl --silent --user "${KEY_ID:?}:${KEY_SECRET:?}" -X GET -H "Content-Type: appl ### Connecting to a remote database {#connecting-to-a-remote-database} -Let's say you are trying to use the [MySQL](../../sql-reference/table-functions/mysql.md) or [PostgreSQL](../../sql-reference/table-functions/postgresql.md) table functions in ClickHouse Cloud and connect to your database hosted in GCP. GCP PSC cannot be used to enable this connection securely. PSC is a one-way, unidirectional connection. It allows your internal network or GCP VPC to connect securely to ClickHouse Cloud, but it does not allow ClickHouse Cloud to connect to your internal network. +Let's say you are trying to use the [MySQL](/sql-reference/table-functions/mysql) or [PostgreSQL](/sql-reference/table-functions/postgresql) table functions in ClickHouse Cloud and connect to your database hosted in GCP. GCP PSC cannot be used to enable this connection securely. PSC is a one-way, unidirectional connection. It allows your internal network or GCP VPC to connect securely to ClickHouse Cloud, but it does not allow ClickHouse Cloud to connect to your internal network. According to the [GCP Private Service Connect documentation](https://cloud.google.com/vpc/docs/private-service-connect): diff --git a/docs/cloud/security/private-link-overview.md b/docs/cloud/features/06_security/03_connectivity/private_networking/private-link-overview.md similarity index 62% rename from docs/cloud/security/private-link-overview.md rename to docs/cloud/features/06_security/03_connectivity/private_networking/private-link-overview.md index 183362a8e58..8d6be0c413e 100644 --- a/docs/cloud/security/private-link-overview.md +++ b/docs/cloud/features/06_security/03_connectivity/private_networking/private-link-overview.md @@ -9,6 +9,6 @@ description: 'Landing page for private link' ClickHouse Cloud provides the ability to connect your services to your cloud virtual network. Refer to the guides below for your provider: -- [AWS private Link](/cloud/security/aws-privatelink.md) -- [GCP private service connect](/cloud/security/gcp-private-service-connect.md) -- [Azure private link](/cloud/security/azure-privatelink.md) +- [AWS private Link](/manage/security/aws-privatelink) +- [GCP private service connect](/manage/security/gcp-private-service-connect) +- [Azure private link](/cloud/security/azure-privatelink) diff --git a/docs/cloud/security/setting-ip-filters.md b/docs/cloud/features/06_security/03_connectivity/setting-ip-filters.md similarity index 100% rename from docs/cloud/security/setting-ip-filters.md rename to docs/cloud/features/06_security/03_connectivity/setting-ip-filters.md diff --git a/docs/cloud/features/06_security/_category_.json b/docs/cloud/features/06_security/_category_.json new file mode 100644 index 00000000000..aed26fa7f7a --- /dev/null +++ b/docs/cloud/features/06_security/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Security", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/security/cmek.md b/docs/cloud/features/06_security/cmek.md similarity index 100% rename from docs/cloud/security/cmek.md rename to docs/cloud/features/06_security/cmek.md diff --git a/docs/cloud/features/07_monitoring/_category_.json b/docs/cloud/features/07_monitoring/_category_.json new file mode 100644 index 00000000000..ef0bd973e2c --- /dev/null +++ b/docs/cloud/features/07_monitoring/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Monitoring", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/manage/monitoring/advanced_dashboard.md b/docs/cloud/features/07_monitoring/advanced_dashboard.md similarity index 99% rename from docs/cloud/manage/monitoring/advanced_dashboard.md rename to docs/cloud/features/07_monitoring/advanced_dashboard.md index ab320eb6ebe..578a412846e 100644 --- a/docs/cloud/manage/monitoring/advanced_dashboard.md +++ b/docs/cloud/features/07_monitoring/advanced_dashboard.md @@ -110,7 +110,7 @@ interface can help detect issues. | Network receive bytes/sec | Tracks the current speed of outbound network traffic | | Concurrent network connections | Tracks the number of current concurrent network connections | -## Identifying issues with the Advanced dashboard {#identifying-issues-with-the-advanced-dashboard} +## Identifying issues using the advanced dashboard {#identifying-issues-with-the-advanced-dashboard} Having this real-time view of the health of your ClickHouse service greatly helps mitigate issues before they impact your business or help solve them. Below are a diff --git a/docs/cloud/manage/notifications.md b/docs/cloud/features/07_monitoring/notifications.md similarity index 100% rename from docs/cloud/manage/notifications.md rename to docs/cloud/features/07_monitoring/notifications.md diff --git a/docs/cloud/manage/monitoring/prometheus.md b/docs/cloud/features/07_monitoring/prometheus.md similarity index 100% rename from docs/cloud/manage/monitoring/prometheus.md rename to docs/cloud/features/07_monitoring/prometheus.md diff --git a/docs/cloud/manage/backups/configurable-backups.md b/docs/cloud/features/08_backups/configurable-backups.md similarity index 100% rename from docs/cloud/manage/backups/configurable-backups.md rename to docs/cloud/features/08_backups/configurable-backups.md diff --git a/docs/cloud/manage/backups/export-backups-to-own-cloud-account.md b/docs/cloud/features/08_backups/export-backups-to-own-cloud-account.md similarity index 97% rename from docs/cloud/manage/backups/export-backups-to-own-cloud-account.md rename to docs/cloud/features/08_backups/export-backups-to-own-cloud-account.md index 4cd5ea78b62..0bb2be7cda7 100644 --- a/docs/cloud/manage/backups/export-backups-to-own-cloud-account.md +++ b/docs/cloud/features/08_backups/export-backups-to-own-cloud-account.md @@ -15,7 +15,7 @@ For details of how ClickHouse Cloud backups work, including "full" vs. "incremen Here we show examples of how to take full and incremental backups to AWS, GCP, Azure object storage as well as how to restore from the backups. :::note -Users should be aware that any usage where backups are being exported to a different region in the same cloud provider, will incur [data transfer](../network-data-transfer.mdx) charges. Currently we do not support cross cloud backups. +Users should be aware that any usage where backups are being exported to a different region in the same cloud provider, will incur [data transfer](/cloud/manage/network-data-transfer) charges. Currently we do not support cross cloud backups. ::: ## Requirements {#requirements} diff --git a/docs/cloud/manage/backups/index.md b/docs/cloud/features/08_backups/index.md similarity index 100% rename from docs/cloud/manage/backups/index.md rename to docs/cloud/features/08_backups/index.md diff --git a/docs/cloud/manage/backups/overview.md b/docs/cloud/features/08_backups/overview.md similarity index 100% rename from docs/cloud/manage/backups/overview.md rename to docs/cloud/features/08_backups/overview.md diff --git a/docs/cloud/support.md b/docs/cloud/features/09_support.md similarity index 88% rename from docs/cloud/support.md rename to docs/cloud/features/09_support.md index 836382cd3c5..e6b73fc87a0 100644 --- a/docs/cloud/support.md +++ b/docs/cloud/features/09_support.md @@ -1,6 +1,6 @@ --- sidebar_label: 'Cloud Support' -title: 'Cloud Support' +title: 'Support' slug: /cloud/support description: 'Learn about Cloud Support' hide_title: true diff --git a/docs/cloud/features/_category_.json b/docs/cloud/features/_category_.json new file mode 100644 index 00000000000..383c8150644 --- /dev/null +++ b/docs/cloud/features/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Features", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/get-started/index.md b/docs/cloud/get-started/index.md deleted file mode 100644 index 3c30f63f149..00000000000 --- a/docs/cloud/get-started/index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -slug: /cloud/get-started -title: 'Get Started' -description: 'Get Started Table Of Contents' -keywords: ['Cloud Quick Start', 'SQL Console', 'Query Insights', 'Query API Endpoints', 'Dashboards', 'Cloud Support'] ---- - -Welcome to ClickHouse Cloud! Explore the pages below to learn more about what ClickHouse Cloud has to offer. - -| Page | Description | -|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Overview](/cloud/overview) | Overview of the benefits of using ClickHouse Cloud and what version of ClickHouse is used for it. | -| [SQL Console](/cloud/get-started/sql-console) | Learn about the interactive SQL console available in Cloud | -| [Query Insights](/cloud/get-started/query-insights) | Learn about how Cloud's Query Insights feature makes ClickHouse's built-in query log easier to use through various visualizations and tables. | -| [Query Endpoints](/cloud/get-started/query-endpoints) | Learn about the Query API Endpoints feature which allows you to create an API endpoint directly from any saved SQL query in the ClickHouse Cloud console. | -| [Dashboards](/cloud/manage/dashboards) | Learn about how SQL Console's dashboards feature allows you to collect and share visualizations from saved queries. | -| [Cloud Support](/cloud/support) | Learn more about Support Services for ClickHouse Cloud users and customers. | diff --git a/docs/cloud/bestpractices/_category_.yml b/docs/cloud/guides/_category_.yml similarity index 83% rename from docs/cloud/bestpractices/_category_.yml rename to docs/cloud/guides/_category_.yml index 1648e8a79cb..747e5fb1796 100644 --- a/docs/cloud/bestpractices/_category_.yml +++ b/docs/cloud/guides/_category_.yml @@ -1,4 +1,4 @@ -label: 'Best Practices' +label: 'Guides' collapsible: true collapsed: true link: diff --git a/docs/cloud/guides/best_practices/_category_.json b/docs/cloud/guides/best_practices/_category_.json new file mode 100644 index 00000000000..21f95c55bca --- /dev/null +++ b/docs/cloud/guides/best_practices/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Best practices", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/guides/best_practices/index.md b/docs/cloud/guides/best_practices/index.md new file mode 100644 index 00000000000..4719ea2750a --- /dev/null +++ b/docs/cloud/guides/best_practices/index.md @@ -0,0 +1,22 @@ +--- +slug: /cloud/bestpractices +keywords: ['Cloud', 'Best Practices', 'Bulk Inserts', 'Asynchronous Inserts', 'Avoid Mutations', 'Avoid Nullable Columns', 'Avoid Optimize Final', 'Low Cardinality Partitioning Key', 'Multi Tenancy', 'Usage Limits'] +title: 'Overview' +hide_title: true +description: 'Landing page for Best Practices section in ClickHouse Cloud' +--- + +import TableOfContents from '@site/docs/best-practices/_snippets/_table_of_contents.md'; + +# Best Practices in ClickHouse Cloud {#best-practices-in-clickhouse-cloud} + +This section provides best practices you will want to follow to get the most out of ClickHouse Cloud. + +| Page | Description | +|----------------------------------------------------------|----------------------------------------------------------------------------| +| [Usage Limits](/cloud/bestpractices/usage-limits)| Explore the limits of ClickHouse. | +| [Multi tenancy](/cloud/bestpractices/multi-tenancy)| Learn about different strategies to implement multi-tenancy. | + +These are in addition to the standard best practices which apply to all deployments of ClickHouse. + + \ No newline at end of file diff --git a/docs/cloud/bestpractices/multitenancy.md b/docs/cloud/guides/best_practices/multitenancy.md similarity index 99% rename from docs/cloud/bestpractices/multitenancy.md rename to docs/cloud/guides/best_practices/multitenancy.md index 5289a09b067..5f7df65427a 100644 --- a/docs/cloud/bestpractices/multitenancy.md +++ b/docs/cloud/guides/best_practices/multitenancy.md @@ -1,6 +1,6 @@ --- slug: /cloud/bestpractices/multi-tenancy -sidebar_label: 'Implement multi tenancy' +sidebar_label: 'Multi tenancy' title: 'Multi tenancy' description: 'Best practices to implement multi tenancy' --- diff --git a/docs/cloud/guides/best_practices/usagelimits.md b/docs/cloud/guides/best_practices/usagelimits.md new file mode 100644 index 00000000000..af49f5956be --- /dev/null +++ b/docs/cloud/guides/best_practices/usagelimits.md @@ -0,0 +1,40 @@ +--- +slug: /cloud/bestpractices/usage-limits +sidebar_label: 'Service limits' +title: 'Usage limits' +description: 'Describes the recommended usage limits in ClickHouse Cloud' +--- + +While ClickHouse is known for its speed and reliability, optimal performance is +achieved within certain operating parameters. For example, having too many tables, +databases or parts could negatively impact performance. To avoid this, Clickhouse +Cloud has guardrails set up for several types of items. You can find details of +these guardrails below. + +:::tip +If you've run up against one of these guardrails, it's possible that you are +implementing your use case in an unoptimized way. Contact our support team and +we will gladly help you refine your use case to avoid exceeding the guardrails +or look together at how we can increase them in a controlled manner. +::: + +| Dimension | Limit | +|-------------------------------|------------------------------------------------------------| +| **Databases** | 1000 | +| **Tables** | 5000 | +| **Columns** | ∼1000 (wide format is preferred to compact) | +| **Partitions** | 50k | +| **Parts** | 100k across the entire instance | +| **Part size** | 150gb | +| **Services per organization** | 20 (soft) | +| **Services per warehouse** | 5 (soft) | +| **Low cardinality** | 10k or less | +| **Primary keys in a table** | 4-5 that sufficiently filter down the data | +| **Query concurrency** | 1000 | +| **Batch ingest** | anything > 1M will be split by the system in 1M row blocks | + +:::note +For Single Replica Services, the maximum number of databases is restricted to +100, and the maximum number of tables is restricted to 500. In addition, storage +for Basic Tier Services is limited to 1 TB. +::: diff --git a/docs/cloud/reference/cloud-compatibility.md b/docs/cloud/guides/cloud-compatibility.md similarity index 99% rename from docs/cloud/reference/cloud-compatibility.md rename to docs/cloud/guides/cloud-compatibility.md index 86dafbfefd5..59c238c9c08 100644 --- a/docs/cloud/reference/cloud-compatibility.md +++ b/docs/cloud/guides/cloud-compatibility.md @@ -1,6 +1,6 @@ --- slug: /whats-new/cloud-compatibility -sidebar_label: 'Cloud Compatibility' +sidebar_label: 'Cloud compatibility' title: 'Cloud Compatibility' description: 'This guide provides an overview of what to expect functionally and operationally in ClickHouse Cloud.' --- diff --git a/docs/cloud/guides/index.md b/docs/cloud/guides/index.md new file mode 100644 index 00000000000..2355ca4370c --- /dev/null +++ b/docs/cloud/guides/index.md @@ -0,0 +1,6 @@ +--- +slug: /cloud/guides +title: 'Guides' +hide_title: true +description: 'Table of contents page for the ClickHouse Cloud guides section' +--- \ No newline at end of file diff --git a/docs/cloud/guides/security/_category_.json b/docs/cloud/guides/security/_category_.json new file mode 100644 index 00000000000..aed26fa7f7a --- /dev/null +++ b/docs/cloud/guides/security/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Security", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/guides/security/cloud_access_management/_category_.json b/docs/cloud/guides/security/cloud_access_management/_category_.json new file mode 100644 index 00000000000..abfdcebed27 --- /dev/null +++ b/docs/cloud/guides/security/cloud_access_management/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Cloud Access Management", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/security/common-access-management-queries.md b/docs/cloud/guides/security/cloud_access_management/common-access-management-queries.md similarity index 100% rename from docs/cloud/security/common-access-management-queries.md rename to docs/cloud/guides/security/cloud_access_management/common-access-management-queries.md diff --git a/docs/cloud/security/saml-sso-setup.md b/docs/cloud/guides/security/cloud_access_management/saml-sso-setup.md similarity index 100% rename from docs/cloud/security/saml-sso-setup.md rename to docs/cloud/guides/security/cloud_access_management/saml-sso-setup.md diff --git a/docs/cloud/guides/security/connectivity/_category_.json b/docs/cloud/guides/security/connectivity/_category_.json new file mode 100644 index 00000000000..6e137e0592d --- /dev/null +++ b/docs/cloud/guides/security/connectivity/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Connectivity", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/security/accessing-s3-data-securely.md b/docs/cloud/guides/security/connectivity/accessing-s3-data-securely.md similarity index 100% rename from docs/cloud/security/accessing-s3-data-securely.md rename to docs/cloud/guides/security/connectivity/accessing-s3-data-securely.md diff --git a/docs/cloud/manage/_category_.yml b/docs/cloud/manage/_category_.yml deleted file mode 100644 index 59089856c86..00000000000 --- a/docs/cloud/manage/_category_.yml +++ /dev/null @@ -1,6 +0,0 @@ -label: 'Manage Cloud' -collapsible: true -collapsed: true -link: - type: generated-index - title: Manage ClickHouse Cloud diff --git a/docs/cloud/manage/index.md b/docs/cloud/manage/index.md deleted file mode 100644 index 46c407d0c6b..00000000000 --- a/docs/cloud/manage/index.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -slug: /cloud/manage -keywords: ['AWS', 'Cloud', 'serverless', 'management'] -title: 'Overview' -hide_title: true -description: 'Overview page for Managing Cloud' ---- - -# Managing Cloud - -In this section of the docs you will find all the information you may need about managing ClickHouse cloud. This section contains the following pages: - -| Page | Description | -|-----------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| [ClickHouse Cloud Tiers](/cloud/manage/cloud-tiers) | Describes the different cloud tiers, their features, and considerations for choosing the right one. | -| [Integrations](/manage/integrations) | Covers ClickHouse Cloud's built-in integrations, custom integrations, and integrations that are not supported. | -| [Backups](/cloud/manage/backups) | Describes how backups work in ClickHouse Cloud, what options you have to configure backups for your service, and how to restore from a backup. | -| [Monitoring](/integrations/prometheus) | How to integrate Prometheus as a way to monitor ClickHouse cloud. | -| [Billing](/cloud/manage/billing/overview) | Explains the pricing model for ClickHouse Cloud, including the factors that affect the cost of your service. | -| [Configuring Settings](/manage/settings) | Describes how to configure settings for ClickHouse Cloud. | -| [Replica-aware Routing](/manage/replica-aware-routing) | Explains what Replica-aware Routing in ClickHouse Cloud is, its limitations, and how to configure it. | -| [Automatic Scaling](/manage/scaling) | Explains how ClickHouse Cloud services can be scaled manually or automatically based on your resource needs. | -| [Service Uptime and SLA](/cloud/manage/service-uptime) | Information about service uptime and Service Level Agreements offered for production instances. | -| [Notifications](/cloud/notifications) | Shows how ClickHouse Cloud notifications are received and how they can be customized. | -| [Upgrades](/manage/updates) | Information on how upgrades are rolled out in ClickHouse Cloud. | -| [Delete Account](/cloud/manage/close_account) | Information on how to close or delete your account when necessary. | -| [Programmatic API Access with Postman](/cloud/manage/postman) | A guide to help you test the ClickHouse API using Postman. | -| [Troubleshooting](/faq/troubleshooting) | A collection of commonly encountered issues and how to troubleshoot them. | -| [Data Transfer](./network-data-transfer.mdx) | Learn more about how ClickHouse Cloud meters data transferred ingress and egress. | -| [Jan 2025 Changes FAQ](./jan2025_faq/index.md) | Learn more about changes to Cloud introduced in Jan 2025. | diff --git a/docs/cloud/manage/integrations.md b/docs/cloud/manage/integrations.md deleted file mode 100644 index 7f4472b33e7..00000000000 --- a/docs/cloud/manage/integrations.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -sidebar_label: 'Integrations' -slug: /manage/integrations -title: 'Integrations' -description: 'Integrations for ClickHouse' ---- - -To see a full list of integrations for ClickHouse, please see [this page](/integrations). - -## Proprietary integrations for ClickHouse Cloud {#proprietary-integrations-for-clickhouse-cloud} - -Besides the dozens of integrations available for ClickHouse, there are also some proprietary integrations only available for ClickHouse Cloud: - -### ClickPipes {#clickpipes} - -[ClickPipes](/integrations/clickpipes) is a managed integration platform to ingest data into ClickHouse Cloud using a simple, web-based UI. It currently supports Apache Kafka, S3, GCS and Amazon Kinesis, with more integrations coming soon. - -### Looker Studio for ClickHouse Cloud {#looker-studio-for-clickhouse-cloud} - -[Looker Studio](https://lookerstudio.google.com/) is a popular business intelligence tool provided by Google. Looker Studio does not currently provide a ClickHouse connector but instead relies on the MySQL wire protocol to connect to ClickHouse. - -Looker Studio can be connected to ClickHouse Cloud by enabling the [MySQL interface](/interfaces/mysql). Please see [this page](/interfaces/mysql#enabling-the-mysql-interface-on-clickhouse-cloud) for details on connecting Looker Studio to ClickHouse Cloud. - -### MySQL Interface {#mysql-interface} - -Some applications currently do not support the ClickHouse wire protocol. To use ClickHouse Cloud with these applications, you can enable the MySQL wire protocol through the Cloud console. Please see [this page](/interfaces/mysql#enabling-the-mysql-interface-on-clickhouse-cloud) for details on how to enable the MySQL wire protocol through the Cloud console. - -## Unsupported integrations {#unsupported-integrations} - -The following features for integrations are not currently available for ClickHouse Cloud as they are experimental features. If you need to support these features in your application, please contact support@clickhouse.com. - -- [MaterializedPostgreSQL](/engines/table-engines/integrations/materialized-postgresql) diff --git a/docs/cloud/manage/jan2025_faq/_snippets/_clickpipes_faq.md b/docs/cloud/manage/jan2025_faq/_snippets/_clickpipes_faq.md deleted file mode 100644 index 436a1cb705b..00000000000 --- a/docs/cloud/manage/jan2025_faq/_snippets/_clickpipes_faq.md +++ /dev/null @@ -1,145 +0,0 @@ -import Image from '@theme/IdealImage'; -import clickpipesPricingFaq1 from '@site/static/images/cloud/manage/jan2025_faq/external_clickpipes_pricing_faq_1.png'; -import clickpipesPricingFaq2 from '@site/static/images/cloud/manage/jan2025_faq/external_clickpipes_pricing_faq_2.png'; -import clickpipesPricingFaq3 from '@site/static/images/cloud/manage/jan2025_faq/external_clickpipes_pricing_faq_3.png'; - -
- -Why are we introducing a pricing model for ClickPipes now? - -We decided to initially launch ClickPipes for free with the idea to gather -feedback, refine features, and ensure it meets user needs. -As the GA platform has grown, it has effectively stood the test of time by -moving trillions of rows. Introducing a pricing model allows us to continue -improving the service, maintaining the infrastructure, and providing dedicated -support and new connectors. - -
- -
- -What are ClickPipes replicas? - -ClickPipes ingests data from remote data sources via a dedicated infrastructure -that runs and scales independently of the ClickHouse Cloud service. -For this reason, it uses dedicated compute replicas. -The diagrams below show a simplified architecture. - -For streaming ClickPipes, ClickPipes replicas access the remote data sources (e.g., a Kafka broker), -pull the data, process and ingest it into the destination ClickHouse service. - -ClickPipes Replicas - Streaming ClickPipes - -In the case of object storage ClickPipes, -the ClickPipes replica orchestrates the data loading task -(identifying files to copy, maintaining the state, and moving partitions), -while the data is pulled directly from the ClickHouse service. - -ClickPipes Replicas - Object Storage ClickPipes - -
- -
- -What's the default number of replicas and their size? - -Each ClickPipe defaults to 1 replica that's provided with 2 GiB of RAM and 0.5 vCPU. -This corresponds to **0.25** ClickHouse compute units (1 unit = 8 GiB RAM, 2 vCPUs). - -
- -
- -Can ClickPipes replicas be scaled? - -Yes, ClickPipes for streaming can be scaled both horizontally and vertically. -Horizontal scaling adds more replicas to increase throughput, while vertical scaling increases the resources (CPU and RAM) allocated to each replica to handle more intensive workloads. -This can be configured during ClickPipe creation, or at any other point under **Settings** -> **Advanced Settings** -> **Scaling**. - -
- -
- -How many ClickPipes replicas do I need? - -It depends on the workload throughput and latency requirements. -We recommend starting with the default value of 1 replica, measuring your latency, and adding replicas if needed. -Keep in mind that for Kafka ClickPipes, you also have to scale the Kafka broker partitions accordingly. -The scaling controls are available under "settings" for each streaming ClickPipe. - -ClickPipes Replicas - How many ClickPipes replicas do I need? - -
- -
- -What does the ClickPipes pricing structure look like? - -It consists of two dimensions: -- **Compute**: Price per unit per hour - Compute represents the cost of running the ClickPipes replica pods whether they actively ingest data or not. - It applies to all ClickPipes types. -- **Ingested data**: per GB pricing - The ingested data rate applies to all streaming ClickPipes - (Kafka, Confluent, Amazon MSK, Amazon Kinesis, Redpanda, WarpStream, - Azure Event Hubs) for the data transferred via the replica pods. - The ingested data size (GB) is charged based on bytes received from the source (uncompressed or compressed). - -
- -
- -What are the ClickPipes public prices? - -- Compute: \$0.20 per unit per hour ($0.05 per replica per hour) -- Ingested data: $0.04 per GB - -
- -
- -How does it look in an illustrative example? - -For example, ingesting 1 TB of data over 24 hours using the Kafka connector using a single replica (0.25 compute unit) costs: - -$$ -(0.25 \times 0.20 \times 24) + (0.04 \times 1000) = \$41.2 -$$ -
- -For object storage connectors (S3 and GCS), -only the ClickPipes compute cost is incurred since the ClickPipes pod is not processing data -but only orchestrating the transfer which is operated by the underlying ClickHouse service: - -$$ -0.25 \times 0,20 \times 24 = \$1.2 -$$ - -
- -
- -When does the new pricing model take effect? - -The new pricing model takes effect for all organizations created after January 27th, 2025. - -
- -
- -What happens to current users? - -Existing users will have a **60-day grace period** where the ClickPipes service continues to be offered for free. -Billing will automatically start for ClickPipes for existing users on **March 24th, 2025.** - -
- -
- -How does ClickPipes pricing compare to the market? - -The philosophy behind ClickPipes pricing is -to cover the operating costs of the platform while offering an easy and reliable way to move data to ClickHouse Cloud. -From that angle, our market analysis revealed that we are positioned competitively. - -
diff --git a/docs/cloud/manage/jan2025_faq/backup.md b/docs/cloud/manage/jan2025_faq/backup.md deleted file mode 100644 index 579788f8dec..00000000000 --- a/docs/cloud/manage/jan2025_faq/backup.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: 'Backup Policy' -slug: /cloud/manage/jan-2025-faq/backup -keywords: ['new tiers', 'plans', 'pricing', 'backups'] -description: 'Backup policy in new tiers' ---- - -## What is the backup policy? {#what-is-the-backup-policy} -In Basic, Scale, and Enterprise tiers backups are metered and billed separately from storage. -All services will default to one daily backup with the ability to configure more, starting with the Scale tier, via the Settings tab of the Cloud console. Each backup will be retained for at least 24 hours. - -## What happens to current configurations that users have set up separate from default backups? {#what-happens-to-current-configurations-that-users-have-set-up-separate-from-default-backups} - -Customer specific backup configurations will carry over. Users can change these as they see fit in the new tiers. - -## Are backups charged differently across tiers? {#are-backups-charged-differently-across-tiers} - -The cost of backups is the same across all tiers. diff --git a/docs/cloud/manage/jan2025_faq/billing.md b/docs/cloud/manage/jan2025_faq/billing.md deleted file mode 100644 index 4147cc9976d..00000000000 --- a/docs/cloud/manage/jan2025_faq/billing.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: 'Billing' -slug: /cloud/manage/jan-2025-faq/billing -keywords: ['new pricing', 'billing'] -description: 'Billing details for new pricing tiers' ---- - -## Billing {#billing} - -### Are there any changes to how usage is metered and charged? {#are-there-any-changes-to-how-usage-is-metered-and-charged} - -The per-dimension unit cost for compute and storage has changed, and there are two additional dimensions to account for data transfer and ClickPipes usage. - -Some notable changes: - -- Storage price per TB will be reduced, and storage cost will no longer include backups (we will charge for them separately and will make only one backup required). Storage costs are the same across tiers and vary by region and cloud service provider. -- Compute costs will vary by tier, region, and cloud service provider. -- The new pricing dimension for data transfer is applicable for data egress across regions and on the public internet only. -- New pricing dimension for ClickPipes usage. - -### What happens to users with existing committed spend contracts? {#what-happens-to-users-with-existing-committed-spend-contracts} - -Users with active committed spend contracts will not be affected by the new per-dimension unit cost prices for compute and storage until their contract expires. However, the new pricing dimensions for data transfer and ClickPipes will be applicable starting March 24, 2025. Most customers will not see a significant increase in their monthly bill from these new dimensions. - -### Can users on a committed spend agreement with ClickHouse continue to launch services on the old plan? {#can-users-on-a-committed-spend-agreement-with-clickhouse-continue-to-launch-services-on-the-old-plan} - -Yes, users will be able to launch Development and Production services until the end date of their contract, and renewals will reflect the new pricing plan. - -If you need to modify your contract or have questions about how these changes might affect you in the future, please contact our support team or your sales representative. - -### What happens if users exhaust their credits before the end of the contract and go to PAYG? {#what-happens-if-users-exhaust-their-credits-before-the-end-of-the-contract-and-go-to-payg} - -If committed spend contracts exhaust credits before their renewal date, we bill them at the current rates until renewal (as per current policy). - -### What happens to users on the monthly PAYG? {#what-happens-to-users-on-the-monthly-payg} - -Users on a monthly PAYG plan will continue to be billed using the old pricing plan for the Development and Production services. They have until July 23, 2025, to migrate to the new plan self-serve, or they will all be migrated to the Scale configuration on this day and billed based on the new plan. - -### Where can I reference legacy plans? {#where-can-i-reference-legacy-plans} - -Legacy plans are available for reference [here](https://clickhouse.com/pricing?legacy=true). - -## Marketplaces {#marketplaces} - -### Are there changes to how users are charged via the CSP marketplaces? {#are-there-changes-to-how-users-are-charged-via-the-csp-marketplaces} - -Users who sign up to ClickHouse Cloud via a CSP Marketplace incur usage in terms of CHCs (ClickHouse Cloud Credits). This behavior has not changed. However, the underlying composition of the credit usage will align with the pricing and packaging changes outlined here and include charges for any data transfer usage and ClickPipes once those are live. diff --git a/docs/cloud/manage/jan2025_faq/dimensions.md b/docs/cloud/manage/jan2025_faq/dimensions.md deleted file mode 100644 index c4dd9268593..00000000000 --- a/docs/cloud/manage/jan2025_faq/dimensions.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: 'New Pricing Dimensions' -slug: /cloud/manage/jan-2025-faq/pricing-dimensions -keywords: ['new pricing', 'dimensions'] -description: 'Pricing dimensions for data transfer and ClickPipes' ---- - -import Image from '@theme/IdealImage'; -import clickpipesPricingFaq1 from '@site/static/images/cloud/manage/jan2025_faq/external_clickpipes_pricing_faq_1.png'; -import clickpipesPricingFaq2 from '@site/static/images/cloud/manage/jan2025_faq/external_clickpipes_pricing_faq_2.png'; -import clickpipesPricingFaq3 from '@site/static/images/cloud/manage/jan2025_faq/external_clickpipes_pricing_faq_3.png'; -import NetworkPricing from '@site/docs/cloud/manage/_snippets/_network_transfer_rates.md'; -import ClickPipesFAQ from './_snippets/_clickpipes_faq.md' - -The following dimensions have been added to the new ClickHouse Cloud pricing. - -:::note -Data transfer and ClickPipes pricing doesn't apply to legacy plans, i.e. Development, Production, and Dedicated, until 24 March 2025. -::: - -## Data transfer pricing {#data-transfer-pricing} - -### How are users charged for data transfer, and will this vary across organization tiers and regions? {#how-are-users-charged-for-data-transfer-and-will-this-vary-across-organization-tiers-and-regions} - -- Users pay for data transfer along two dimensions — public internet egress and inter-region egress. There are no charges for intra-region data transfer or Private Link/Private Service Connect use and data transfer. However, we reserve the right to implement additional data transfer pricing dimensions if we see usage patterns that impact our ability to charge users appropriately. -- Data transfer pricing varies by Cloud Service Provider (CSP) and region. -- Data transfer pricing does **not** vary between organizational tiers. -- Public egress pricing is based only on the origin region. Inter-region (or cross-region) pricing depends on both the origin and destination regions. - - - -### Will data transfer pricing be tiered as usage increases? {#will-data-transfer-pricing-be-tiered-as-usage-increases} - -Data transfer prices will **not** be tiered as usage increases. Pricing varies by region and cloud service provider. - -## ClickPipes pricing FAQ {#clickpipes-pricing-faq} - - diff --git a/docs/cloud/manage/jan2025_faq/index.md b/docs/cloud/manage/jan2025_faq/index.md deleted file mode 100644 index 840e07c06e7..00000000000 --- a/docs/cloud/manage/jan2025_faq/index.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: 'Jan 2025 Changes FAQ' -slug: /cloud/manage/jan-2025-faq -description: 'Index page for new pricing FAQ' -keywords: ['new pricing', 'faq'] ---- - - - - - diff --git a/docs/cloud/manage/jan2025_faq/new_tiers.md b/docs/cloud/manage/jan2025_faq/new_tiers.md deleted file mode 100644 index b90874aedb8..00000000000 --- a/docs/cloud/manage/jan2025_faq/new_tiers.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: 'Description of New Tiers' -slug: /cloud/manage/jan-2025-faq/new-tiers -keywords: ['new tiers', 'features', 'pricing', 'description'] -description: 'Description of new tiers and features' ---- - -## Summary of key changes {#summary-of-key-changes} - -### What key changes to expect with regard to features to tier mapping? {#what-key-changes-to-expect-with-regard-to-features-to-tier-mapping} - -- **Private Link/Private Service Connect:** Private connections are now supported across all service types on Scale and Enterprise tiers (including single-replica services). This means you can now have Private Link for both your production (large scale) and development (small scale) environments. -- **Backups:** All services now come with one backup by default and additional backups are charged separately. Users can leverage the configurable backup controls to manage additional backups. This means that services with lesser backup requirements do not need to pay a higher bundled price. Please see more details in the Backup FAQ. -- **Enhanced Encryption:** This feature is available in Enterprise tier services, including for single replica services, in AWS and GCP. Services are encrypted by our key by default and can be rotated to their key to enable Customer Managed Encryption Keys (CMEK). -- **Single Sign On (SSO):** This feature is offered in Enterprise tier and requires a support ticket to be enabled for an Organization. Users who have multiple Organizations should ensure all of their organizations are on the Enterprise tier to use SSO for each organization. - -## Basic tier {#basic-tier} - -### What are the considerations for the Basic tier? {#what-are-the-considerations-for-the-basic-tier} - -The basic tier is meant for small workloads - users want to deploy a small analytics application that does not require high availability or work on a prototype. This tier is not suitable for workloads that need scale, reliability (DR/HA), and data durability. The tier supports single replica services of fixed size 1x8GiB or 1x12GiB. Please refer to the docs and [support policy](https://clickhouse.com/support/program) for more information. - -### Can users on the Basic tier access Private Link and Private Service Connect? {#can-users-on-the-basic-tier-access-private-link-and-private-service-connect} - -No, Users will need to upgrade to Scale or Enterprise to access this feature. - -### Can users on the Basic and Scale tiers set up SSO for the organization? {#can-users-on-the-basic-and-scale-tiers-set-up-sso-for-the-organization} - -No, users will need to upgrade to the Enterprise tier to access this feature. - -### Can users launch single replica services in all tiers? {#can-users-launch-single-replica-services-in-all-tiers} - -Yes, single replica services are supported on all three tiers. Users can scale out, but are not permitted to scale into a single replica. - -### Can users scale up/down and add more replicas on the Basic tier? {#can-users-scale-updown-and-add-more-replicas-on-the-basic-tier} - -No, services on this tier are meant to support workloads that are small and fixed size (single replica `1x8GiB` or `1x12GiB`). The size of the single replica is fixed upon service creation and cannot be adjusted or scaled after service creation. If users need to scale up/down or add replicas, they will be prompted to upgrade to Scale or Enterprise tiers. - -## Scale tier {#scale-tier} - -### Which tiers on the new plans (Basic/Scale/Enterprise) support compute-compute separation? {#which-tiers-on-the-new-plans-basicscaleenterprise-support-compute-compute-separation} - -Only Scale and Enterprise tiers support compute-compute separation. Please also note that this capability requires running at least a 2+ replica parent service. - -### Can users on the legacy plans (Production/Development) access compute-compute separation? {#can-users-on-the-legacy-plans-productiondevelopment-access-compute-compute-separation} - -Compute-compute separation is not supported on existing Development and Production services, except for users who already participated in the Private Preview and Beta. If you have additional questions, please contact [support](https://clickhouse.com/support/program). - -## Enterprise tier {#enterprise-tier} - -### What different hardware profiles are supported for the Enterprise tier? {#what-different-hardware-profiles-are-supported-for-the-enterprise-tier} - -The enterprise tier will support standard profiles (1:4 vCPU:memory ratio), as well as `highMem (1:8 ratio)` and `highCPU (1:2 ratio)` **custom profiles,** offering users more flexibility to select the configuration that best suits their needs. The Enterprise Tier will use shared compute resources deployed alongside the Basic and Scale tiers. - -### What are the features exclusively offered on the Enterprise tier? {#what-are-the-features-exclusively-offered-on-the-enterprise-tier} - -- **Custom profiles:** Options for instance type selection standard profiles (1:4 vCPU: memory ratio) and `highMem (1:8 ratio)` and `highCPU (1:2 ratio)` custom profiles. -- **Enterprise-grade security:** - - **Single Sign On (SSO**) - - **Enhanced Encryption:** For AWS and GCP services. Services are encrypted by our key by default and can be rotated to their key to enable Customer Managed Encryption Keys (CMEK). -- **Scheduled upgrades:** Users can select the day of the week/time window for upgrades, both database and cloud releases. -- **HIPAA Compliance:** The customer must sign a Business Associate Agreement (BAA) through Legal before we enable HIPAA-compliant regions for them. -- **Private Regions:** It is not self-serve enabled and will need users to route requests through Sales sales@clickhouse.com. -- **Export Backups** to the customer's cloud account. diff --git a/docs/cloud/manage/jan2025_faq/plan_migrations.md b/docs/cloud/manage/jan2025_faq/plan_migrations.md deleted file mode 100644 index fffdebfe45b..00000000000 --- a/docs/cloud/manage/jan2025_faq/plan_migrations.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -title: 'Migrating to New Plans' -slug: /cloud/manage/jan-2025-faq/plan-migrations -keywords: ['migration', 'new tiers', 'pricing', 'cost', 'estimation'] -description: 'Migrating to new plans, tiers, pricing, how to decide and estimate costs' ---- - -## Choosing new plans {#choosing-new-plans} - -### Can new organizations launch services on the old (legacy) plan? {#can-new-organizations-launch-services-on-the-old-legacy-plan} - -No, newly created organizations will not have access to the old plan after the announcement. - -### Can users migrate to the new pricing plan self-serve? {#can-users-migrate-to-the-new-pricing-plan-self-serve} - -Yes, see below for guidance on self-serve migrations: - -| Current Plan | New Plan | Self-Serve Migration | -|--------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| Development | Basic | Supported if all services in the organization support are Development | -| Development | Scale (2 replicas+) | :white_check_mark: | -| Development | Enterprise (2 replicas+) | :white_check_mark: | -| Production | Scale (3 replicas+) | :white_check_mark: | -| Production | Enterprise (3 replicas+) | :white_check_mark: | -| Dedicated | Contact [support](https://clickhouse.com/support/program) | - -### What will the experience be for users in trial running Development and Production services? {#what-will-the-experience-be-for-users-in-trial-running-development-and-production-services} - -Users can upgrade during the trial and continue to use the trial credits to evaluate the new service tiers and the features it supports. However, if they choose to continue using the same Development and Production services, they can do so and upgrade to PAYG. They will still have to migrate before July 23, 2025. - -### Can users upgrade their tiers {#can-users-upgrade-their-tiers-ie-basic--scale-scale--enterprise-etc} - -Can users upgrade their tiers, for example, Basic → Scale, Scale → Enterprise, etc. -Yes, users can upgrade self-serve, and the pricing will reflect the tier selection after upgrade. - -### Can users move from a higher to a lower-cost tier {#can-users-move-from-a-higher-to-a-lower-cost-tier-eg-enterprise--scale-scale--basic-enterprise--basic-self-serve} - -For example, Enterprise → Scale, Scale → Basic, Enterprise → Basic self-serve? -Yes, but users will need to remove all premium features and may be guided to scale their multi-replica services into a single replica. - -### Can users with only development services in the organization migrate to the Basic tier? {#can-users-with-only-development-services-in-the-organization-migrate-to-the-basic-tier} - -Yes, this would be permitted. Users will be given a recommendation based on their past use and can select Basic `1x8GiB` or `1x12GiB`. - -### Can users with a development and production service in the same organization move to the basic tier? {#can-users-with-a-development-and-production-service-in-the-same-organization-move-to-the-basic-tier} - -No, if a user has both Development and Production services in the same organization, they can self-serve and migrate only to the Scale or Enterprise tier. If they want to migrate to Basic, they should delete all existing Production services. - -### Are there any changes related to the Scaling behavior with the new tiers? {#are-there-any-changes-related-to-the-scaling-behavior-with-the-new-tiers} - -We are introducing a new vertical scaling mechanism for compute replicas, which we call "Make Before Break" (MBB). This approach adds one or more replicas of the new size before removing the old replicas, preventing any loss of capacity during scaling operations. By eliminating the gap between removing existing replicas and adding new ones, MBB creates a more seamless and less disruptive scaling process. It is especially beneficial in scale-up scenarios, where high resource utilization triggers the need for additional capacity, since removing replicas prematurely would only exacerbate the resource constraints. - -Please note that as part of this change, historical system table data will be retained for up to a maximum of 30 days as part of scaling events. In addition, any system table data older than December 19, 2024, for services on AWS or GCP and older than January 14, 2025, for services on Azure will not be retained as part of the migration to the new organization tiers. - -## Estimating costs {#estimating-costs} - -### How will users be guided during migration, understanding what tier best fits their needs? {#how-will-users-be-guided-during-migration-understanding-what-tier-best-fits-their-needs} - -The console will prompt you with recommended options for each service based on historical use if you have a service. New users can review the capabilities and features listed in detail and decide on the tier that best suits their needs. - -### How do users size and estimate the cost of "warehouses" in the new pricing? {#how-do-users-size-and-estimate-the-cost-of-warehouses-in-the-new-pricing} - -Please refer to the pricing calculator on the [Pricing](https://clickhouse.com/pricing) page, which will help estimate the cost based on your workload size and tier selection. - -## Undertaking the migration {#undertaking-the-migration} - -### What are service version pre-requisites to undertaking the migration? {#what-are-service-version-pre-requisites-to-undertaking-the-migration} - -Your service has to be on version 24.8 or later and already migrated to SharedMergeTree. - -### What is the migration experience for users of the current Development and Production services? Do users need to plan for a maintenance window where the service is unavailable? {#what-is-the-migration-experience-for-users-of-the-current-development-and-production-services-do-users-need-to-plan-for-a-maintenance-window-where-the-service-is-unavailable} - -Migrations of Development and Production services to the new pricing tiers may trigger a rolling restart. To migrate a Dedicated service, please contact [support](https://clickhouse.com/support/program). - -### What other actions should a user take after the migration? {#what-other-actions-should-a-user-take-after-the-migration} - -API access patterns will be different. - -Users that use our OpenAPI to create new services will be required to remove the `tier` field in the service creation `POST` request. - -The `tier` field has been removed from the service object as we no longer have service tiers. -This will affect the objects returned by the `POST`, `GET`, and `PATCH` service requests. Therefore, any code that consumes these APIs may need to be adjusted to handle these changes. - -The number of replicas each service will be created with defaults to 3 for the Scale and Enterprise tiers, while it defaults to 1 for the Basic tier. -For the Scale and the Enterprise tiers it is possible to adjust it by passing a `numReplicas` field in the service creation request. -The value of the `numReplicas` field must be between 2 and 20 for the first service in a warehouse. Services that are created in an existing warehouse can have a number of replicas as low as 1. - -### What changes should the users make if using the existing Terraform provider for automation? {#what-changes-should-the-users-make-if-using-the-existing-terraform-provider-for-automation} - -Once an organization has been migrated to one of the new plans, users will be required to use our Terraform provider version 2.0.0 or above. - -The new Terraform provider is required to handle changes in the `tier` attribute of the service. - -After the migration, the `tier` field is no longer accepted, and references to it should be removed. - -Users will also be able to specify the `num_replicas` field as a property of the service resource. - -The number of replicas each service will be created with defaults to 3 for the Scale and Enterprise tiers, while it defaults to 1 for the Basic tier. -For the Scale and the Enterprise tiers, it is possible to adjust it by passing a `numReplicas` field in the service creation request. -The value of the `num_replicas` filed must be between 2 and 20 for the first service in a warehouse. Services that are created in an existing warehouse can have a number of replicas as low as 1. - -### Will users have to make any changes to the database access? {#will-users-have-to-make-any-changes-to-the-database-access} - -No, the database username/password will work the same as before. - -### Will users have to reconfigure private networking features? {#will-users-have-to-reconfigure-private-networking-features} - -No, users can use their existing private networking (Private Link, PSC, etc..) configuration after moving their Production service to Scale or Enterprise. diff --git a/docs/cloud/manage/jan2025_faq/scaling.md b/docs/cloud/manage/jan2025_faq/scaling.md deleted file mode 100644 index e65aff7345e..00000000000 --- a/docs/cloud/manage/jan2025_faq/scaling.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: 'Scaling' -slug: /cloud/manage/jan-2025-faq/scaling -keywords: ['new pricing', 'faq', 'scaling'] -description: 'Scaling behavior in new pricing tiers' ---- - -ClickHouse Cloud allows scaling in both directions - vertical (increasing replica size) and horizontal (adding more replicas). - -## What scaling options will be available for each tier? {#what-scaling-options-will-be-available-for-each-tier} - -The scaling behavior per tier is as follows: - -* **Basic**: Basic tier supports only single replica services. These services are meant to be fixed in size and do not allow vertical or horizontal scaling. Users can upgrade to the Scale or Enterprise tier to scale their services. -* **Scale**: Scale tier supports single and multi-replica services. Scaling will be permitted for Multi-replica services. - * Services can vertically scale to the maximum replica size supported for a CSP/region AFTER they have scaled to a multi-replica setup; only 2+ replicas can be vertically scaled. - * Manual horizontal scaling will be available. -* **Enterprise**: Enterprise tier supports single and multi-replica services, and scaling will be permitted for Multi-replica services - * Services can vertically scale to maximum replica sizes supported for a CSP/region. - * Standard profiles (1:4 CPU to memory ratio) will support vertical auto-scaling - * Custom profiles (`highMemory` and `highCPU`) can be scaled vertically through a support ticket. - * Manual horizontal scaling will be available. - -:::note -Services can scale horizontally to a maximum of 20 replicas. If you need additional replicas, please contact our support team. -::: - -## Can users scale in their service? {#can-users-scale-in-their-service} - -Scaling in will be restricted to 2+ replicas. Once scaled out, users will not be permitted to scale down to a single replica, as this may result in instability and potential data loss. - -## Are there any changes related to the Scaling behavior with the new tiers? {#are-there-any-changes-related-to-the-scaling-behavior-with-the-new-tiers} - -We are introducing a new vertical scaling mechanism for compute replicas, which we call "Make Before Break" (MBB). This approach adds one or more replicas of the new size before removing the old replicas, preventing any loss of capacity during scaling operations. By eliminating the gap between removing existing replicas and adding new ones, MBB creates a more seamless and less disruptive scaling process. It is especially beneficial in scale-up scenarios, where high resource utilization triggers the need for additional capacity, since removing replicas prematurely would only exacerbate the resource constraints. - -Please note that as part of this change, historical system table data will be retained for up to a maximum of 30 days as part of scaling events. In addition, any system table data older than December 19, 2024, for services on AWS or GCP and older than January 14, 2025, for services on Azure will not be retained as part of the migration to the new organization tiers. diff --git a/docs/cloud/manage/jan2025_faq/summary.md b/docs/cloud/manage/jan2025_faq/summary.md deleted file mode 100644 index dfeafe642d3..00000000000 --- a/docs/cloud/manage/jan2025_faq/summary.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: 'Summary' -slug: /cloud/manage/jan-2025-faq/summary -keywords: ['new tiers', 'packaging', 'pricing faq', 'summary'] -description: 'Summary of New ClickHouse Cloud Tiers' ---- - -The following FAQ summarizes common questions with respect to new tiers introduced in ClickHouse Cloud starting in January 2025. - -## What has changed with ClickHouse Cloud tiers? {#what-has-changed-with-clickhouse-cloud-tiers} - -At ClickHouse, we are dedicated to adapting our products to meet the ever-changing requirements of our customers. Since its introduction in GA over the past two years, ClickHouse Cloud has evolved substantially, and we've gained invaluable insights into how our customers leverage our cloud offerings. - -We are introducing new features to optimize the sizing and cost-efficiency of ClickHouse Cloud services for your workloads. These include compute-compute separation, high-performance machine types, and single-replica services. We are also evolving automatic scaling and managed upgrades to execute in a more seamless and reactive fashion. - -We are adding a new Enterprise tier to serve the needs of the most demanding customers and workloads, with focus on industry-specific security and compliance features, even more controls over underlying hardware and upgrades, and advanced disaster recovery features. - -You can read about these and other functional changes in this [blog](https://clickhouse.com/blog/evolution-of-clickhouse-cloud-new-features-superior-performance-tailored-offerings). - -## What action is required? {#what-action-is-required} - -To support these changes, we are restructuring our current tiers to more closely match how our evolving customer base is using our offerings, and you need to take action to select a new plan. - -Details and timelines for making these selections are described below. - -## How are tiers changing? {#how-are-tiers-changing} - -We are transitioning from a model that organizes paid tiers purely by "service types" which are delineated by both capacity and features (namely, these are Development, Production, and Dedicated tiers) to one that organizes paid tiers by feature availability. These new tiers are called Basic, Scale, and Enterprise and are described in more detail below. - -This change brings several key benefits: - -* **Consistent Feature Access**: Features present in a tier will be available in that tier for all sizes of services, as well as in all tiers above it. For example, private networking, previously available only for Production service types, will now be accessible for all services starting with the Scale tier, so you can deploy it for services sized both for development and production workloads as you see fit. - -* **Organizational-Level Features**: We can now provide features built at an organizational level with the appropriate plan, ensuring that customers receive the tools they need at the right level of service. For example, access to SSO (single-sign-on) and CMEK (customer-managed encryption keys) will be available at the Enterprise tier. - -* **Optimized Support Plans**: The new packaging structure also allows us to align support response times with paid tiers, which more effectively meet the needs of our diverse customer base. For example, we are now making named support engineers available to our Enterprise tier customers. - -Below we provide an overview of the new tiers, describe how they relate to use cases, and outline key features. - -**Basic: A taste of ClickHouse** - -* Basic tier is designed to offer a budget-friendly option for organizations with smaller data volumes and less demanding workloads. It allows you to run single-replica deployments with up to 12GB of memory and less than 1TB of storage and is ideal for small-scale use cases that do not require reliability guarantees. - -**Scale: Enhanced SLAs and scalability** - -* Scale tier is suitable for workloads that require enhanced SLAs, greater scalability, and advanced security measures. -* It offers unlimited compute and storage with any replication factor, access to compute-compute separation, and automatic vertical and horizontal scaling. -* Key features include: - * Support for private networking, customized backup controls, multi-factor auth, and more - * Compute-compute separation for optimized resource usage - * Flexible scaling options (both vertical and horizontal) to meet changing demands - -**Enterprise: Mission-critical deployments** - -* Enterprise tier is the best place to run large-scale, mission-critical ClickHouse deployments. -* It is best suited for organizations with stringent security and compliance needs, requiring the highest levels of performance and reliability. -* Key features include: - * Industry-specific compliance certifications, such as HIPAA - * Self-service access to SSO (Single Sign-On) and CMEK (Customer Managed Encryption Keys) - * Scheduled upgrades to ensure minimal disruption - * Support for custom configurations, including high-memory, high-CPU options, and private regions - -New tiers are described in more detail on our [website](https://clickhouse.com/pricing). - -## How is pricing changing? {#how-is-pricing-changing} - -In addition to evolving our paid tiers, we are making the following adjustments to our overall pricing structure and price points: - -* **Storage**: Storage price per TB will be reduced and will no longer bundle backups in the storage cost. -* **Backups**: Backups will be charged separately, with only one backup being mandatory. -* **Compute**: Compute costs will increase, varying by tier and region. This increase may be balanced by the introduction of compute-compute separation and single-replica services, which allow you to optimize compute usage by deploying and right-sizing services tailored to different workload types. -* **Data Transfer**: We are introducing charges for data egress, specifically for data transfer over the internet and cross region. Based on our analysis, most customers will not see a substantial increase in their monthly bill based on this new dimension. -* **ClickPipes**: Our managed ingest service, which was offered for free during the introductory period, will now incur charges based on compute and ingested data. Based on our analysis, most customers will not see a substantial increase in their monthly bill based on this new dimension. - -## When will these changes take effect? {#when-will-these-changes-take-effect} - -While changes are effective immediately for new customers, existing customers will have from 6 months to a year to transition to new plans. - -Detailed breakdown of effective dates is below: - -* **New Customers**: The new plans will take effect on **January 27, 2025** for new customers of ClickHouse Cloud. -* **Existing PAYG Customers**: Pay-as-you-go (PAYG) customers will have 6 months until **July 23, 2025** to migrate to new plans. -* **Existing Committed Spend Customers**: Customers with committed spend agreements can renegotiate their terms at the end of their current contract. -* **New usage dimensions** for Data Transfer and ClickPipes are effective for both PAYG and Committed Spend customers 8 weeks following this announcement on **March 24, 2025**. - -## What actions should you take? {#what-actions-should-you-take} - -If you are a **pay-as-you-go (PAYG) customer**, you can migrate to a new plan through the self-service options available in your ClickHouse Cloud console. - -If you are a **committed spend customer**, please reach out to your account representative to discuss your custom migration plan and timeline. - -**Need assistance?** -We're here to support you through this transition. If you have any questions or need personalized help, please reach out to your account representative or contact our support team. diff --git a/docs/cloud/manage/network-data-transfer.mdx b/docs/cloud/manage/network-data-transfer.mdx deleted file mode 100644 index 92725e6015c..00000000000 --- a/docs/cloud/manage/network-data-transfer.mdx +++ /dev/null @@ -1,36 +0,0 @@ ---- -sidebar_label: 'Data Transfer' -slug: /cloud/manage/network-data-transfer -title: 'Data Transfer' -description: 'Learn more about how ClickHouse Cloud meters data transferred ingress and egress' ---- - -import NetworkPricing from '@site/docs/cloud/manage/_snippets/_network_transfer_rates.md'; - -ClickHouse Cloud meters data transferred ingress and egress. -This includes any data in and out of ClickHouse Cloud as well as any intra-region and cross-region data transfer. -This usage is tracked at the service level. Based on this usage, customers incur data transfer charges that are then added to their monthly bill. - -ClickHouse Cloud charges for: -- Data egress from ClickHouse Cloud to the public Internet, including to other regions of other cloud providers. -- Data egress to another region in the same cloud provider. - -There are no charges for intra-region data transfer or Private Link/Private Service Connect use and data transfer. -However, we reserve the right to implement additional data transfer pricing dimensions if we see usage patterns that impact our ability to charge users appropriately. - -Data transfer charges vary by Cloud Service Provider (CSP) and region, and prices will not be tiered as usage increases. Public internet egress pricing is based only on the origin region. -Inter-region (or cross-region) pricing depends on both the origin and destination regions. Data transfer pricing does **not** vary between organizational tiers. - -**Best Practices to minimize Data Transfer Costs** - -There are some patterns to keep in mind when ingressing and egressing data to minimize data transfer costs. -1. When ingressing or egressing data from Clickhouse Cloud, use compression where possible, to minimize the amount of data transferred and the associated cost. -2. Be aware that when doing an INSERT over the native protocol with non-inlined values (e.g. INSERT INTO [TABLE] FROM INFILE [FILE] FORMAT NATIVE), ClickHouse clients pull metadata from servers to pack the data. If the metadata is larger than the INSERT payload, you might counterintuitively see more egress than there is ingress from the server perspective. If this is unacceptable, consider inlining data with VALUES syntax or using the HTTP protocol. - -The tables below shows how data transfer charges for egress vary across public internet or cross-region by cloud provider and region. - -:::note -ClickHouse Cloud meters inter-region usage in terms of tiers, Tier 1 through Tier 4, depending on the origin and destination regions. The table below shows the tier for each combination of inter-region data transfer. In the Billing usage screen on ClickHouse Cloud you will see data transfer usage broken out by tiers. -::: - - diff --git a/docs/cloud/migrate/upload-a-csv-file.md b/docs/cloud/migrate/upload-a-csv-file.md deleted file mode 100644 index 71347b0f55e..00000000000 --- a/docs/cloud/migrate/upload-a-csv-file.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -title: 'Uploading files' -slug: /cloud/migrate/upload-a-csv-file -description: 'Learn how to upload files to Cloud' ---- - -import Image from '@theme/IdealImage'; -import csv_01 from '@site/static/images/cloud/migrate/csv_01.png'; -import csv_02 from '@site/static/images/cloud/migrate/csv_02.png'; -import csv_03 from '@site/static/images/cloud/migrate/csv_03.png'; -import csv_04 from '@site/static/images/cloud/migrate/csv_04.png'; -import csv_05 from '@site/static/images/cloud/migrate/csv_05.png'; -import csv_06 from '@site/static/images/cloud/migrate/csv_06.png'; -import csv_07 from '@site/static/images/cloud/migrate/csv_07.png'; -import csv_08 from '@site/static/images/cloud/migrate/csv_08.png'; -import csv_09 from '@site/static/images/cloud/migrate/csv_09.png'; -import csv_10 from '@site/static/images/cloud/migrate/csv_10.png'; - -# Upload files to Cloud - -ClickHouse Cloud provides an easy way to import your files and supports the -following formats: - -| Format | -|---------------------------------| -| `CSV` | -| `CSVWithNamesAndTypes` | -| `CSVWithNames` | -| `JSONEachRow` | -| `TabSeparated` | -| `TabSeparatedWithNames` | -| `TabSeparatedWithNamesAndTypes` | - - - -## Upload a file {#upload-file} - -From the Cloud homepage, select your service as shown below: - -upload_file_02 - -If your service is idle you will need to wake it. - -Select `Data sources` in the left hand tab as shown below: - -upload_file_03 - -Next select `Upload a file` on the right side of the data sources page: - -upload_file_04 - -A file dialogue will pop up allowing you to select the file that you wish to -use to insert data into a table on your Cloud service. - -upload_file_05 - -## Configure table {#configure-table} - -Once the file has uploaded you will be able to configure the table where you want -to insert the data to. A preview of the table with the first three rows is shown. - -upload_file_08 - -You can now select a destination table. The options are: - -- a new table -- an existing table - -
-You can specify which database you want to upload the data to, and in the case of -a new table, the name of the table that will be created. You will also be able to select the sorting key: - -upload_file_05 - -Columns read from the file are shown as `Source field`s and for each field, you -can change: -- the inferred type -- the default value -- whether to make the column [Nullable](/sql-reference/data-types/nullable) or not - -upload_file_06 - -:::note Excluding fields -You can also remove a field if you don't want to include it in the import -::: - -You can specify the type of table engine that you want to use: - -- `MergeTree` -- `ReplacingMergeTree` -- `SummingMergeTree` -- `Null` -
-You can specify a partitioning key expression and primary -key expression. - -upload_file_07 - -Click `Import to ClickHouse` (shown above) to import the data. The data import will be queued as -indicated by the `queued` status badge in the `Status` column as shown below. You can also click -`Open as query` (shown above) to open the insert query in the SQL console. The query will insert -the file which was uploaded to an S3 bucket using the `URL` table function. - -upload_file_09 - -If the job fails you will see a `failed` status badge under the `Status` column of -the `Data upload history` tab. You can click `View Details` for more information -on why the upload failed. You may need to modify the table configuration or clean -the data based on the error message for the failed insert. - -upload_file_11 - - \ No newline at end of file diff --git a/docs/cloud/onboard/01_discover/01_what_is.md b/docs/cloud/onboard/01_discover/01_what_is.md new file mode 100644 index 00000000000..37a515a551f --- /dev/null +++ b/docs/cloud/onboard/01_discover/01_what_is.md @@ -0,0 +1,47 @@ +--- +slug: /cloud/overview +title: 'Introduction' +description: 'Learn what ClickHouse Cloud is, its benefits over open-source, and key features of the fully managed analytics platform' +keywords: ['clickhouse cloud', 'what is clickhouse cloud', 'clickhouse cloud overview', 'clickhouse cloud features'] +hide_title: true +--- + +## What is ClickHouse Cloud? {#what-is-clickhouse-cloud} + +ClickHouse Cloud is a fully managed cloud service created by the original creators +of ClickHouse, the fastest and most popular open-source columnar online analytical +processing database. + +With Cloud, infrastructure, maintenance, scaling, and operations are taken care of +for you, so that you can focus on what matters most to you, which is building value +for your organization and your customers faster. + +## Benefits of ClickHouse Cloud {#benefits-of-clickhouse-cloud} + +ClickHouse Cloud offers several major benefits over the open-source version: + +- **Fast time to value**: Start building instantly without having to size and scale your cluster. +- **Seamless scaling**: Automatic scaling adjusts to variable workloads so you don't have to over-provision for peak usage. +- **Serverless operations**: Sit back while we take care of sizing, scaling, security, reliability, and upgrades. +- **Transparent pricing**: Pay only for what you use, with resource reservations and scaling controls. +- **Total cost of ownership**: Best price / performance ratio and low administrative overhead. +- **Broad ecosystem**: Bring your favorite data connectors, visualization tools, SQL and language clients with you. + +## OSS vs ClickHouse Cloud comparison {#oss-vs-clickhouse-cloud} + +| Feature | Benefits | OSS ClickHouse | ClickHouse Cloud | +|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|-------------------| +| **Deployment modes** | ClickHouse provides the flexibility to self-manage with open-source or deploy in the cloud. Use ClickHouse local for local files without a server or chDB to embed ClickHouse directly into your application. | ✅ | ✅ | +| **Storage** | As an open-source and cloud-hosted product, ClickHouse can be deployed in both shared-disk and shared-nothing architectures. | ✅ | ✅ | +| **Monitoring and alerting** | Monitoring and alerting about the status of your services is critical to ensuring optimal performance and a proactive approach to detect and triage potential issues. | ✅ | ✅ | +| **ClickPipes** | ClickPipes is ClickHouse's managed ingestion pipeline that allows you to seamlessly connect your external data sources like databases, APIs, and streaming services into ClickHouse Cloud, eliminating the need for managing pipelines, custom jobs, or ETL processes. It supports workloads of all sizes. | ❌ | ✅ | +| **Pre-built integrations** | ClickHouse provides pre-built integrations that connect ClickHouse to popular tools and services such as data lakes, SQL and language clients, visualization libraries, and more. | ❌ | ✅ | +| **SQL console** | The SQL console offers a fast, intuitive way to connect, explore, and query ClickHouse databases, featuring a slick caption, query interface, data import tools, visualizations, collaboration features, and GenAI-powered SQL assistance. | ❌ | ✅ | +| **Compliance** | ClickHouse Cloud compliance includes CCPA, EU-US DPF, GDPR, HIPAA, ISO 27001, ISO 27001 SoA, PCI DSS, SOC2. ClickHouse Cloud's security, availability, processing integrity, and confidentiality processes are all independently audited. Details: trust.clickhouse.com. | ❌ | ✅ | +| **Enterprise-grade security** | Support for advanced security features such as SSO, multi-factor authentication, role-based access control (RBAC), private and secure connections with support for Private Link and Private Service Connect, IP filtering, customer-managed encryption keys (CMEK), and more. | ❌ | ✅ | +| **Scaling and optimization** | Seamlessly scales up or down based on workload, supporting both horizontal and vertical scaling. With automated backups, replication, and high availability, ClickHouse, it provides users with optimal resource allocation. | ❌ | ✅ | +| **Support services** | Our best-in-class support services and open-source community resources provide coverage for whichever deployment model you choose. | ❌ | ✅ | +| **Database upgrades** | Regular database upgrades are essential to establish a strong security posture and access the latest features and performance improvements. | ❌ | ✅ | +| **Backups** | Backups and restore functionality ensures data durability and supports graceful recovery in the event of outages or other disruptions. | ❌ | ✅ | +| **Compute-compute separation** | Users can scale compute resources independently of storage, so teams and workloads can share the same storage and maintain dedicated compute resources. This ensures that the performance of one workload doesn't interfere with another, enhancing flexibility, performance, and cost-efficiency. | ❌ | ✅ | +| **Managed services** | With a cloud-managed service, teams can focus on business outcomes and accelerate time-to-market without having to worry about the operational overhead of sizing, setup, and maintenance of ClickHouse. | ❌ | ✅ | diff --git a/docs/cloud/onboard/01_discover/02_use_cases/00_overview.md b/docs/cloud/onboard/01_discover/02_use_cases/00_overview.md new file mode 100644 index 00000000000..8bbd22c0914 --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/00_overview.md @@ -0,0 +1,21 @@ +--- +slug: /cloud/get-started/cloud/use-cases/overview +title: 'Building on ClickHouse Cloud' +description: 'Explore ClickHouse Cloud use cases including real-time analytics, observability, data lake & warehouse, and machine learning applications' +keywords: ['use cases', 'Cloud'] +sidebar_label: 'Overview' +--- + +ClickHouse Cloud is suitable for use as both a **primary data store** and as an **analytics +layer**. + +ClickHouse's columnar architecture, vectorized processing, and cloud-native design +make it uniquely suited for analytical workloads that require both speed and scale. +Broadly, the most common use cases for ClickHouse Cloud are: + +| Use case | Description | +|------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Real-Time analytics](/cloud/get-started/cloud/use-cases/real-time-analytics) | ClickHouse Cloud excels at real-time analytics by delivering sub-second query responses on billions of rows through its columnar storage architecture and vectorized execution engine. The platform handles high-throughput data ingestion of millions of events per second while enabling direct queries on raw data without requiring pre-aggregation. Materialized Views provide real-time aggregations and pre-computed results, while approximate functions for quantiles and counts deliver instant insights perfect for interactive dashboards and real-time decision making. | +| [Observability](/cloud/get-started/cloud/use-cases/observability) | ClickHouse Cloud is well suited for observability workloads, featuring specialized engines and functions optimized for time-series data that can ingest and query terabytes of logs, metrics, and traces with ease. Through ClickStack, ClickHouse's comprehensive observability solution, organizations can break down the traditional three silos of logs, metrics, and traces by unifying all observability data in a single platform, enabling correlated analysis and eliminating the complexity of managing separate systems. This unified approach makes it ideal for application performance monitoring, infrastructure monitoring, and security event analysis at enterprise scale, with ClickStack providing the tools and integrations needed for complete observability workflows without data silos. | +| [Data warehousing](/cloud/get-started/cloud/use-cases/data_lake_and_warehouse) | ClickHouse's data warehousing ecosystem connectivity allows users to get set up with a few clicks, and easily get their data into ClickHouse. With excellent support for historical data analysis, data lakes, query federation and JSON as a native data type it enables users to store their data with cost efficiency at scale. | +| [Machine Learning and Artificial Intelligence](/cloud/get-started/cloud/use-cases/AI_ML) | ClickHouse Cloud can be used across the ML value chain, from exploration and preparation through to training, testing and inference. Tools like Clickhouse-local, Clickhouse-server and chdb can be used for data exploration, discovery and transformation, while ClickHouse can be used as a feature store, vector store or MLOps observability store. Furthermore, it enables agentic analytics through built-in tools like fully managed remote MCP server, inline text completion for queries, AI-powered chart configuration and Ask AI in product. | diff --git a/docs/cloud/onboard/01_discover/02_use_cases/01_real-time-analytics.md b/docs/cloud/onboard/01_discover/02_use_cases/01_real-time-analytics.md new file mode 100644 index 00000000000..67aa054a4b0 --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/01_real-time-analytics.md @@ -0,0 +1,160 @@ +--- +slug: /cloud/get-started/cloud/use-cases/real-time-analytics +title: 'Real-time analytics' +description: 'Learn how to build real-time analytics applications with ClickHouse Cloud for instant insights and data-driven decision making' +keywords: ['use cases', 'real-time analytics'] +sidebar_label: 'Real-time analytics' +--- + +import Image from '@theme/IdealImage'; +import rta_0 from '@site/static/images/cloud/onboard/discover/use_cases/0_rta.png'; +import rta_1 from '@site/static/images/cloud/onboard/discover/use_cases/1_rta.png'; +import rta_2 from '@site/static/images/cloud/onboard/discover/use_cases/2_rta.png'; +import rta_3 from '@site/static/images/cloud/onboard/discover/use_cases/3_rta.png'; + + + +## What is real-time analytics? {#what-is-real-time-analytics} + +Real-time analytics refers to data processing that delivers insights to end users +and customers as soon as the data is generated. It differs from traditional or +batch analytics, where data is collected in batches and processed, often a long +time after it was generated. + +Real-time analytics systems are built on top of event streams, which consist of +a series of events ordered in time. An event is something that’s already happened. +It could be the addition of an item to the shopping cart on an e-commerce website, +the emission of a reading from an Internet of Things (IoT) sensor, or a shot on +goal in a football (soccer) match. + +An event (from an imaginary IoT sensor) is shown below, as an example: + +```json +{ + "deviceId": "sensor-001", + "timestamp": "2023-10-05T14:30:00Z", + "eventType": "temperatureAlert", + "data": { + "temperature": 28.5, + "unit": "Celsius", + "thresholdExceeded": true + } +} +``` + +Organizations can discover insights about their customers by aggregating and +analyzing events like this. This has traditionally been done using batch analytics, +and in the next section, we’ll compare batch and real-time analytics. + +## Real-Time analytics vs batch analytics {#real-time-analytics-vs-batch-analytics} + +The diagram below shows what a typical batch analytics system would look like +from the perspective of an individual event: + +batch analytics diagram + +You can see that there’s quite a big gap from when the event happens until we +process and gain some insight from it. Traditionally, this was the only means of +data analysis, and we’d need to create artificial time boundaries to process +the data in batches. For example, we might process all the data collected at the +end of a day. This worked for many use cases, but for others, it’s sub-optimal +because we’re working with stale data, and it doesn’t allow us to react to the +data quickly enough. + +By contrast, in real-time analytics systems, we react to an event as soon as it +happens, as shown in the following diagram: + +Real-time analytics diagram + +We can now derive insights from events almost as soon as they’re generated. But +why is this useful? + +## Benefits of real-time analytics {#benefits-of-real-time-analytics} + +In today's fast-paced world, organizations rely on real-time analytics to stay +agile and responsive to ever-changing conditions. A real-time analytics system +can benefit a business in many ways. + +### Better decision-making {#better-decision-making} + +Decision-making can be improved by having access to actionable insights via +real-time analytics. When business operators can see events as they’re happening, +it makes it much easier to make timely interventions. + +For example, if we make changes to an application and want to know whether it’s +having a detrimental effect on the user experience, we want to know this as +quickly as possible so that we can revert the changes if necessary. With a less +real-time approach, we might have to wait until the next day to do this +analysis, by which type we’ll have a lot of unhappy users. + +### New products and revenue streams {#new-products-and-revenue-streams} + +Real-time analytics can help businesses generate new revenue streams. Organizations +can develop new data-centered products and services that give users access to +analytical querying capabilities. These products are often compelling enough for +users to pay for access. + +In addition, existing applications can be made stickier, increasing user +engagement and retention. This will result in more application use, creating more +revenue for the organization. + +### Improved customer experience {#improved-customer-experience} + +With real-time analytics, businesses can gain instant insights into customer +behavior, preferences, and needs. This lets businesses offer timely assistance, +personalize interactions, and create more engaging experiences that keep +customers returning. + +## Real-time analytics use cases {#real-time-analytics-use-cases} + +The actual value of real-time analytics becomes evident when we consider its +practical applications. Let’s examine some of them. + +### Fraud detection {#fraud-detection} + +Fraud detection is about detecting fraudulent patterns, ranging from fake accounts +to payment fraud. We want to detect this fraud as quickly as possible, flagging +suspicious activities, blocking transactions, and disabling accounts when necessary. + +This use case stretches across industries: healthcare, digital banking, financial +services, retail, and more. + +[Instacart](https://www.instacart.com/) is North America's leading online grocery +company, with millions of active customers and shoppers. It uses ClickHouse as +part of Yoda, its fraud detection platform. In addition to the general types of +fraud described above, it also tries to detect collusion between customers and +shoppers. + +Real-time analytics for fraud detection + +They identified the following characteristics of ClickHouse that enable real-time +fraud detection: + +> ClickHouse supports LSM-tree based MergeTree family engines. +> These are optimized for writing which is suitable for ingesting large amounts +> of data in real-time. + +> ClickHouse is designed and optimized explicitly for analytical queries. This +> fits perfectly with the needs of applications where data is continuously +> analyzed for patterns that might indicate fraud. + +### Time-sensitive decision making {#ftime-sensitive-decision-making} + +Time-sensitive decision-making refers to situations where users or organizations +need to make informed choices quickly based on the most current information +available. Real-time analytics empowers users to make informed choices in +dynamic environments, whether they're traders reacting to market fluctuations, +consumers making purchasing decisions, or professionals adapting to real-time +operational changes. + +Coinhall provides its users with real-time insights into price movements over +time via a candlestick chart, which shows the open, high, low, and close prices +for each trading period. They needed to be able to run these types of queries +quickly and with a large number of concurrent users. + +Real-time analutics for time-sensitive decision making + +> In terms of performance, ClickHouse was the clear winner, executing candlestick queries in 20 milliseconds, compared +> to 400 milliseconds or more for the other databases. It ran latest-price queries in 8 milliseconds, outpacing the +> next-best performance (SingleStore) which came in at 45 milliseconds. Finally, it handled ASOF JOIN queries in +> 50 milliseconds, while Snowflake took 20 minutes and Rockset timed out. diff --git a/docs/cloud/onboard/01_discover/02_use_cases/02_observability.md b/docs/cloud/onboard/01_discover/02_use_cases/02_observability.md new file mode 100644 index 00000000000..97bfb5b263f --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/02_observability.md @@ -0,0 +1,230 @@ +--- +slug: /cloud/get-started/cloud/use-cases/observability +title: 'Observability' +description: 'Use ClickHouse Cloud for observability, monitoring, logging, and system performance analysis in distributed applications' +keywords: ['use cases', 'observability'] +sidebar_label: 'Observability' +--- + + + +Modern software systems are complex. Microservices, cloud infrastructure, and +distributed systems have made it increasingly difficult to understand what's +happening inside our applications. When something goes wrong, teams need to know +where and why quickly. + +This is where observability comes in. It's evolved from simple system monitoring +into a comprehensive approach to understanding system behavior. However, +implementing effective observability isn't straightforward - it requires +understanding technical concepts and organizational challenges. + +## What is Observability? {#what-is-observability} + +Observability is understanding a system's internal state by examining its outputs. +In software systems, this means understanding what's happening inside your +applications and infrastructure through the data they generate. + +This field has evolved significantly and can be understood through two distinct +generations of observability approaches. + +The first generation, often called Observability 1.0, was built around the +traditional "three pillars" approach of metrics, logs, and traces. This approach +required multiple tools and data stores for different types of telemetry. It +often forced engineers to pre-define what they wanted to measure, making it +costly and complex to maintain multiple systems. + +Modern observability, or Observability 2.0, takes a fundamentally different +approach. It's based on collecting wide, structured events for each unit of work +(e.g., an HTTP request and response) in our system. This approach captures +high-cardinality data, such as user IDs, request IDs, Git commit hashes, +instance IDs, Kubernetes pod names, specific route parameters, and vendor +transaction IDs. A rule of thumb is adding a piece of metadata if it could help +us understand how the system behaves. + +This rich data collection enables dynamic slicing and dicing of data without +pre-defining metrics. Teams can derive metrics, traces, and other visualizations +from this base data, allowing them to answer complex questions about system +behavior that weren't anticipated when the instrumentation was first added. + +However, implementing modern observability capabilities presents its challenges. +Organizations need reliable ways to collect, process, and export this rich +telemetry data across diverse systems and technologies. While modern approaches +have evolved beyond traditional boundaries, understanding the fundamental +building blocks of observability remains crucial. + +## The three pillars of observability {#three-pillars-of-observability} + +To better understand how observability has evolved and works in practice, let's +examine the three pillars of observability - logs, metrics, and traces. + +While modern observability has moved beyond treating these as separate concerns, +they remain fundamental concepts for understanding different aspects of system +behavior. + +1. **Logs** - Text-based records of discrete events that occur within a system. +These provide detailed context about specific occurrences, errors, and state changes. +2. **Metrics** - Numerical measurements collected over time. These include counters, +gauges, and histograms that help track system performance, resource usage, and business KPIs. +3. **Traces** - Records that track the journey of requests as they flow through distributed systems. +These help understand the relationships between services and identify performance bottlenecks. + +These pillars enable teams to monitor, troubleshoot, and optimize their systems. +However, the real power comes from understanding how to effectively collect, +analyze, and correlate data across all three pillars to gain meaningful insights +into system behavior. + +## The benefits of observability {#the-benefits-of-observability} + +While the technical aspects of observability - logs, metrics, and traces - are +well understood, the business benefits are equally important to consider. + +In their book ["Observability Engineering"](https://clickhouse.com/engineering-resources/observability#:~:text=Observability%20Engineering) +(O'Reilly, 2022), Charity Majors, Liz Fong-Jones, and George Miranda draw from +industry research and anecdotal feedback to identify four key business benefits +that organizations can expect from implementing proper observability practices. +Let's examine these benefits: + +### Higher incremental revenue {#higher-incremental-revenue} + +The authors note that observability tools that help teams improve uptime and +performance can lead to increased incremental revenue through improved code quality. +This manifests in several ways: + +1. Improved customer experience: Fast problem resolution and prevention of service +degradation leads to higher customer satisfaction and retention +2. Increased system reliability: Better uptime means more successful transactions +and fewer lost business opportunities +3. Enhanced performance: The ability to identify and optimize performance bottlenecks +helps maintain responsive services that keep customers engaged +4. Competitive advantage: Organizations that can maintain high service quality +through comprehensive monitoring and quick issue resolution often gain an edge +over competitors + +### Cost Savings from faster incident response {#cost-savings-from-faster-incident-response} + +One of the most immediate benefits of observability is reduced labor costs +through faster detection and resolution of issues. This comes from: + +* Reduced Mean Time to Detect (MTTD) and Mean Time to Resolve (MTTR) +* Improved query response times, enabling faster investigation +* Quicker identification of performance bottlenecks +* Reduced time spent on-call +* Fewer resources wasted on unnecessary rollbacks + +We see this in practice - [trip.com built their observability system with ClickHouse](trip.com built their observability system with ClickHouse) +and achieved query speeds 4-30x faster than their previous solution, with 90% of +queries completing in under 300ms, enabling rapid issue investigation. + +### Cost savings from incidents avoided {#cost-savings-from-incidents-avoided} + +Observability doesn't just help resolve issues faster - it helps prevent them entirely. +The authors emphasize how teams can prevent critical issues by: + +* Identifying potential problems before they become critical +* Analyzing patterns to prevent recurring issues +* Understanding system behavior under different conditions +* Proactively addressing performance bottlenecks +* Making data-driven decisions about system improvements + +ClickHouse's [own observability platform, LogHouse](https://clickhouse.com/blog/building-a-logging-platform-with-clickhouse-and-saving-millions-over-datadog), +demonstrates this. It enables our core engineers to search historical patterns across all clusters, helping prevent +recurring issues. + +### Cost savings from decreased employee churn {#cost-savings-from-decreased-employee-churn} + +One of the most overlooked benefits is the impact on team satisfaction and retention. +The authors highlight how observability leads to: + +* Improved job satisfaction through better tooling +* Decreased developer burnout from fewer unresolved issues +* Reduced alert fatigue through better signal-to-noise ratio +* Lower on-call stress due to better incident management +* Increased team confidence in system reliability + +We see this in practice - when [Fastly migrated to ClickHouse](https://clickhouse.com/videos/scaling-graphite-with-clickhouse), +their engineers were amazed by the improvement in query performance, noting: + +> "I couldn't believe it. I actually had to go back a couple of times just to +> make sure that I was querying it properly... this is coming back too fast. +> This doesn't make sense." + +As the authors emphasize, while the specific measures of these benefits may vary +depending on the tools and implementation, these fundamental improvements can be +expected across organizations that adopt robust observability practices. The key +is choosing and implementing the right tools effectively to maximize these benefits. + +Achieving these benefits requires overcoming several significant hurdles. Even +organizations that understand the value of observability often find that +implementation presents unexpected complexities and challenges that must be +carefully navigated. + +## Challenges in implementing observability {#challenges-in-implementing-observability} + +Implementing observability within an organization is a transformative step toward +gaining deeper insights into system performance and reliability. However, this +journey is not without its challenges. As organizations strive to harness the +full potential of observability, they encounter various obstacles that can impede +progress. Let’s go through some of them. + +### Data volume and scalability {#data-volume-and-scalability} + +One of the primary hurdles in implementing observability is managing the sheer +volume and scalability of telemetry data generated by modern systems. As +organizations grow, so does the data they need to monitor, necessitating +solutions that efficiently handle large-scale data ingestion and +real-time analytics. + +### Integration with existing systems {#integration-with-existing-systems} + +Integration with existing systems poses another significant challenge. Many +organizations operate in heterogeneous environments with diverse technologies, +making it essential for observability tools to seamlessly integrate with current +infrastructure. Open standards are crucial in facilitating this integration, +ensuring interoperability and reducing the complexity of deploying observability +solutions across varied tech stacks. + +### Skill gaps {#skill-gaps} + +Skill gaps can also impede the successful implementation of observability. The +transition to advanced observability solutions often requires specialized +knowledge of data analytics and specific tools. Teams may need to invest in +training or hiring to bridge these gaps and fully leverage the capabilities of +their observability platforms. + +### Cost management {#cost-management} + +Cost management is critical, as observability solutions can become expensive, +particularly at scale. Organizations must balance the costs of these tools with +the value they provide, seeking cost-effective solutions that offer significant +savings compared to traditional approaches. + +### Data retention and storage {#data-retention-and-storage} + +Data retention and storage management present additional challenges. Deciding +how long to retain observability data without compromising performance or +insights requires careful planning and efficient storage solutions that reduce +storage requirements while maintaining data accessibility. + +### Standardization and vendor lock-in {#standardization-and-vendor-lock-in} + +Ensuring standardization and avoiding vendor lock-in are vital for maintaining +flexibility and adaptability in observability solutions. By adhering to open +standards, organizations can prevent being tied to specific vendors and ensure +their observability stack can evolve with their needs. + +### Security and compliance {#security-and-compliance} + +Security and compliance considerations remain crucial, especially when handling +sensitive data within observability systems. Organizations must ensure that their +observability solutions adhere to relevant regulations and effectively protect +sensitive information. + +These challenges underscore the importance of strategic planning and informed +decision-making in implementing observability solutions that effectively meet +organizational needs. + +To address these challenges, organizations need a well-structured approach to +implementing observability. The standard observability pipeline has evolved to +provide a framework for effectively collecting, processing, and analyzing +telemetry data. One of the earliest and most influential examples of this +evolution comes from Twitter's experience in 2013. diff --git a/docs/cloud/onboard/01_discover/02_use_cases/03_data_warehousing.md b/docs/cloud/onboard/01_discover/02_use_cases/03_data_warehousing.md new file mode 100644 index 00000000000..3b2e325161c --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/03_data_warehousing.md @@ -0,0 +1,119 @@ +--- +slug: /cloud/get-started/cloud/use-cases/data_lake_and_warehouse +title: 'Data Lakehouse' +description: 'Build modern data warehousing architectures with ClickHouse Cloud combining the flexibility of data lakes with database performance' +keywords: ['use cases', 'data lake and warehouse'] +sidebar_label: 'Data warehousing' +--- + +import Image from '@theme/IdealImage'; +import datalakehouse_01 from '@site/static/images/cloud/onboard/discover/use_cases/datalakehouse_01.png'; + + + +The data lakehouse is a convergent architecture that applies database principles +to data lake infrastructure while maintaining the flexibility and scale of cloud storage systems. + +The lakehouse is not just taking a database apart but building database-like +capabilities onto a fundamentally different foundation (cloud object storage) +that focuses on supporting traditional analytics and modern AI/ML workloads in +a unified platform. + +## What are the components of the data lakehouse? {#components-of-the-data-lakehouse} + +The modern data lakehouse architecture represents a convergence of data warehouse +and data lake technologies, combining the best aspects of both approaches. This +architecture comprises several distinct but interconnected layers providing a +flexible, robust data storage, management, and analysis platform. + +Understanding these components is essential for organizations looking to +implement or optimize their data lakehouse strategy. The layered approach allows +for component substitution and independent evolution of each layer, providing +architectural flexibility and future-proofing. + +Let's explore the core building blocks of a typical data lakehouse architecture +and how they interact to create a cohesive data management platform. + +Components of the data lakehouse + +| Component | Description | +|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Data sources** | Lakehouse data sources include operational databases, streaming platforms, IoT devices, application logs, and external providers. | +| **Query engine** | Processes analytical queries against the data stored in the object storage, leveraging the metadata and optimizations provided by the table format layer. Supports SQL and potentially other query languages to analyze large volumes of data efficiently. | +| **Metadata catalog** | The [data catalog](https://clickhouse.com/engineering-resources/data-catalog) acts as a central repository for metadata, storing and managing table definitions and schemas, partitioning information, and access control policies. Enables data discovery, lineage tracking, and governance across the lakehouse. | +| **Table format layer** | The [table format layer](https://clickhouse.com/engineering-resources/open-table-formats) manages the logical organization of data files into tables, providing database-like features such as ACID transactions, schema enforcement and evolution, time travel capabilities, and performance optimizations like data skipping and clustering. | +| **Object storage** | This layer provides scalable, durable, cost-effective storage for all data files and metadata. It handles the physical persistence of data in an open format, enabling direct access from multiple tools and systems. | +| **Client applications** | Various tools and applications that connect to the lakehouse to query data, visualize insights, or build data products. These can include BI tools, data science notebooks, custom applications, and ETL/ELT tools. | + +## What are the benefits of the data lakehouse? {#benefits-of-the-data-lakehouse} + +The data lakehouse architecture offers several significant advantages when compared +directly to both traditional data warehouses and data lakes: + +### Compared to traditional data warehouses {#compared-to-traditional-data-warehouses} + +| # | Benefit | Description | +|---|--------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | **Cost efficiency** | Lakehouses leverage inexpensive object storage rather than proprietary storage formats, significantly reducing storage costs compared to data warehouses that charge premium prices for their integrated storage. | +| 2 | **Component flexibility and interchangeability** | The lakehouse architecture allows organizations to substitute different components. Traditional systems require wholesale replacement when requirements change or technology advances, while lakehouses enable incremental evolution by swapping out individual components like query engines or table formats. This flexibility reduces vendor lock-in and allows organizations to adapt to changing needs without disruptive migrations. | +| 3 | **Open format support** | Lakehouses store data in open file formats like Parquet, allowing direct access from various tools without vendor lock-in, unlike proprietary data warehouse formats that restrict access to their ecosystem. | +| 4 | **AI/ML integration** | Lakehouses provide direct access to data for machine learning frameworks and Python/R libraries, whereas data warehouses typically require extracting data before using it for advanced analytics. | +| 5 | **Independent scaling** | Lakehouses separate storage from compute, allowing each to scale independently based on actual needs, unlike many data warehouses, where they scale together. | + +### Compared to data lakes {#compared-to-data-lakes} + +| # | Benefit | Description | +|---|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | **Query performance** | Lakehouses implement indexing, statistics, and data layout optimizations that enable SQL queries to run at speeds comparable to data warehouses, overcoming the poor performance of raw data lakes. | +| 2 | **Data consistency** | Through ACID transaction support, lakehouses ensure consistency during concurrent operations, solving a major limitation of traditional data lakes, where file conflicts can corrupt data. | +| 3 | **Schema management** | Lakehouses enforce schema validation and track schema evolution, preventing the "data swamp" problem common in data lakes where data becomes unusable due to schema inconsistencies. | +| 4 | **Governance capabilities** | Lakehouses provide fine-grained access control and auditing features at row/column levels, addressing the limited security controls in basic data lakes. | +| 5 | **BI Tool support** | Lakehouses offer SQL interfaces and optimizations that make them compatible with standard BI tools, unlike raw data lakes that require additional processing layers before visualization. | + +## Where does ClickHouse fit in the data lakehouse architecture? {#where-does-clickhouse-fit-in-the-data-lakehouse-architecture} + +ClickHouse is a powerful analytical query engine within the modern data lakehouse +ecosystem. It offers organizations a high-performance option for analyzing data +at scale. ClickHouse is a compelling choice due to its exceptional query speed and +efficiency. + +Within the lakehouse architecture, ClickHouse functions as a specialized +processing layer that can flexibly interact with the underlying data. It can +directly query Parquet files stored in cloud object storage systems like S3, +Azure Blob Storage, or Google Cloud Storage, leveraging its optimized columnar +processing capabilities to deliver rapid results even on massive datasets. +This direct query capability allows organizations to analyze their lake data +without complex data movement or transformation processes. + +ClickHouse integrates with open table formats such as Apache Iceberg, Delta Lake, +or Apache Hudi for more sophisticated data management needs. This integration +enables ClickHouse to take advantage of these formats' advanced features, while +still delivering the exceptional query performance it's known for. Organizations +can integrate these table formats directly or connect through metadata catalogs +like AWS Glue, Unity, or other catalog services. + +By incorporating ClickHouse as a query engine in their lakehouse architecture, +organizations can run lightning-fast analytical queries against their data lake +while maintaining the flexibility and openness that define the lakehouse approach. +This combination delivers the performance characteristics of a specialized +analytical database without sacrificing the core benefits of the lakehouse model, +including component interchangeability, open formats, and unified data management. + +## Hybrid architecture: The best of both worlds {#hybrid-architecture-the-best-of-both-worlds} + +While ClickHouse excels at querying lakehouse components, its highly optimized +storage engine offers an additional advantage. For use cases demanding ultra-low +latency queries - such as real-time dashboards, operational analytics, or +interactive user experiences - organizations can selectively store +performance-critical data directly in ClickHouse's native format. This hybrid +approach delivers the best of both worlds: the unmatched query speed of +ClickHouse's specialized storage for time-sensitive analytics and the flexibility +to query the broader data lakehouse when needed. + +This dual capability allows organizations to implement tiered data strategies +where hot, frequently accessed data resides in ClickHouse's optimized storage +for sub-second query responses, while maintaining seamless access to the complete +data history in the lakehouse. Teams can make architectural decisions based on +performance requirements rather than technical limitations, using ClickHouse as +a lightning-fast analytical database for critical workloads and a flexible query +engine for the broader data ecosystem. diff --git a/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/01_machine_learning.md b/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/01_machine_learning.md new file mode 100644 index 00000000000..8f05907fe07 --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/01_machine_learning.md @@ -0,0 +1,171 @@ +--- +slug: /cloud/get-started/cloud/use-cases/AI_ML +title: 'Machine learning' +description: 'Learn how ClickHouse powers machine learning applications across the ML pipeline.' +keywords: ['use cases', 'Machine Learning', 'Generative AI'] +sidebar_label: 'Machine learning' +--- + +import machine_learning_data_layer from '@site/static/images/cloud/onboard/discover/use_cases/ml_data_layer.png' +import online_feature_store from '@site/static/images/cloud/onboard/discover/use_cases/ml_data_layer.png' +import Image from '@theme/IdealImage'; + +## The machine learning data layer {#machine-learning-data-layer} + +You’ve probably heard the lore that 80% of a machine learning practitioner's time is spent cleaning data. +Regardless of whether this myth holds true or not, what does remain true is that data is at the heart of the machine learning problem, from start to finish. +Whether you’re building RAG pipelines, fine-tuning, training your own model, or evaluating model performance, data is the root of each problem. + +Managing data can be tricky, and as a byproduct, the space has experienced a proliferation of tools that are designed to boost productivity by solving a specific slice of a machine learning data problem. +Oftentimes, this takes shape as a layer of abstraction around a more general-purpose solution with an opinionated interface that, on the surface, makes it easier to apply to the specific sub problem at hand. +In effect, this reduces the flexibility that exists with a general-purpose solution in favor of ease-of-use and simplicity of a specific task. + + + +There are several drawbacks to this approach. +A cascading suite of specialized tools, products, and services, in contrast with a general-purpose solution coupled with supporting application code, presents the risk of greater architectural complexity and data costs than necessary. +It’s easy to accidentally find yourself with an endless list of tools and services, each used for just a single step. + +There are two common dimensions to these risks: + +1. **Learning, maintenance, and switching costs** + +Machine learning architectures can become so cluttered with various tools and components that it creates a fragmented and challenging environment to learn and manage, with increased points of failure and expense creep. + +2. **Data duplication and transfer costs** + +Using several discrete yet overlapping data systems in a machine learning pipeline may introduce an unnecessary, and often costly, overhead of shipping data around from one to another. + +A great illustration of this tradeoff is the vector database. +Vector databases are designed for the hyper-specific machine learning task of storing and searching across vectors. +While this may be the right choice in some architectures, a vector database may be an unnecessary new addition to the tech stack in others, as it is yet another system to integrate with, manage, and ship data to and from. +Most modern general-purpose databases come with vector support out-of-the-box (or through a plugin) and have more extensive and cross-cutting capabilities. +In other words, there may be no need for a net new database to specifically handle vectors in those architectures at all. +The importance boils down to whether the vector-specific convenience features (e.g. inbuilt embedding models) are mission-critical and worth the cost. + +### Data exploration {#data-exploration} + +After defining the machine learning problem, goals, and success criteria, a common first step is to explore the relevant data that will be used for model training and evaluation. + +During this step, data is analyzed to understand its characteristics, distributions, and relationships. +This process of evaluation and understanding is an iterative one, often resulting in a series of ad-hoc queries being executed across datasets, where query responsiveness is critical (along with other factors such as cost-efficiency and accuracy). +As companies store increasing amounts of data to leverage for machine learning purposes, the problem of examining the data you have becomes harder. + +This is because analytics and evaluation queries often become tediously or prohibitively slow at scale with traditional data systems. +Some of the big players impose significantly increased costs to bring down query times, and discourage ad-hoc evaluation by way of charging per query or by number of bytes scanned. +Engineers may resort to pulling subsets of data down to their local machines as a compromise for these limitations. + +ClickHouse, on the other hand, is a real-time data warehouse, so users benefit from industry-leading query speeds for analytical computations. +Further, ClickHouse delivers high performance from the start, and doesn’t gate critical query-accelerating features behind higher pricing tiers. +ClickHouse can also query data directly from object storage or data lakes, with support for common formats such as Iceberg, Delta Lake, and Hudi. +This means that no matter where your data lives, ClickHouse can serve as a unifying access and computation layer for your machine learning workloads. + +ClickHouse also has an extensive suite of pre-built statistical and aggregation functions that scale over petabytes of data, making it easy to write and maintain simple SQL that executes complex computations. +With support for the most granular precision data types and codecs, you don't need to worry about reducing the granularity of your data. + +While users can transform data directly in ClickHouse or prior to insertion using SQL queries, ClickHouse can also be used in programming environments such as Python via [chDB](/chdb). +This allows embedded ClickHouse to be exposed as a Python module and used to transform and manipulate large data frames within notebooks. +Data engineers can therefore perform transformation work to be performed client-side, with results potentially materialized as feature tables in a centralized ClickHouse instance. + +### Data preparation and feature extraction {#data-preparation-and-feature-extraction} + +Data is then prepared: cleaned, transformed, and used to extract the features by which the model will be trained and evaluated. +This component is sometimes called a feature generation or extraction pipeline, and is another slice of the machine learning data layer where new tools are often introduced. +MLOps players like Neptune and Hopsworks provide examples of the host of different data transformation products that are used to orchestrate pipelines like these. +However, because they’re separate tools from the database they’re operating on, they can be brittle, and can cause disruptions that need to be manually rectified. + +In contrast, data transformations are easily accomplished directly in ClickHouse through [materialized views](/materialized-views). +These are automatically triggered when new data is inserted into ClickHouse source tables and are used to easily extract, transform, and modify data as it arrives - eliminating the need to build and monitor bespoke pipelines yourself. +When these transformations require aggregations over a complete dataset that may not fit into memory, leveraging ClickHouse ensures you don’t have to try and retrofit this step to work with data frames on your local machine. +For those datasets that are more convenient to evaluate locally, [ClickHouse local](/operations/utilities/clickhouse-local) is a great alternative, along with [chDB](/chdb), that allow users to leverage ClickHouse with standard Python data libraries like Pandas. + +### Training and evaluation {#training-and-evaluation} + +At this point, features will have been split into training, validation, and test sets. +These data sets are versioned, and then utilized by their respective stages. + +It is common in this phase of the pipeline to introduce yet another specialized tool to the machine learning data layer - the feature store. +A feature store is most commonly a layer of abstraction around a database that provides convenience features specific to managing data for model training, inference, and evaluation. +Examples of these convenience features include versioning, access management, and automatically translating the definition of features to SQL statements. + +For feature stores, ClickHouse can act as a: + +**Data source** - With the ability to query or ingest data in over 70 different file formats, including data lake formats such as Iceberg and Delta Lake, ClickHouse makes an ideal long-term store holding or querying data. +By separating storage and compute using object storage, ClickHouse Cloud additionally allows data to be held indefinitely - with compute scaled down or made completely idle to minimize costs. +Flexible codecs, coupled with column-oriented storage and ordering of data on disk, maximize compression rates, thus minimizing the required storage. +Users can easily combine ClickHouse with data lakes, with built-in functions to query data in place on object storage. + +**Transformation engine** - SQL provides a natural means of declaring data transformations. +When extended with ClickHouse’s analytical and statistical functions, these transformations become succinct and optimized. +As well as applying to either ClickHouse tables, in cases where ClickHouse is used as a data store, table functions allow SQL queries to be written against data stored in formats such as Parquet, on-disk or object storage, or even other data stores such as Postgres and MySQL. +A completely parallelization query execution engine, combined with a column-oriented storage format, allows ClickHouse to perform aggregations over PBs of data in seconds - unlike transformations on in memory data frames, users are not memory-bound. +Furthermore, materialized views allow data to be transformed at insert time, thus overloading compute to data load time from query time. +These views can exploit the same range of analytical and statistical functions ideal for data analysis and summarization. +Should any of ClickHouse’s existing analytical functions be insufficient or custom libraries need to be integrated, users can also utilize User Defined Functions (UDFs). + +#### Offline feature store {#offline-feature-store} + +An offline feature store is used for model training. +This generally means that the features themselves are produced through batch-process data transformation pipelines (as described in the above section), and there are typically no strict latency requirements on the availability of those features. + +With capabilities to read data from multiple sources and apply transformations via SQL queries, the results of these queries can also be persisted in ClickHouse via `INSERT INTO SELECT` statements. +With transformations often grouped by an entity ID and returning a number of columns as results, ClickHouse’s schema inference can automatically detect the required types from these results and produce an appropriate table schema to store them. +Functions for generating random numbers and statistical sampling allow data to be efficiently iterated and scaled at millions or rows per second for feeding to model training pipelines. + +Often, features are represented in tables with a timestamp indicating the value for an entity and feature at a specific point in time. +As described earlier, training pipelines often need the state of features at specific points in time and in groups. ClickHouse’s sparse indices allow fast filtering of data to satisfy point-in-time queries and feature selection filters. While other technologies such as Spark, Redshift, and BigQuery rely on slow stateful windowed approaches to identify the state of features at a specific point in time, ClickHouse supports the ASOF (as-of-this-time) LEFT JOIN query and argMax function. +In addition to simplifying syntax, this approach is highly performant on large datasets through the use of a sort and merge algorithm. +This allows feature groups to be built quickly, reducing data preparation time prior to training. + +#### Online feature store {#online-feature-store} + +Online feature stores are used to store the latest version of features used for inference and are applied in real-time. +This means that these features need to be calculated with minimal latency, as they’re used as part of a real-time machine learning service. + + + +As a real-time analytics database, ClickHouse can serve highly concurrent query workloads at low latency. +While this requires data to be typically denormalized, this aligns with the storage of feature groups used at both training and inference time. +Importantly, ClickHouse is able to deliver this query performance while being subject to high write workloads thanks to its log-structured merge tree. +These properties are required in an online store to keep features up-to-date. +Since the features are already available within the offline store, they can easily be materialized to new tables within either the same ClickHouse cluster or a different instance via existing capabilities, e.g. `remoteSecure`. +Integrations with Kafka, through either an exactly-once Kafka Connect offering or via [ClickPipes](/integrations/clickpipes/kafka) in ClickHouse Cloud, also make consuming streaming data from streaming sources simple and reliable. + +Many modern systems require both offline and online stores, and it is easy to jump to the conclusion that two specialized feature stores are required here. +However, this introduces the additional complexity of keeping both of these stores in sync, which of course also includes the cost of replicating data between them. + +A real-time data warehouse like ClickHouse is a single system that can power both offline and online feature management. +ClickHouse efficiently processes streaming and historical data, and has the unlimited scale, performance, and concurrency needed to be relied upon when serving features for real-time inference and offline training. + +In considering the tradeoffs between using a feature store product in this stage versus leveraging a real-time data warehouse directly, it’s worth emphasizing that convenience features such as versioning can be achieved through age-old database paradigms such as table or schema design. +Other functionality, such as converting feature definitions to SQL statements, may provide greater flexibility as part of the application or business logic, rather than existing in an opinionated layer of abstraction. + +### Inference {#inference} + +Model inference is the process of running a trained model to receive an output. +When inference is triggered by database actions - for instance, inserting a new record, or querying records - the inference step could be managed via bespoke jobs or application code. + +On the other hand, it could be managed in the data layer itself. ClickHouse [User Defined Functions (UDFs)](/sql-reference/functions/udf), give users the ability to invoke a model directly from ClickHouse at insert or query time. +This provides the ability to pass incoming data to a model, receive the output, and store these results along with the ingested data automatically - all without having to spin up other processes or jobs. +This also provides a single interface, SQL, by which to manage this step. + +### Vector store {#vector-store} + +A vector store is a specific type of database that is optimized for storing and retrieving vectors, typically embeddings of a piece of data (such as text or images) that numerically capture their underlying meaning. +Vectors are at the core of today’s generative AI wave and are used in countless applications. + +The primary operation in a vector database is a “similarity search” to find the vectors that are “closest” to one another according to a mathematical measure. +Vector databases have become popular because they employ specific tactics intended to make this examination - vector comparisons - as fast as possible. +These techniques generally mean that they approximate the vector comparisons, instead of comparing the input vector to every vector stored. + +The issue with this new class of tools is that many general-purpose databases, including ClickHouse, provide vector support out-of-the-box, and also often have implementations of those approximate approaches built-in. +ClickHouse, in particular, is designed for high-performance large-scale analytics - allowing you to perform non-approximate vector comparisons very effectively. +This means that you can achieve precise results, rather than having to rely on approximations, all without sacrificing speed. + +### Observability {#observability} + +Once your machine learning application is live, it will generate data, including logs and tracing data, that offer valuable insights into model behavior, performance, and potential areas for improvement. + +SQL-based observability is another key use case for ClickHouse, where ClickHouse has been found to be 10-100x more cost-effective than alternatives. +In fact, many observability products are themselves built with ClickHouse under-the-hood. +With best-in-class ingestion rates and compression ratios, ClickHouse provides cost-efficiency and blazing speed to power machine learning observability at any scale. \ No newline at end of file diff --git a/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/02_agent_facing_analytics.md b/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/02_agent_facing_analytics.md new file mode 100644 index 00000000000..518c78c700e --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/02_agent_facing_analytics.md @@ -0,0 +1,169 @@ +--- +slug: /cloud/get-started/cloud/use-cases/AI_ML/agent_facing_analytics +title: 'Agent facing analytics' +description: 'Build agent-facing analytics systems with ClickHouse Cloud for AI agents and autonomous systems requiring real-time data access' +keywords: ['use cases', 'Machine Learning', 'Generative AI', 'agent facing analytics', 'agents'] +sidebar_label: 'Agent facing analytics' +--- + +import Image from '@theme/IdealImage'; +import ml_ai_05 from '@site/static/images/cloud/onboard/discover/use_cases/ml_ai_05.png'; +import ml_ai_06 from '@site/static/images/cloud/onboard/discover/use_cases/ml_ai_06.png'; +import ml_ai_07 from '@site/static/images/cloud/onboard/discover/use_cases/ml_ai_07.png'; +import ml_ai_08 from '@site/static/images/cloud/onboard/discover/use_cases/ml_ai_08.png'; +import ml_ai_09 from '@site/static/images/cloud/onboard/discover/use_cases/ml_ai_09.png'; + +## Agent-facing analytics concepts {#agent-facing-analytics} + +### What are "agents"? {#agents} + +One can think of AI agents as digital assistants that have evolved beyond +simple task execution (or function calling): they can understand context, +make decisions, and take meaningful actions toward specific goals. They +operate in a "sense-think-act" loop (see ReAct agents), processing various +inputs (text, media, data), analyzing situations, and then doing something +useful with that information. Most importantly, depending on the application +domain, they can theoretically operate at various levels of autonomy, +requiring or not human supervision. + +The game changer here has been the advent of Large Language Models (LLMs). +While we had the notion of AI agents for quite a while, LLMs like the GPT +series have given them a massive upgrade in their ability to "understand" +and communicate. It's as if they've suddenly become more fluent in "human" +aka. able to grasp requests and respond with relevant contextual information +drawn from the model's training. + +### AI agents superpowers: “Tools” {#tools} + +These agents really shine through their access to “tools”. Tools enhance AI agents +by giving them abilities to perform tasks. Rather than just being conversational +interfaces, they can now get things done whether it’s crunching numbers, searching +for information, or managing customer communications. Think of it as the difference +between having someone who can describe how to solve a problem and someone who +can actually solve it. + +For example, ChatGPT is now shipped by default with a search tool. This +integration with search providers allows the model to pull current information +from the web during conversations. This means it can fact-check responses, access +recent events and data, and provide up-to-date information rather than relying +solely on its training data. + +Agents equipped with tools + +Tools can also be used to simplify the implementation of Retrieval-Augmented +Generation (RAG) pipelines. Instead of relying only on what an AI model +learned during training, RAG lets the model pull in relevant information +before formulating a response. Here's an example: Using an AI assistant to +help with customer support (e.g. Salesforce AgentForce, ServiceNow AI +Agents). Without RAG, it would only use its general training to answer +questions. But with RAG, when a customer asks about the latest product +feature, the system retrieves the most recent documentation, release notes, +and historical support tickets before crafting its response. This means that +answers are now grounded in the latest information available to the AI +model. + +### Reasoning models {#reasoning-models} + +Another development in the AI space, and perhaps one of the most +interesting, is the emergence of reasoning models. Systems like OpenAI o1, +Anthropic Claude, or DeepSeek-R1 take a more methodical approach by +introducing a "thinking" step before responding to a prompt. Instead of +generating the answer straightaway, reasoning models use prompting +techniques like Chain-of-Thought (CoT) to analyze problems from multiple +angles, break them down into steps, and use the tools available to them to +gather contextual information when needed. + +This represents a shift toward more capable systems that can handle more +complex tasks through a combination of reasoning and practical tools. One of +the latest examples in this area is the introduction of OpenAI's deep +research, an agent that can autonomously conduct complex multi-step research +tasks online. It processes and synthesizes information from various sources, +including text, images, and PDFs, to generate comprehensive reports within five +to thirty minutes, a task that would traditionally take a human several hours. + +Reasoning models + +## Real-time analytics for AI agents {#real-time-analytics-for-ai-agents} + +Let's take the case of an agentic AI assistant with access to a +real-time analytics database containing the company's CRM data. When a user asks +about the latest (up-to-the-minute) sales trends, the AI assistant queries the +connected data source. It iteratively analyzes the data to identify meaningful +patterns and trends, such as month-over-month growth, seasonal variations, or +emerging product categories. Finally, it generates a natural language response +explaining key findings, often with supporting visualizations. When the main +interface is chat-based like in this case, performance matters since these +iterative explorations trigger a series of queries that can scan large amounts of +data to extract relevant insights. + +Some properties make real-time databases especially suitable for such +workloads. For example, real-time analytics databases are designed to work +with near real-time data, allowing them to process and deliver insights +almost immediately as new data arrives. This is crucial for AI agents, as +they can require up-to-date information to make (or help make) timely and +relevant decisions. + +The core analytical capabilities are also important. Real-time analytics +databases shine in performing complex aggregations and pattern detection +across large datasets. Unlike operational databases focusing primarily on +raw data storage or retrieval, these systems are optimized for analyzing +vast amounts of information. This makes them particularly well-suited for AI +agents that need to uncover trends, detect anomalies, and derive actionable +insights. + +Real-time analytics databases are also expected to deliver fast +performance for interactive querying, essential for chat-based interaction +and high-frequency explorative workloads. They ensure consistent performance +even with large data volumes and high query concurrency, enabling responsive +dialogues and a smoother user experience. + +Finally, real-time analytics databases often serve as the ultimate "data +sinks" effectively consolidating valuable domain-specific data in a single +location. By co-locating essential data across different sources and formats +under the same tent, these databases ensure that AI agents have access to a +unified view of the domain information, decoupled from operational systems. + +Classic real-time analytics + +Agent real-time analytics + +These properties already empower real-time databases to play a vital role +in serving AI data retrieval use cases at scale (e.g. OpenAI's acquisition +of Rockset). They can also enable AI agents to provide fast data-driven +responses while offloading the heavy computational work. + +It positions the real-time analytics database as a preferred "context +provider" for AI agents when it comes to insights. + +## AI agents as an emerging user persona {#ai-agents-as-an-emerging-user-persona} + +A useful way to think about AI agents leveraging real-time analytics databases +is to perceive them as a new category of users, or in product manager speak: +a user persona. + +Agents as an emerging user persona + +From the database perspective, we can expect a potentially unlimited number of +AI agents, concurrently running a large number of queries on behalf of users, +or in autonomy, to perform investigations, refine iterative research and insights, +and execute tasks. + +Over the years, real-time databases have had the time to adapt to human +interactive users, directly connected to the system or via a middleware +application layer. Classic personas examples include database administrators, +business analysts, data scientists, or software developers building applications +on top of the database. The industry has progressively learned their usage +patterns and requirements and organically, provided the interfaces, the operators, +the UIs, the formats, the clients, and the performance to satisfy their various +use cases. + +The question now becomes, are we ready to accommodate the AI agent's workloads? +What specific features do we need to re-think or create from scratch for these +usage patterns? + +ClickHouse is rapidly providing answers to some of these questions through a host +of features aimed at providing a feature-complete AI experience. + +## ClickHouse.ai {#clickhouse-ai} + +For more information about features coming soon to ClickHouse Cloud, see [ClickHouse.ai](https://clickhouse.com/clickhouse-ai/). diff --git a/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/_category_.json b/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/_category_.json new file mode 100644 index 00000000000..01f4f00d897 --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "ML/AI", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/onboard/01_discover/02_use_cases/_category_.json b/docs/cloud/onboard/01_discover/02_use_cases/_category_.json new file mode 100644 index 00000000000..70c6591bd01 --- /dev/null +++ b/docs/cloud/onboard/01_discover/02_use_cases/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Use cases", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/integrations/migration/overview.md b/docs/cloud/onboard/02_migrate/01_migration_guides/01_overview.md similarity index 50% rename from docs/integrations/migration/overview.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/01_overview.md index 46457d3c294..652a40c74f8 100644 --- a/docs/integrations/migration/overview.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/01_overview.md @@ -25,9 +25,9 @@ description: 'Page describing the options available for migrating data into Clic There are several options for migrating data into ClickHouse Cloud, depending on where your data resides now: -- [Self-managed to Cloud](./clickhouse-to-cloud.md): use the `remoteSecure` function to transfer data -- [Another DBMS](./clickhouse-local-etl.md): use the [clickhouse-local] ETL tool along with the appropriate ClickHouse table function for your current DBMS -- [Anywhere!](./etl-tool-to-clickhouse.md): use one of the many popular ETL/ELT tools that connect to all kinds of different data sources -- [Object Storage](./object-storage-to-clickhouse.md): easily insert data from S3 into ClickHouse +- [Self-managed to Cloud](/cloud/migration/clickhouse-to-cloud): use the `remoteSecure` function to transfer data +- [Another DBMS](/cloud/migration/clickhouse-local): use the [clickhouse-local] ETL tool along with the appropriate ClickHouse table function for your current DBMS +- [Anywhere!](/cloud/migration/etl-tool-to-clickhouse): use one of the many popular ETL/ELT tools that connect to all kinds of different data sources +- [Object Storage](/integrations/migration/object-storage-to-clickhouse): easily insert data from S3 into ClickHouse -In the example [Migrate from Redshift](/integrations/data-ingestion/redshift/index.md), we present three different ways to migrate data to ClickHouse. +In the example [Migrate from Redshift](/migrations/redshift/migration-guide), we present three different ways to migrate data to ClickHouse. diff --git a/docs/migrations/postgres/overview.md b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/01_overview.md similarity index 97% rename from docs/migrations/postgres/overview.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/01_overview.md index ca1d195b914..b8be25dcc58 100644 --- a/docs/migrations/postgres/overview.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/01_overview.md @@ -1,10 +1,13 @@ --- slug: /migrations/postgresql/overview -title: 'Migrating from PostgreSQL to ClickHouse' +title: 'Comparing PostgreSQL and ClickHouse' description: 'A guide to migrating from PostgreSQL to ClickHouse' keywords: ['postgres', 'postgresql', 'migrate', 'migration'] +sidebar_label: 'Overview' --- +# Comparing ClickHouse and PostgreSQL + ## Why use ClickHouse over Postgres? {#why-use-clickhouse-over-postgres} TLDR: Because ClickHouse is designed for fast analytics, specifically `GROUP BY` queries, as an OLAP database whereas Postgres is an OLTP database designed for transactional workloads. diff --git a/docs/migrations/postgres/appendix.md b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/appendix.md similarity index 100% rename from docs/migrations/postgres/appendix.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/appendix.md diff --git a/docs/migrations/postgres/index.md b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/index.md similarity index 90% rename from docs/migrations/postgres/index.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/index.md index e4052fd1ab9..35837f4d34c 100644 --- a/docs/migrations/postgres/index.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/index.md @@ -8,7 +8,7 @@ description: 'Landing page for the PostgreSQL migrations section' | Page | Description | |----------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Overview](./overview.md) | Introduction page for this section | +| [Overview](/migrations/postgresql/overview) | Introduction page for this section | | [Connecting to PostgreSQL](/integrations/postgresql/connecting-to-postgresql) | This page covers the following options for integrating PostgreSQL with ClickHouse: ClickPipes, PeerDB, PostgreSQL table engine, MaterializedPostgreSQL database engine. | | [Migrating data](/migrations/postgresql/dataset) | Part 1 of a guide on migrating from PostgreSQL to ClickHouse. Using a practical example, it demonstrates how to efficiently carry out the migration with a real-time replication (CDC) approach. Many of the concepts covered are also applicable to manual bulk data transfers from PostgreSQL to ClickHouse. | |[Rewriting PostgreSQL Queries](/migrations/postgresql/rewriting-queries)|Part 2 of a guide on migrating from PostgreSQL to ClickHouse. Using a practical example, it demonstrates how to efficiently carry out the migration with a real-time replication (CDC) approach. Many of the concepts covered are also applicable to manual bulk data transfers from PostgreSQL to ClickHouse.| diff --git a/docs/migrations/postgres/dataset.md b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/01_migration_guide_part1.md similarity index 99% rename from docs/migrations/postgres/dataset.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/01_migration_guide_part1.md index 2574252e1da..a2260255e9b 100644 --- a/docs/migrations/postgres/dataset.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/01_migration_guide_part1.md @@ -4,6 +4,7 @@ title: 'Migrating data' description: 'Dataset example to migrate from PostgreSQL to ClickHouse' keywords: ['Postgres'] show_related_blogs: true +sidebar_label: 'Part 1' --- import postgres_stackoverflow_schema from '@site/static/images/migrations/postgres-stackoverflow-schema.png'; @@ -177,4 +178,4 @@ INSERT INTO stackoverflow.posts SELECT * FROM postgresql('', 'postgres', ' > A possible method to detect UPDATE operations when using query replication is using the [`XMIN` system column](https://www.postgresql.org/docs/9.1/ddl-system-columns.html) (transaction IDs) as a watermark - a change in this column is indicative of a change and therefore can be applied to the destination table. Users employing this approach should be aware that `XMIN` values can wrap around and comparisons require a full table scan, making tracking changes more complex. -[Click here for Part 2](./rewriting-queries.md) +[Click here for Part 2](/migrations/postgresql/rewriting-queries) diff --git a/docs/migrations/postgres/rewriting-queries.md b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/02_migration_guide_part2.md similarity index 99% rename from docs/migrations/postgres/rewriting-queries.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/02_migration_guide_part2.md index 451d1b37d9a..8866fb91cd9 100644 --- a/docs/migrations/postgres/rewriting-queries.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/02_migration_guide_part2.md @@ -3,6 +3,7 @@ slug: /migrations/postgresql/rewriting-queries title: 'Rewriting PostgreSQL Queries' keywords: ['postgres', 'postgresql', 'rewriting queries'] description: 'Part 2 of a guide on migrating from PostgreSQL to ClickHouse' +sidebar_label: 'Part 2' --- > This is **Part 2** of a guide on migrating from PostgreSQL to ClickHouse. Using a practical example, it demonstrates how to efficiently carry out the migration with a real-time replication (CDC) approach. Many of the concepts covered are also applicable to manual bulk data transfers from PostgreSQL to ClickHouse. @@ -269,4 +270,4 @@ LIMIT 5; Time: 116750.131 ms (01:56.750) ``` -[Click here for Part 3](./data-modeling-techniques.md) +[Click here for Part 3](/migrations/postgresql/data-modeling-techniques) diff --git a/docs/migrations/postgres/data-modeling-techniques.md b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/03_migration_guide_part3.md similarity index 99% rename from docs/migrations/postgres/data-modeling-techniques.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/03_migration_guide_part3.md index f864bd8fb3e..db4468289d8 100644 --- a/docs/migrations/postgres/data-modeling-techniques.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/03_migration_guide_part3.md @@ -1,9 +1,10 @@ --- slug: /migrations/postgresql/data-modeling-techniques title: 'Data modeling techniques' -description: 'Data modeling for migrating from PostgreSQL to ClickHouse' +description: 'Part 3 of a guide on migrating from PostgreSQL to ClickHouse' keywords: ['postgres', 'postgresql'] show_related_blogs: true +sidebar_label: 'Part 3' --- import postgres_b_tree from '@site/static/images/migrations/postgres-b-tree.png'; diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/_category_.json b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/_category_.json new file mode 100644 index 00000000000..ad514aeb890 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/migration_guide/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Migration guide", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/migrations/bigquery/equivalent-concepts.md b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/01_overview.md similarity index 98% rename from docs/migrations/bigquery/equivalent-concepts.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/01_overview.md index ee330a0610c..729112ee81e 100644 --- a/docs/migrations/bigquery/equivalent-concepts.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/01_overview.md @@ -4,12 +4,13 @@ slug: /migrations/bigquery/biquery-vs-clickhouse-cloud description: 'How BigQuery differs from ClickHouse Cloud' keywords: ['BigQuery'] show_related_blogs: true +sidebar_label: 'Overview' --- import bigquery_1 from '@site/static/images/migrations/bigquery-1.png'; import Image from '@theme/IdealImage'; -# BigQuery vs ClickHouse Cloud: equivalent and different concepts +# Comparing ClickHouse Cloud and BigQuery ## Resource organization {#resource-organization} @@ -21,7 +22,7 @@ The way resources are organized in ClickHouse Cloud is similar to [BigQuery's re Similar to BigQuery, organizations are the root nodes in the ClickHouse cloud resource hierarchy. The first user you set up in your ClickHouse Cloud account is automatically assigned to an organization owned by the user. The user may invite additional users to the organization. -### BigQuery projects vs ClickHouse Cloud services {#bigquery-projects-vs-clickhouse-cloud-services} +### BigQuery Projects vs ClickHouse Cloud Services {#bigquery-projects-vs-clickhouse-cloud-services} Within organizations, you can create services loosely equivalent to BigQuery projects because stored data in ClickHouse Cloud is associated with a service. There are [several service types available](/cloud/manage/cloud-tiers) in ClickHouse Cloud. Each ClickHouse Cloud service is deployed in a specific region and includes: @@ -29,15 +30,15 @@ Within organizations, you can create services loosely equivalent to BigQuery pro 2. An object storage folder where the service stores all the data. 3. An endpoint (or multiple endpoints created via ClickHouse Cloud UI console) - a service URL that you use to connect to the service (for example, `https://dv2fzne24g.us-east-1.aws.clickhouse.cloud:8443`) -### BigQuery datasets vs ClickHouse Cloud databases {#bigquery-datasets-vs-clickhouse-cloud-databases} +### BigQuery Datasets vs ClickHouse Cloud Databases {#bigquery-datasets-vs-clickhouse-cloud-databases} ClickHouse logically groups tables into databases. Like BigQuery datasets, ClickHouse databases are logical containers that organize and control access to table data. -### BigQuery folders {#bigquery-folders} +### BigQuery Folders {#bigquery-folders} ClickHouse Cloud currently has no concept equivalent to BigQuery folders. -### BigQuery slot reservations and quotas {#bigquery-slot-reservations-and-quotas} +### BigQuery Slot reservations and Quotas {#bigquery-slot-reservations-and-quotas} Like BigQuery slot reservations, you can [configure vertical and horizontal autoscaling](/manage/scaling#configuring-vertical-auto-scaling) in ClickHouse Cloud. For vertical autoscaling, you can set the minimum and maximum size for the memory and CPU cores of the compute nodes for a service. The service will then scale as needed within those bounds. These settings are also available during the initial service creation flow. Each compute node in the service has the same size. You can change the number of compute nodes within a service with [horizontal scaling](/manage/scaling#manual-horizontal-scaling). @@ -78,7 +79,7 @@ When presented with multiple options for ClickHouse types, consider the actual r ## Query acceleration techniques {#query-acceleration-techniques} -### Primary and foreign keys and primary index {#primary-and-foreign-keys-and-primary-index} +### Primary and Foreign keys and Primary index {#primary-and-foreign-keys-and-primary-index} In BigQuery, a table can have [primary key and foreign key constraints](https://cloud.google.com/bigquery/docs/information-schema-table-constraints). Typically, primary and foreign keys are used in relational databases to ensure data integrity. A primary key value is normally unique for each row and is not `NULL`. Each foreign key value in a row must be present in the primary key column of the primary key table or be `NULL`. In BigQuery, these constraints are not enforced, but the query optimizer may use this information to optimize queries better. diff --git a/docs/migrations/bigquery/migrating-to-clickhouse-cloud.md b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/02_migrating-to-clickhouse-cloud.md similarity index 99% rename from docs/migrations/bigquery/migrating-to-clickhouse-cloud.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/02_migrating-to-clickhouse-cloud.md index 44f8c8c7d20..0118a912fec 100644 --- a/docs/migrations/bigquery/migrating-to-clickhouse-cloud.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/02_migrating-to-clickhouse-cloud.md @@ -4,6 +4,7 @@ slug: /migrations/bigquery/migrating-to-clickhouse-cloud description: 'How to migrate your data from BigQuery to ClickHouse Cloud' keywords: ['BigQuery'] show_related_blogs: true +sidebar_label: 'Migration guide' --- import bigquery_2 from '@site/static/images/migrations/bigquery-2.png'; diff --git a/docs/migrations/bigquery/loading-data.md b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/03_loading-data.md similarity index 96% rename from docs/migrations/bigquery/loading-data.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/03_loading-data.md index c39430ec10d..add277a4004 100644 --- a/docs/migrations/bigquery/loading-data.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/03_loading-data.md @@ -24,7 +24,9 @@ Exporting data from BigQuery to ClickHouse is dependent on the size of your data | [contracts](https://github.com/ClickHouse/examples/blob/main/ethereum/schemas/contracts.md) | 57,225,837 | 350 | 45.35GB | 16 sec | 1 hr 51 min | 39.4 secs | | Total | 8.26 billion | 23,577 | 3.982TB | 8 min 3 sec | \> 6 days 5 hrs | 53 mins 45 secs | -## 1. Export table data to GCS {#1-export-table-data-to-gcs} + + +## Export table data to GCS {#1-export-table-data-to-gcs} In this step, we utilize the [BigQuery SQL workspace](https://cloud.google.com/bigquery/docs/bigquery-web-ui) to execute our SQL commands. Below, we export a BigQuery table named `mytable` to a GCS bucket using the [`EXPORT DATA`](https://cloud.google.com/bigquery/docs/reference/standard-sql/other-statements) statement. @@ -60,7 +62,7 @@ This approach has a number of advantages: - Exports produce multiple files automatically, limiting each to a maximum of 1GB of table data. This is beneficial to ClickHouse since it allows imports to be parallelized. - Parquet, as a column-oriented format, represents a better interchange format since it is inherently compressed and faster for BigQuery to export and ClickHouse to query -## 2. Importing data into ClickHouse from GCS {#2-importing-data-into-clickhouse-from-gcs} +## Importing data into ClickHouse from GCS {#2-importing-data-into-clickhouse-from-gcs} Once the export is complete, we can import this data into a ClickHouse table. You can use the [ClickHouse SQL console](/integrations/sql-clients/sql-console) or [`clickhouse-client`](/interfaces/cli) to execute the commands below. @@ -111,7 +113,7 @@ In the above query, we use the [`ifNull` function](/sql-reference/functions/func Alternatively, you can `SET input_format_null_as_default=1` and any missing or NULL values will be replaced by default values for their respective columns, if those defaults are specified. ::: -## 3. Testing successful data export {#3-testing-successful-data-export} +## Testing successful data export {#3-testing-successful-data-export} To test whether your data was properly inserted, simply run a `SELECT` query on your new table: @@ -121,6 +123,8 @@ SELECT * FROM mytable LIMIT 10; To export more BigQuery tables, simply redo the steps above for each additional table. + + ## Further reading and support {#further-reading-and-support} In addition to this guide, we also recommend reading our blog post that shows [how to use ClickHouse to speed up BigQuery and how to handle incremental imports](https://clickhouse.com/blog/clickhouse-bigquery-migrating-data-for-realtime-queries). diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/_04_sql_translation_reference.md b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/_04_sql_translation_reference.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/migrations/bigquery/index.md b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/index.md similarity index 52% rename from docs/migrations/bigquery/index.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/index.md index fdb90ce1ab8..9b793545e38 100644 --- a/docs/migrations/bigquery/index.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/03_bigquery/index.md @@ -11,6 +11,6 @@ In this section of the docs, learn more about the similarities and differences b | Page | Description | |-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| [BigQuery vs ClickHouse Cloud](./equivalent-concepts.md) | The way resources are organized in ClickHouse Cloud is similar to BigQuery's resource hierarchy. We describe the specific differences in this article. | -| [Migrating from BigQuery to ClickHouse Cloud](./migrating-to-clickhouse-cloud.md) | Learn about why you might want to migrate from BigQuery to ClickHouse Cloud. | -| [Loading Data](./loading-data.md) | A guide showing you how to migrate data from BigQuery to ClickHouse. | +| [BigQuery vs ClickHouse Cloud](/migrations/bigquery/biquery-vs-clickhouse-cloud) | The way resources are organized in ClickHouse Cloud is similar to BigQuery's resource hierarchy. We describe the specific differences in this article. | +| [Migrating from BigQuery to ClickHouse Cloud](/migrations/bigquery/migrating-to-clickhouse-cloud) | Learn about why you might want to migrate from BigQuery to ClickHouse Cloud. | +| [Loading Data](/migrations/bigquery/loading-data) | A guide showing you how to migrate data from BigQuery to ClickHouse. | diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/01_overview.md b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/01_overview.md new file mode 100644 index 00000000000..6139ff66887 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/01_overview.md @@ -0,0 +1,184 @@ +--- +sidebar_label: 'Overview' +slug: /migrations/snowflake-overview +description: 'Migrating from Snowflake to ClickHouse' +keywords: ['Snowflake'] +title: 'Migrate from Snowflake to ClickHouse' +show_related_blogs: true +--- + +import snowflake_architecture from '@site/static/images/cloud/onboard/discover/use_cases/snowflake_architecture.png'; +import cloud_architecture from '@site/static/images/cloud/onboard/discover/use_cases/cloud_architecture.png'; +import Image from '@theme/IdealImage'; + +# Snowflake to ClickHouse migration + +> This document provides an introduction to migrating data from Snowflake to ClickHouse. + +Snowflake is a cloud data warehouse primarily focused on migrating legacy on-premise +data warehousing workloads to the cloud. It is well-optimized for executing +long-running reports at scale. As datasets migrate to the cloud, data owners start +thinking about how else they can extract value from this data, including using +these datasets to power real-time applications for internal and external use cases. +When this happens, they often realize they need a database optimized for +powering real-time analytics, like ClickHouse. + +## Comparison {#comparison} + +In this section, we'll compare the key features of ClickHouse and Snowflake. + +### Similarities {#similarities} + +Snowflake is a cloud-based data warehousing platform that provides a scalable +and efficient solution for storing, processing, and analyzing large amounts of +data. +Like ClickHouse, Snowflake is not built on existing technologies but relies +on its own SQL query engine and custom architecture. + +Snowflake’s architecture is described as a hybrid between a shared-storage (shared-disk) +architecture and a shared-nothing architecture. A shared-storage architecture is +one where data is both accessible from all compute nodes using object +stores such as S3. A shared-nothing architecture is one where each compute node +stores a portion of the entire data set locally to respond to queries. This, in +theory, delivers the best of both models: the simplicity of a shared-disk +architecture and the scalability of a shared-nothing architecture. + +This design fundamentally relies on object storage as the primary storage medium, +which scales almost infinitely under concurrent access while providing high +resilience and scalable throughput guarantees. + +The image below from [docs.snowflake.com](https://docs.snowflake.com/en/user-guide/intro-key-concepts) +shows this architecture: + +Snowflake architecture + +Conversely, as an open-source and cloud-hosted product, ClickHouse can be deployed +in both shared-disk and shared-nothing architectures. The latter is typical for +self-managed deployments. While allowing for CPU and memory to be easily scaled, +shared-nothing configurations introduce classic data management challenges and +overhead of data replication, especially during membership changes. + +For this reason, ClickHouse Cloud utilizes a shared-storage architecture that is +conceptually similar to Snowflake. Data is stored once in an object store +(single copy), such as S3 or GCS, providing virtually infinite storage with +strong redundancy guarantees. Each node has access to this single copy of the +data as well as its own local SSDs for cache purposes. Nodes can, in turn, be +scaled to provide additional CPU and memory resources as required. Like Snowflake, +S3’s scalability properties address the classic limitation of shared-disk +architectures (disk I/O and network bottlenecks) by ensuring the I/O throughput +available to current nodes in a cluster is not impacted as additional nodes are +added. + +ClickHouse Cloud architecture + +### Differences {#differences} + +Aside from the underlying storage formats and query engines, these architectures +differ in a few subtle ways: + +* Compute resources in Snowflake are provided through a concept of [warehouses](https://docs.snowflake.com/en/user-guide/warehouses). + These consist of a number of nodes, each of a set size. While Snowflake + doesn't publish the specific architecture of their warehouses, it is + [generally understood](https://select.dev/posts/snowflake-warehouse-sizing) + that each node consists of 8 vCPUs, 16GiB, and 200GB of local storage (for cache). + The number of nodes depends on a t-shirt size, e.g. an x-small has one node, + a small 2, medium 4, large 8, etc. These warehouses are independent of the data + and can be used to query any database residing on object storage. When idle + and not subjected to query load, warehouses are paused - resuming when a query + is received. While storage costs are always reflected in billing, warehouses + are only charged when active. + +* ClickHouse Cloud utilizes a similar principle of nodes with local cache + storage. Rather than t-shirt sizes, users deploy a service with a total + amount of compute and available RAM. This, in turn, transparently + auto-scales (within defined limits) based on the query load - either + vertically by increasing (or decreasing) the resources for each node or + horizontally by raising/lowering the total number of nodes. ClickHouse + Cloud nodes currently have a 1 CPU-to-memory ratio, unlike Snowflake's 1. + While a looser coupling is possible, services are currently coupled to the + data, unlike Snowflake warehouses. Nodes will also pause if idle and + resume if subjected to queries. Users can also manually resize services if + needed. + +* ClickHouse Cloud's query cache is currently node specific, unlike + Snowflake's, which is delivered at a service layer independent of the + warehouse. Based on benchmarks, ClickHouse Cloud's node cache outperforms + Snowflake's. + +* Snowflake and ClickHouse Cloud take different approaches to scaling to + increase query concurrency. Snowflake addresses this through a feature + known as [multi-cluster warehouses](https://docs.snowflake.com/en/user-guide/warehouses-multicluster#benefits-of-multi-cluster-warehouses). + This feature allows users to add clusters to a warehouse. While this offers no + improvement to query latency, it does provide additional parallelization and + allows higher query concurrency. ClickHouse achieves this by adding more memory + and CPU to a service through vertical or horizontal scaling. We do not explore the + capabilities of these services to scale to higher concurrency in this blog, + focusing instead on latency, but acknowledge that this work should be done + for a complete comparison. However, we would expect ClickHouse to perform + well in any concurrency test, with Snowflake explicitly limiting the number + of concurrent queries allowed for a [warehouse to 8 by default](https://docs.snowflake.com/en/sql-reference/parameters#max-concurrency-level). + In comparison, ClickHouse Cloud allows up to 1000 queries to be executed per + node. + +* Snowflake's ability to switch compute size on a dataset, coupled with fast + resume times for warehouses, makes it an excellent experience for ad hoc + querying. For data warehouse and data lake use cases, this provides an + advantage over other systems. + +### Real-time analytics {#real-time-analytics} + +Based on public [benchmark](https://benchmark.clickhouse.com/#system=+%E2%98%81w|%EF%B8%8Fr|C%20c|nfe&type=-&machine=-ca2|gl|6ax|6ale|3al&cluster_size=-&opensource=-&tuned=+n&metric=hot&queries=-) data, +ClickHouse outperforms Snowflake for real-time analytics applications in the following areas: + +* **Query latency**: Snowflake queries have a higher query latency even + when clustering is applied to tables to optimize performance. In our + testing, Snowflake requires over twice the compute to achieve equivalent + ClickHouse performance on queries where a filter is applied that is part + of the Snowflake clustering key or ClickHouse primary key. While + Snowflake's [persistent query cache](https://docs.snowflake.com/en/user-guide/querying-persisted-results) + offsets some of these latency challenges, this is ineffective in cases + where the filter criteria are more diverse. This query cache effectiveness + can be further impacted by changes to the underlying data, with cache + entries invalidated when the table changes. While this is not the case in + the benchmark for our application, a real deployment would require the new, + more recent data to be inserted. Note that ClickHouse's query cache is + node specific and not [transactionally consistent](https://clickhouse.com/blog/introduction-to-the-clickhouse-query-cache-and-design), + making it [better suited ](https://clickhouse.com/blog/introduction-to-the-clickhouse-query-cache-and-design) + to real-time analytics. Users also have granular control over its use + with the ability to control its use on a [per-query basis](/operations/settings/settings#use_query_cache), + its [precise size](/operations/settings/settings#query_cache_max_size_in_bytes), + whether a [query is cached](/operations/settings/settings#enable_writes_to_query_cache) + (limits on duration or required number of executions), and whether it is + only [passively used](https://clickhouse.com/blog/introduction-to-the-clickhouse-query-cache-and-design#using-logs-and-settings). + +* **Lower cost**: Snowflake warehouses can be configured to suspend after + a period of query inactivity. Once suspended, charges are not incurred. + Practically, this inactivity check can [only be lowered to 60s](https://docs.snowflake.com/en/sql-reference/sql/alter-warehouse). + Warehouses will automatically resume, within several seconds, once a query + is received. With Snowflake only charging for resources when a warehouse + is under use, this behavior caters to workloads that often sit idle, like + ad-hoc querying. + + However, many real-time analytics workloads require ongoing real-time data + ingestion and frequent querying that doesn't benefit from idling (like + customer-facing dashboards). This means warehouses must often be fully + active and incurring charges. This negates the cost-benefit of idling as + well as any performance advantage that may be associated with Snowflake's + ability to resume a responsive state faster than alternatives. This active + state requirement, when combined with ClickHouse Cloud's lower per-second + cost for an active state, results in ClickHouse Cloud offering a + significantly lower total cost for these kinds of workloads. + +* **Predictable pricing of features:** Features such as materialized views + and clustering (equivalent to ClickHouse's ORDER BY) are required to reach + the highest levels of performance in real-time analytics use cases. These + features incur additional charges in Snowflake, requiring not only a + higher tier, which increases costs per credit by 1.5x, but also + unpredictable background costs. For instance, materialized views incur a + background maintenance cost, as does clustering, which is hard to predict + prior to use. In contrast, these features incur no additional cost in + ClickHouse Cloud, except additional CPU and memory usage at insert time, + typically negligible outside of high insert workload use cases. We have + observed in our benchmark that these differences, along with lower query + latencies and higher compression, result in significantly lower costs with + ClickHouse. diff --git a/docs/migrations/snowflake.md b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/02_migration_guide.md similarity index 77% rename from docs/migrations/snowflake.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/02_migration_guide.md index 38d3b8dfac1..468a8b6193b 100644 --- a/docs/migrations/snowflake.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/02_migration_guide.md @@ -1,23 +1,27 @@ --- -sidebar_label: 'Snowflake' -sidebar_position: 20 +sidebar_label: 'Migration guide' slug: /migrations/snowflake description: 'Migrating from Snowflake to ClickHouse' keywords: ['Snowflake'] title: 'Migrating from Snowflake to ClickHouse' -show_related_blogs: true +show_related_blogs: false --- import migrate_snowflake_clickhouse from '@site/static/images/migrations/migrate_snowflake_clickhouse.png'; import Image from '@theme/IdealImage'; -# Migrating from Snowflake to ClickHouse +# Migrate from Snowflake to ClickHouse -This guide shows how to migrate data from Snowflake to ClickHouse. +> This guide shows you how to migrate data from Snowflake to ClickHouse. -Migrating data between Snowflake and ClickHouse requires the use of an object store, such as S3, as an intermediate storage for transfer. The migration process also relies on using the commands `COPY INTO` from Snowflake and `INSERT INTO SELECT` of ClickHouse. +Migrating data between Snowflake and ClickHouse requires the use of an object store, +such as S3, as an intermediate storage for transfer. The migration process also +relies on using the commands `COPY INTO` from Snowflake and `INSERT INTO SELECT` +of ClickHouse. -## 1. Exporting data from Snowflake {#1-exporting-data-from-snowflake} + + +## Export data from Snowflake {#1-exporting-data-from-snowflake} Migrating from Snowflake to ClickHouse @@ -54,7 +58,7 @@ COPY INTO @external_stage/mydataset from mydataset max_file_size=157286400 heade For a dataset around 5TB of data with a maximum file size of 150MB, and using a 2X-Large Snowflake warehouse located in the same AWS `us-east-1` region, copying data to the S3 bucket will take around 30 minutes. -## 2. Importing to ClickHouse {#2-importing-to-clickhouse} +## Import to ClickHouse {#2-importing-to-clickhouse} Once the data is staged in intermediary object storage, ClickHouse functions such as the [s3 table function](/sql-reference/table-functions/s3) can be used to insert the data into a table, as shown below. @@ -65,10 +69,10 @@ Assuming the following table target schema: ```sql CREATE TABLE default.mydataset ( - `timestamp` DateTime64(6), - `some_text` String, - `some_file` Tuple(filename String, version String), - `complex_data` Tuple(name String, description String), + `timestamp` DateTime64(6), + `some_text` String, + `some_file` Tuple(filename String, version String), + `complex_data` Tuple(name String, description String), ) ENGINE = MergeTree ORDER BY (timestamp) @@ -79,16 +83,16 @@ We can then use the `INSERT INTO SELECT` command to insert the data from S3 into ```sql INSERT INTO mydataset SELECT - timestamp, - some_text, - JSONExtract( - ifNull(some_file, '{}'), - 'Tuple(filename String, version String)' - ) AS some_file, - JSONExtract( - ifNull(complex_data, '{}'), - 'Tuple(filename String, description String)' - ) AS complex_data, + timestamp, + some_text, + JSONExtract( + ifNull(some_file, '{}'), + 'Tuple(filename String, version String)' + ) AS some_file, + JSONExtract( + ifNull(complex_data, '{}'), + 'Tuple(filename String, description String)' + ) AS complex_data, FROM s3('https://mybucket.s3.amazonaws.com/mydataset/mydataset*.parquet') SETTINGS input_format_null_as_default = 1, -- Ensure columns are inserted as default if values are null input_format_parquet_case_insensitive_column_matching = 1 -- Column matching between source data and target table should be case insensitive @@ -100,10 +104,12 @@ The `VARIANT` and `OBJECT` columns in the original Snowflake table schema will b Nested structures such as `some_file` are converted to JSON strings on copy by Snowflake. Importing this data requires us to transform these structures to Tuples at insert time in ClickHouse, using the [JSONExtract function](/sql-reference/functions/json-functions#jsonextract) as shown above. ::: -## 3. Testing successful data export {#3-testing-successful-data-export} +## Test successful data export {#3-testing-successful-data-export} To test whether your data was properly inserted, simply run a `SELECT` query on your new table: ```sql SELECT * FROM mydataset LIMIT 10; ``` + + \ No newline at end of file diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/03_sql_translation_reference.md b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/03_sql_translation_reference.md new file mode 100644 index 00000000000..450a58fe32f --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/03_sql_translation_reference.md @@ -0,0 +1,114 @@ +--- +sidebar_label: 'SQL translation reference' +slug: /migrations/snowflake-translation-reference +description: 'SQL translation reference' +keywords: ['Snowflake'] +title: 'Migrating from Snowflake to ClickHouse' +show_related_blogs: true +--- + +# Snowflake SQL translation guide + +## Data types {#data-types} + +### Numerics {#numerics} + +Users moving data between ClickHouse and Snowflake will immediately notice that +ClickHouse offers more granular precision concerning declaring numerics. For example, +Snowflake offers the type Number for numerics. This requires the user to specify a +precision (total number of digits) and scale (digits to the right of the decimal place) +up to a total of 38. Integer declarations are synonymous with Number, and simply +define a fixed precision and scale where the range is the same. This convenience +is possible as modifying the precision (scale is 0 for integers) does not impact the +size of data on disk in Snowflake - the minimal required bytes are used for a +numeric range at write time at a micro partition level. The scale does, however, +impact storage space and is offset with compression. A `Float64` type offers a +wider range of values with a loss of precision. + +Contrast this with ClickHouse, which offers multiple signed and unsigned +precision for floats and integers. With these, ClickHouse users can be explicit about +the precision required for integers to optimize storage and memory overhead. A +Decimal type, equivalent to Snowflake’s Number type, also offers twice the +precision and scale at 76 digits. In addition to a similar `Float64` value, +ClickHouse also provides a `Float32` for when precision is less critical and +compression paramount. + +### Strings {#strings} + +ClickHouse and Snowflake take contrasting approaches to the storage of string +data. The `VARCHAR` in Snowflake holds Unicode characters in UTF-8, allowing the +user to specify a maximum length. This length has no impact on storage or +performance, with the minimum number of bytes always used to store a string, and +rather provides only constraints useful for downstream tooling. Other types, such +as `Text` and `NChar`, are simply aliases for this type. ClickHouse conversely +stores all [string data as raw bytes](/sql-reference/data-types/string) with a `String` +type (no length specification required), deferring encoding to the user, with +[query time functions](/sql-reference/functions/string-functions#lengthutf8) +available for different encodings. We refer the reader to ["Opaque data argument"](https://utf8everywhere.org/#cookie) +for the motivation as to why. The ClickHouse `String` is thus more comparable +to the Snowflake Binary type in its implementation. Both [Snowflake](https://docs.snowflake.com/en/sql-reference/collation) +and [ClickHouse](/sql-reference/statements/select/order-by#collation-support) +support “collation”, allowing users to override how strings are sorted and compared. + +### Semi-structured types {#semi-structured-data} + +Snowflake supports the `VARIANT`, `OBJECT` and `ARRAY` types for semi-structured +data. + +ClickHouse offers the equivalent [`Variant`](/sql-reference/data-types/variant), +[`Object`](/sql-reference/data-types/object-data-type) (deprecated) and [`Array`](/sql-reference/data-types/array) +types. Additionally, ClickHouse has the [`JSON`](/sql-reference/data-types/newjson) +type which replaces the now deprecated `Object('json')` type and is particularly +performant and storage efficient in [comparison to other native JSON types](https://jsonbench.com/). + +ClickHouse also supports named [`Tuple`s](/sql-reference/data-types/tuple) and arrays of Tuples +via the [`Nested`](/sql-reference/data-types/nested-data-structures/nested) type, +allowing users to explicitly map nested structures. This allows codecs and type +optimizations to be applied throughout the hierarchy, unlike Snowflake, which +requires the user to use the `OBJECT`, `VARIANT`, and `ARRAY` types for the outer +object and does not allow [explicit internal typing](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#characteristics-of-an-object). +This internal typing also simplifies queries on nested numerics in ClickHouse, +which do not need to be cast and can be used in index definitions. + +In ClickHouse, codecs and optimized types can also be applied to substructures. +This provides an added benefit that compression with nested structures remains +excellent, and comparable, to flattened data. In contrast, as a result of the +inability to apply specific types to substructures, Snowflake recommends [flattening +data to achieve optimal compression](https://docs.snowflake.com/en/user-guide/semistructured-considerations#storing-semi-structured-data-in-a-variant-column-vs-flattening-the-nested-structure). +Snowflake also [imposes size restrictions](https://docs.snowflake.com/en/user-guide/semistructured-considerations#data-size-limitations) +for these data types. + +### Type reference {#type-reference} + +| Snowflake | ClickHouse | Note | +|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`NUMBER`](https://docs.snowflake.com/en/sql-reference/data-types-numeric) | [`Decimal`](/sql-reference/data-types/decimal) | ClickHouse supports twice the precision and scale than Snowflake - 76 digits vs. 38. | +| [`FLOAT`, `FLOAT4`, `FLOAT8`](https://docs.snowflake.com/en/sql-reference/data-types-numeric#data-types-for-floating-point-numbers) | [`Float32`, `Float64`](/sql-reference/data-types/float) | All floats in Snowflake are 64 bit. | +| [`VARCHAR`](https://docs.snowflake.com/en/sql-reference/data-types-text#varchar) | [`String`](/sql-reference/data-types/string) | | +| [`BINARY`](https://docs.snowflake.com/en/sql-reference/data-types-text#binary) | [`String`](/sql-reference/data-types/string) | | +| [`BOOLEAN`](https://docs.snowflake.com/en/sql-reference/data-types-logical) | [`Bool`](/sql-reference/data-types/boolean) | | +| [`DATE`](https://docs.snowflake.com/en/sql-reference/data-types-datetime#date) | [`Date`](/sql-reference/data-types/date), [`Date32`](/sql-reference/data-types/date32) | `DATE` in Snowflake offers a wider date range than ClickHouse e.g. min for `Date32` is `1900-01-01` and `Date` `1970-01-01`. `Date` in ClickHouse provides more cost efficient (two byte) storage. | +| [`TIME(N)`](https://docs.snowflake.com/en/sql-reference/data-types-datetime#time) | No direct equivalent but can be represented by [`DateTime`](/sql-reference/data-types/datetime) and [`DateTime64(N)`](/sql-reference/data-types/datetime64). | `DateTime64` uses the same concepts of precision. | +| [`TIMESTAMP`](https://docs.snowflake.com/en/sql-reference/data-types-datetime#timestamp) - [`TIMESTAMP_LTZ`](https://docs.snowflake.com/en/sql-reference/data-types-datetime#timestamp-ltz-timestamp-ntz-timestamp-tz), [`TIMESTAMP_NTZ`](https://docs.snowflake.com/en/sql-reference/data-types-datetime#timestamp-ltz-timestamp-ntz-timestamp-tz), [`TIMESTAMP_TZ`](https://docs.snowflake.com/en/sql-reference/data-types-datetime#timestamp-ltz-timestamp-ntz-timestamp-tz) | [`DateTime`](/sql-reference/data-types/datetime) and [`DateTime64`](/sql-reference/data-types/datetime64) | `DateTime` and `DateTime64` can optionally have a TZ parameter defined for the column. If not present, the server's timezone is used. Additionally a `--use_client_time_zone` parameter is available for the client. | +| [`VARIANT`](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#variant) | [`JSON`, `Tuple`, `Nested`](/interfaces/formats) | `JSON` type is experimental in ClickHouse. This type infers the column types at insert time. `Tuple`, `Nested` and `Array` can also be used to build explicitly type structures as an alternative. | +| [`OBJECT`](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#object) | [`Tuple`, `Map`, `JSON`](/interfaces/formats) | Both `OBJECT` and `Map` are analogous to `JSON` type in ClickHouse where the keys are a `String`. ClickHouse requires the value to be consistent and strongly typed whereas Snowflake uses `VARIANT`. This means the values of different keys can be a different type. If this is required in ClickHouse, explicitly define the hierarchy using `Tuple` or rely on `JSON` type. | +| [`ARRAY`](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#array) | [`Array`](/sql-reference/data-types/array), [`Nested`](/sql-reference/data-types/nested-data-structures/nested) | `ARRAY` in Snowflake uses `VARIANT` for the elements - a super type. Conversely these are strongly typed in ClickHouse. | +| [`GEOGRAPHY`](https://docs.snowflake.com/en/sql-reference/data-types-geospatial#geography-data-type) | [`Point`, `Ring`, `Polygon`, `MultiPolygon`](/sql-reference/data-types/geo) | Snowflake imposes a coordinate system (WGS 84) while ClickHouse applies at query time. | +| [`GEOMETRY`](https://docs.snowflake.com/en/sql-reference/data-types-geospatial#geometry-data-type) | [`Point`, `Ring`, `Polygon`, `MultiPolygon`](/sql-reference/data-types/geo) | | | + +| ClickHouse Type | Description | +|-------------------|-----------------------------------------------------------------------------------------------------| +| `IPv4` and `IPv6` | IP-specific types, potentially allowing more efficient storage than Snowflake. | +| `FixedString` | Allows a fixed length of bytes to be used, which is useful for hashes. | +| `LowCardinality` | Allows any type to be dictionary encoded. Useful for when the cardinality is expected to be < 100k. | +| `Enum` | Allows efficient encoding of named values in either 8 or 16-bit ranges. | +| `UUID` | For efficient storage of UUIDs. | +| `Array(Float32)` | Vectors can be represented as an Array of Float32 with supported distance functions. | + +Finally, ClickHouse offers the unique ability to store the intermediate +[state of aggregate functions](/sql-reference/data-types/aggregatefunction). This +state is implementation-specific, but allows the result of an aggregation to be +stored and later queried (with corresponding merge functions). Typically, this +feature is used via a materialized view and, as demonstrated below, offers the +ability to improve performance of specific queries with minimal storage cost by +storing the incremental result of queries over inserted data (more details here). diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/_category_.json b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/_category_.json new file mode 100644 index 00000000000..50b05cb45a0 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/04_snowflake/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Snowflake", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/05_elastic/01_overview.md b/docs/cloud/onboard/02_migrate/01_migration_guides/05_elastic/01_overview.md new file mode 100644 index 00000000000..2104a173b02 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/05_elastic/01_overview.md @@ -0,0 +1,12 @@ +--- +sidebar_label: 'Overview' +slug: /migrations/elastic-overview +description: 'Migrating from Snowflake to ClickHouse' +keywords: ['Snowflake'] +title: 'Migrate from Snowflake to ClickHouse' +show_related_blogs: true +--- + +# Elasticsearch to ClickHouse migration + +For observability use cases, see the [Elasticsearch to ClickStack](/use-cases/observability/clickstack/migration/elastic) migration docs. diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/05_elastic/_category_.json b/docs/cloud/onboard/02_migrate/01_migration_guides/05_elastic/_category_.json new file mode 100644 index 00000000000..4f49621cf3d --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/05_elastic/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Elasticsearch", + "collapsible": true, + "collapsed": true +} \ No newline at end of file diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/01_overview.md b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/01_overview.md new file mode 100644 index 00000000000..785eba5d98a --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/01_overview.md @@ -0,0 +1,57 @@ +--- +sidebar_label: 'Overview' +slug: /migrations/redshift-overview +description: 'Migrating from Amazon Redshift to ClickHouse' +keywords: ['Redshift'] +title: 'Comparing ClickHouse Cloud and Amazon Redshift' +--- + +# Amazon Redshift to ClickHouse migration + +> This document provides an introduction to migrating data from Amazon +Redshift to ClickHouse. + +## Introduction {#introduction} + +Amazon Redshift is a cloud data warehouse that provides reporting and +analytics capabilities for structured and semi-structured data. It was +designed to handle analytical workloads on big data sets using +column-oriented database principles similar to ClickHouse. As part of the +AWS offering, it is often the default solution AWS users turn to for their +analytical data needs. + +While attractive to existing AWS users due to its tight integration with the +Amazon ecosystem, Redshift users that adopt it to power real-time analytics +applications find themselves in need of a more optimized solution for this +purpose. As a result, they increasingly turn to ClickHouse to benefit from +superior query performance and data compression, either as a replacement or +a "speed layer" deployed alongside existing Redshift workloads. + +## ClickHouse vs Redshift {#clickhouse-vs-redshift} + +For users heavily invested in the AWS ecosystem, Redshift represents a +natural choice when faced with data warehousing needs. Redshift differs from +ClickHouse in this important aspect – it optimizes its engine for data +warehousing workloads requiring complex reporting and analytical queries. +Across all deployment modes, the following two limitations make it difficult +to use Redshift for real-time analytical workloads: +* Redshift [compiles code for each query execution plan](https://docs.aws.amazon.com/redshift/latest/dg/c-query-performance.html), +which adds significant overhead to first-time query execution. This overhead can +be justified when query patterns are predictable and compiled execution plans +can be stored in a query cache. However, this introduces challenges for interactive +applications with variable queries. Even when Redshift is able to exploit this +code compilation cache, ClickHouse is faster on most queries. See ["ClickBench"](https://benchmark.clickhouse.com/#system=+%E2%98%81w|%EF%B8%8Fr|C%20c|Rf&type=-&machine=-ca2|gl|6ax|6ale|3al&cluster_size=-&opensource=-&tuned=+n&metric=hot&queries=-). +* Redshift [limits concurrency to 50 across all queues](https://docs.aws.amazon.com/redshift/latest/dg/c_workload_mngmt_classification.html), +which (while adequate for BI) makes it inappropriate for highly concurrent +analytical applications. + +Conversely, while ClickHouse can also be utilized for complex analytical queries +it is optimized for real-time analytical workloads, either powering applications +or acting as a warehouse acceleration later. As a result, Redshift users typically +replace or augment Redshift with ClickHouse for the following reasons: + +| Advantage | Description | +|------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Lower query latencies** | ClickHouse achieves lower query latencies, including for varied query patterns, under high concurrency and while subjected to streaming inserts. Even when your query misses a cache, which is inevitable in interactive user-facing analytics, ClickHouse can still process it fast. | +| **Higher concurrent query limits** | ClickHouse places much higher limits on concurrent queries, which is vital for real-time application experiences. In ClickHouse, self-managed as well as cloud, you can scale up your compute allocation to achieve the concurrency your application needs for each service. The level of permitted query concurrency is configurable in ClickHouse, with ClickHouse Cloud defaulting to a value of 1000. | +| **Superior data compression** | ClickHouse offers superior data compression, which allows users to reduce their total storage (and thus cost) or persist more data at the same cost and derive more real-time insights from their data. See "ClickHouse vs Redshift Storage Efficiency" below. | diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/02_migration_guide.md b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/02_migration_guide.md new file mode 100644 index 00000000000..506c9957e58 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/02_migration_guide.md @@ -0,0 +1,13 @@ +--- +sidebar_label: 'Migration guide' +slug: /migrations/redshift/migration-guide +description: 'Migrating from Amazon Redshift to ClickHouse' +keywords: ['Redshift'] +title: 'Amazon Redshift to ClickHouse migration guide' +--- + +import MigrationGuide from '@site/docs/integrations/data-ingestion/redshift/_snippets/_migration_guide.md' + +# Amazon Redshift to ClickHouse migration guide + + \ No newline at end of file diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/03_sql_translation_reference.md b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/03_sql_translation_reference.md new file mode 100644 index 00000000000..67585e4ea72 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/03_sql_translation_reference.md @@ -0,0 +1,95 @@ +--- +sidebar_label: 'SQL translation reference' +slug: /migrations/redshift/sql-translation-reference +description: 'SQL translation reference for Amazon Redshift to ClickHouse' +keywords: ['Redshift'] +title: 'Amazon Redshift SQL translation guide' +--- + +# Amazon Redshift SQL translation guide + +## Data types {#data-types} + +Users moving data between ClickHouse and Redshift will immediately notice +that ClickHouse offers a more extensive range of types, which are also less +restrictive. While Redshift requires users to specify possible string +lengths, even if variable, ClickHouse removes this restriction and burden +from the user by storing strings without encoding as bytes. The ClickHouse +String type thus has no limits or length specification requirements. + +Furthermore, users can exploit Arrays, Tuples, and Enums - absent from +Redshift as first-class citizens (although Arrays/Structs can be imitated +with `SUPER`) and a common frustration of users. ClickHouse additionally +allows the persistence, either at query time or even in a table, of +aggregation states. This will enable data to be pre-aggregated, typically +using a materialized view, and can dramatically improve query performance +for common queries. + +Below we map the equivalent ClickHouse type for each Redshift type: + +| Redshift | ClickHouse | +|------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`SMALLINT`](https://docs.aws.amazon.com/redshift/latest/dg/r_Numeric_types201.html#r_Numeric_types201-integer-types) | [`Int8`](/sql-reference/data-types/int-uint) * | +| [`INTEGER`](https://docs.aws.amazon.com/redshift/latest/dg/r_Numeric_types201.html#r_Numeric_types201-integer-types) | [`Int32`](/sql-reference/data-types/int-uint) * | +| [`BIGINT`](https://docs.aws.amazon.com/redshift/latest/dg/r_Numeric_types201.html#r_Numeric_types201-integer-types) | [`Int64`](/sql-reference/data-types/int-uint) * | +| [`DECIMAL`](https://docs.aws.amazon.com/redshift/latest/dg/r_Numeric_types201.html#r_Numeric_types201-decimal-or-numeric-type) | [`UInt128`, `UInt256`, `Int128`, `Int256`](/sql-reference/data-types/int-uint), [`Decimal(P, S)`, `Decimal32(S)`, `Decimal64(S)`, `Decimal128(S)`, `Decimal256(S)`](/sql-reference/data-types/decimal) - (high precision and ranges possible) | +| [`REAL`](https://docs.aws.amazon.com/redshift/latest/dg/r_Numeric_types201.html#r_Numeric_types201-floating-point-types) | [`Float32`](/sql-reference/data-types/float) | +| [`DOUBLE PRECISION`](https://docs.aws.amazon.com/redshift/latest/dg/r_Numeric_types201.html#r_Numeric_types201-floating-point-types) | [`Float64`](/sql-reference/data-types/float) | +| [`BOOLEAN`](https://docs.aws.amazon.com/redshift/latest/dg/r_Boolean_type.html) | [`Bool`](/sql-reference/data-types/boolean) | +| [`CHAR`](https://docs.aws.amazon.com/redshift/latest/dg/r_Character_types.html#r_Character_types-char-or-character) | [`String`](/sql-reference/data-types/string), [`FixedString`](/sql-reference/data-types/fixedstring) | +| [`VARCHAR`](https://docs.aws.amazon.com/redshift/latest/dg/r_Character_types.html#r_Character_types-varchar-or-character-varying) ** | [`String`](/sql-reference/data-types/string) | +| [`DATE`](https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-date) | [`Date32`](/sql-reference/data-types/date32) | +| [`TIMESTAMP`](https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-timestamp) | [`DateTime`](/sql-reference/data-types/datetime), [`DateTime64`](/sql-reference/data-types/datetime64) | +| [`TIMESTAMPTZ`](https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-timestamptz) | [`DateTime`](/sql-reference/data-types/datetime), [`DateTime64`](/sql-reference/data-types/datetime64) | +| [`GEOMETRY`](https://docs.aws.amazon.com/redshift/latest/dg/geospatial-overview.html) | [Geo Data Types](/sql-reference/data-types/geo) | +| [`GEOGRAPHY`](https://docs.aws.amazon.com/redshift/latest/dg/geospatial-overview.html) | [Geo Data Types](/sql-reference/data-types/geo) (less developed e.g. no coordinate systems - can be emulated [with functions](/sql-reference/functions/geo/)) | +| [`HLLSKETCH`](https://docs.aws.amazon.com/redshift/latest/dg/r_HLLSKTECH_type.html) | [`AggregateFunction(uniqHLL12, X)`](/sql-reference/data-types/aggregatefunction) | +| [`SUPER`](https://docs.aws.amazon.com/redshift/latest/dg/r_SUPER_type.html) | [`Tuple`](/sql-reference/data-types/tuple), [`Nested`](/sql-reference/data-types/nested-data-structures/nested), [`Array`](/sql-reference/data-types/array), [`JSON`](/sql-reference/data-types/newjson), [`Map`](/sql-reference/data-types/map) | +| [`TIME`](https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-time) | [`DateTime`](/sql-reference/data-types/datetime), [`DateTime64`](/sql-reference/data-types/datetime64) | +| [`TIMETZ`](https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-timetz) | [`DateTime`](/sql-reference/data-types/datetime), [`DateTime64`](/sql-reference/data-types/datetime64) | +| [`VARBYTE`](https://docs.aws.amazon.com/redshift/latest/dg/r_VARBYTE_type.html) ** | [`String`](/sql-reference/data-types/string) combined with [`Bit`](/sql-reference/functions/bit-functions) and [Encoding](/sql-reference/functions/encoding-functions/#hex) functions | + +* ClickHouse additionally supports unsigned integers with extended ranges i.e. `UInt8`, `UInt32`, `UInt32` and `UInt64`.
+**ClickHouse’s String type is unlimited by default but can be constrained to specific lengths using Constraints. + +## DDL syntax {#compression} + +### Sorting keys {#sorting-keys} + +Both ClickHouse and Redshift have the concept of a “sorting key”, which define +how data is sorted when being stored. Redshift defines the sorting key using the +`SORTKEY` clause: + +```sql +CREATE TABLE some_table(...) SORTKEY (column1, column2) +``` + +Comparatively, ClickHouse uses an `ORDER BY` clause to specify the sort order: + +```sql +CREATE TABLE some_table(...) ENGINE = MergeTree ORDER BY (column1, column2) +``` + +In most cases, you can use the same sorting key columns and order in ClickHouse +as Redshift, assuming you are using the default `COMPOUND` type. When data is +added to Redshift, you should run the `VACUUM` and `ANALYZE` commands to re-sort +newly added data and update the statistics for the query planner - otherwise, the +unsorted space grows. No such process is required for ClickHouse. + +Redshift supports a couple of convenience features for sorting keys. The first is +automatic sorting keys (using `SORTKEY AUTO`). While this may be appropriate for +getting started, explicit sorting keys ensure the best performance and storage +efficiency when the sorting key is optimal. The second is the `INTERLEAVED` sort key, +which gives equal weight to a subset of columns in the sort key to improve +performance when a query uses one or more secondary sort columns. ClickHouse +supports explicit [projections](/data-modeling/projections), which achieve the +same end-result with a slightly different setup. + +Users should be aware that the “primary key” concept represents different things +in ClickHouse and Redshift. In Redshift, the primary key resembles the traditional +RDMS concept intended to enforce constraints. However, they are not strictly +enforced in Redshift and instead act as hints for the query planner and data +distribution among nodes. In ClickHouse, the primary key denotes columns used +to construct the sparse primary index, used to ensure the data is ordered on +disk, maximizing compression while avoiding pollution of the primary index and +wasting memory. diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/_category_.json b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/_category_.json new file mode 100644 index 00000000000..95419dcb41c --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/06_redshift/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Redshift", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/integrations/migration/clickhouse-to-cloud.md b/docs/cloud/onboard/02_migrate/01_migration_guides/07_OSS_to_Cloud/01_clickhouse-to-cloud.md similarity index 95% rename from docs/integrations/migration/clickhouse-to-cloud.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/07_OSS_to_Cloud/01_clickhouse-to-cloud.md index 551314651e2..fed90c525c3 100644 --- a/docs/integrations/migration/clickhouse-to-cloud.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/07_OSS_to_Cloud/01_clickhouse-to-cloud.md @@ -1,6 +1,5 @@ --- -sidebar_position: 10 -sidebar_label: 'ClickHouse to ClickHouse Cloud' +sidebar_label: 'ClickHouse OSS' slug: /cloud/migration/clickhouse-to-cloud title: 'Migrating between self-managed ClickHouse and ClickHouse Cloud' description: 'Page describing how to migrate between self-managed ClickHouse and ClickHouse Cloud' @@ -19,7 +18,7 @@ import self_managed_06 from '@site/static/images/integrations/migration/self-man Migrating Self-managed ClickHouse -This guide will show how to migrate from a self-managed ClickHouse server to ClickHouse Cloud, and also how to migrate between ClickHouse Cloud services. The [`remoteSecure`](../../sql-reference/table-functions/remote.md) function is used in `SELECT` and `INSERT` queries to allow access to remote ClickHouse servers, which makes migrating tables as simple as writing an `INSERT INTO` query with an embedded `SELECT`. +This guide will show how to migrate from a self-managed ClickHouse server to ClickHouse Cloud, and also how to migrate between ClickHouse Cloud services. The [`remoteSecure`](/sql-reference/table-functions/remote) function is used in `SELECT` and `INSERT` queries to allow access to remote ClickHouse servers, which makes migrating tables as simple as writing an `INSERT INTO` query with an embedded `SELECT`. ## Migrating from Self-managed ClickHouse to ClickHouse Cloud {#migrating-from-self-managed-clickhouse-to-clickhouse-cloud} diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/07_OSS_to_Cloud/_category_.json b/docs/cloud/onboard/02_migrate/01_migration_guides/07_OSS_to_Cloud/_category_.json new file mode 100644 index 00000000000..9720f826193 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/07_OSS_to_Cloud/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "OSS to Cloud", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/integrations/migration/clickhouse-local-etl.md b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/01_clickhouse-local-etl.md similarity index 99% rename from docs/integrations/migration/clickhouse-local-etl.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/01_clickhouse-local-etl.md index 2faf0a935d7..5e3eabc70c9 100644 --- a/docs/integrations/migration/clickhouse-local-etl.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/01_clickhouse-local-etl.md @@ -1,6 +1,5 @@ --- sidebar_label: 'Using clickhouse-local' -sidebar_position: 20 keywords: ['clickhouse', 'migrate', 'migration', 'migrating', 'data', 'etl', 'elt', 'clickhouse-local', 'clickhouse-client'] slug: /cloud/migration/clickhouse-local title: 'Migrating to ClickHouse using clickhouse-local' diff --git a/docs/integrations/migration/etl-tool-to-clickhouse.md b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/02_etl-tool-to-clickhouse.md similarity index 98% rename from docs/integrations/migration/etl-tool-to-clickhouse.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/02_etl-tool-to-clickhouse.md index f66e6ff2c47..32a0c168c5a 100644 --- a/docs/integrations/migration/etl-tool-to-clickhouse.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/02_etl-tool-to-clickhouse.md @@ -1,6 +1,5 @@ --- sidebar_label: 'Using a 3rd-party ETL Tool' -sidebar_position: 20 keywords: ['clickhouse', 'migrate', 'migration', 'migrating', 'data', 'etl', 'elt', 'clickhouse-local', 'clickhouse-client'] slug: /cloud/migration/etl-tool-to-clickhouse title: 'Using a 3rd-party ETL Tool' diff --git a/docs/integrations/migration/object-storage-to-clickhouse.md b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/03_object-storage-to-clickhouse.md similarity index 92% rename from docs/integrations/migration/object-storage-to-clickhouse.md rename to docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/03_object-storage-to-clickhouse.md index 2f323db04ef..a0788a80aa0 100644 --- a/docs/integrations/migration/object-storage-to-clickhouse.md +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/03_object-storage-to-clickhouse.md @@ -1,5 +1,5 @@ --- -title: 'Object Storage to ClickHouse Cloud' +title: 'Using object storage' description: 'Moving data from object storage to ClickHouse Cloud' keywords: ['object storage', 's3', 'azure blob', 'gcs', 'migration'] slug: /integrations/migration/object-storage-to-clickhouse @@ -20,7 +20,7 @@ table functions for migrating data stored in Cloud Object Storage into a ClickHo - [gcs](/sql-reference/table-functions/gcs) - [azureBlobStorage](/sql-reference/table-functions/azureBlobStorage) -If your current database system is not able to directly offload data into a Cloud Object Storage, you could use a [third-party ETL/ELT tool](./etl-tool-to-clickhouse.md) or [clickhouse-local](./clickhouse-local-etl.md) for moving data +If your current database system is not able to directly offload data into a Cloud Object Storage, you could use a [third-party ETL/ELT tool](/cloud/migration/etl-tool-to-clickhouse) or [clickhouse-local](/cloud/migration/clickhouse-local) for moving data from you current database system to Cloud Object Storage, in order to migrate that data in a second step into a ClickHouse Cloud table. Although this is a two steps process (offload data into a Cloud Object Storage, then load into ClickHouse), the advantage is that this diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/_category_.json b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/_category_.json new file mode 100644 index 00000000000..61c592ce8a0 --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/08_other_methods/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Other...", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/onboard/02_migrate/01_migration_guides/_category_.json b/docs/cloud/onboard/02_migrate/01_migration_guides/_category_.json new file mode 100644 index 00000000000..aca0c529bce --- /dev/null +++ b/docs/cloud/onboard/02_migrate/01_migration_guides/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Migration guides", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/onboard/03_tune/_snippets/_monitoring_table_of_contents.md b/docs/cloud/onboard/03_tune/_snippets/_monitoring_table_of_contents.md new file mode 100644 index 00000000000..e5d813d8226 --- /dev/null +++ b/docs/cloud/onboard/03_tune/_snippets/_monitoring_table_of_contents.md @@ -0,0 +1,3 @@ +| Page | Description | +|------|-------------| +| | | diff --git a/docs/cloud/onboard/03_tune/resource_tour.md b/docs/cloud/onboard/03_tune/resource_tour.md new file mode 100644 index 00000000000..09217a392f9 --- /dev/null +++ b/docs/cloud/onboard/03_tune/resource_tour.md @@ -0,0 +1,50 @@ +--- +slug: /cloud/get-started/cloud/resource-tour +title: 'Resource tour' +description: 'Overview of ClickHouse Cloud documentation resources for query optimization, scaling strategies, monitoring, and best practices' +keywords: ['clickhouse cloud'] +hide_title: true +--- + +import TableOfContentsBestPractices from '@site/docs/best-practices/_snippets/_table_of_contents.md'; +import TableOfContentsOptimizationAndPerformance from '@site/docs/guides/best-practices/_snippets/_performance_optimizations_table_of_contents.md'; +import TableOfContentsSecurity from '@site/docs/cloud/_snippets/_security_table_of_contents.md'; + +# Resource tour + +This article is intended to provide you with an overview of the resources available +to you in the docs to learn how to get the most out of your ClickHouse Cloud deployment. +Explore resource organised by the following topics: + +- [Query optimization techniques and performance tuning](#query-optimization) +- [Monitoring](#monitoring) +- [Security best practices and compliance features](#security) +- [Cost optimization and billing](#cost-optimization) + +Before diving into more specific topics, we recommend you start with our general +ClickHouse best practice guides which cover general best practices to follow when +using ClickHouse: + + + +## Query optimization techniques and performance tuning {#query-optimization} + + + +## Monitoring {#monitoring} + +| Page | Description | +|-----------------------------------------------------------------|-------------------------------------------------------------------------------| +| [Advanced dashboard](/cloud/manage/monitor/advanced-dashboard) | Use the built in advanced dashboard to monitor service health and performance | +| [Prometheus integration](/integrations/prometheus) | Use Prometheus to monitor Cloud services | + +## Security {#security} + + + +## Cost optimization and billing {#cost-optimization} + +| Page | Description | +|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------| +| [Data transfer](/cloud/manage/network-data-transfer)| Understand how ClickHouse Cloud meters data transferred ingress and egress | +| [Notifications](/cloud/notifications) | Set up notifications for your ClickHouse Cloud service. For example, when credit usage passes a threshold | diff --git a/docs/cloud/onboard/index.md b/docs/cloud/onboard/index.md new file mode 100644 index 00000000000..742a9c87254 --- /dev/null +++ b/docs/cloud/onboard/index.md @@ -0,0 +1,45 @@ +--- +slug: /cloud/get-started +title: 'Get started with ClickHouse Cloud' +description: 'Complete guide to getting started with ClickHouse Cloud - from discovering features to deployment and optimization' +hide_title: true +--- + +# Get started with ClickHouse Cloud + +New to ClickHouse Cloud and not sure where to begin? In this section of the docs, +we'll walk you through everything you need to get up and running quickly. We've +arranged this getting started section into three subsections to help guide +you through each step of the process as you explore ClickHouse Cloud. + + + +## Discover ClickHouse Cloud {#discover-clickhouse-cloud} + +- [Learn](/cloud/overview) about what ClickHouse Cloud is, and how it differs from the open-source version +- [Discover](/cloud/get-started/cloud/use-cases/overview) the main use-cases of ClickHouse Cloud + +## Get set up with ClickHouse Cloud {#get-set-up-with-clickhouse-cloud} + +Now that you know what ClickHouse Cloud is, we'll walk you through the process +of getting your data into ClickHouse Cloud, show you the main features available +and point you towards some general best practices you should know. + +Topics include: + +- [Migration guides](/integrations/migration/overview) from various platforms + +## Tune your ClickHouse Cloud deployment {#evaluate-clickhouse-cloud} + +Now that your data is in ClickHouse Cloud, we'll walk you through some more advanced +topics to help you get the most out of your ClickHouse Cloud experience and explore +what the platform has to offer. + +Topics include: + +- [Query performance and optimization](/cloud/get-started/cloud/resource-tour#query-optimization) +- [Monitoring](/cloud/get-started/cloud/resource-tour#monitoring) +- [Security considerations](/cloud/get-started/cloud/resource-tour#security) +- Troubleshooting tips + + \ No newline at end of file diff --git a/docs/cloud/reference/changelog.md b/docs/cloud/reference/01_changelog/01_changelog.md similarity index 99% rename from docs/cloud/reference/changelog.md rename to docs/cloud/reference/01_changelog/01_changelog.md index 024a14b200c..44197a15198 100644 --- a/docs/cloud/reference/changelog.md +++ b/docs/cloud/reference/01_changelog/01_changelog.md @@ -29,7 +29,7 @@ import share_queries from '@site/static/images/cloud/reference/may-30-share-quer import query_endpoints from '@site/static/images/cloud/reference/may-17-query-endpoints.png'; import dashboards from '@site/static/images/cloud/reference/may-30-dashboards.png'; -In addition to this ClickHouse Cloud changelog, please see the [Cloud Compatibility](/cloud/reference/cloud-compatibility.md) page. +In addition to this ClickHouse Cloud changelog, please see the [Cloud Compatibility](/whats-new/cloud-compatibility) page. ## August 29, 2025 {#august-29-2025} @@ -85,10 +85,10 @@ to get up and running. - New services now store database and table metadata in a central **SharedCatalog**, a new model for coordination and object lifecycles which enables: - - **Cloud-scale DDL**, even under high concurrency - - **Resilient deletion and new DDL operations** - - **Fast spin-up and wake-ups** as stateless nodes now launch with no disk dependencies - - **Stateless compute across both native and open formats**, including Iceberg and Delta Lake + - **Cloud-scale DDL**, even under high concurrency + - **Resilient deletion and new DDL operations** + - **Fast spin-up and wake-ups** as stateless nodes now launch with no disk dependencies + - **Stateless compute across both native and open formats**, including Iceberg and Delta Lake Read more about SharedCatalog in our [blog](https://clickhouse.com/blog/clickhouse-cloud-stateless-compute) @@ -183,7 +183,7 @@ to get up and running. ## April 4, 2025 {#april-4-2025} - Slack notifications for ClickHouse Cloud: ClickHouse Cloud now supports Slack notifications for billing, scaling, and ClickPipes events, in addition to in-console and email notifications. These notifications are sent via the ClickHouse Cloud Slack application. Organization admins can configure these notifications via the notification center by specifying slack channels to which notifications should be sent. -- Users running Production and Development services will now see ClickPipes and data transfer usage price on their bills. Please refer to the [announcement](/cloud/manage/jan-2025-faq/pricing-dimensions) from January 2025 for more details. +- Users running Production and Development services will now see ClickPipes and data transfer usage price on their bills. ## March 21, 2025 {#march-21-2025} @@ -266,7 +266,7 @@ We are adding a **new Enterprise tier** to serve the needs of the most demanding To support these changes, we are restructuring our current **Development** and **Production** tiers to more closely match how our evolving customer base is using our offerings. We are introducing the **Basic** tier, oriented toward users that are testing out new ideas and projects, and the **Scale** tier, matching users working with production workloads and data at scale. -You can read about these and other functional changes in this [blog](https://clickhouse.com/blog/evolution-of-clickhouse-cloud-new-features-superior-performance-tailored-offerings). Existing customers will need to take action to select a [new plan](https://clickhouse.com/pricing). Customer-facing communication was sent via email to organization administrators, and the following [FAQ](/cloud/manage/jan-2025-faq/summary) covers the key changes and timelines. +You can read about these and other functional changes in this [blog](https://clickhouse.com/blog/evolution-of-clickhouse-cloud-new-features-superior-performance-tailored-offerings). Existing customers will need to take action to select a [new plan](https://clickhouse.com/pricing). Customer-facing communication was sent via email to organization administrators. ### Warehouses: Compute-compute separation (GA) {#warehouses-compute-compute-separation-ga} @@ -294,7 +294,7 @@ Safe managed upgrades deliver significant value to our users by allowing them to ### HIPAA support {#hipaa-support} -We now support HIPAA in compliant regions, including AWS `us-east-1`, `us-west-2` and GCP `us-central1`, `us-east1`. Customers wishing to onboard must sign a Business Associate Agreement (BAA) and deploy to the compliant version of the region. For more information on HIPAA, please refer to the [documentation](/cloud/security/security-and-compliance). +We now support HIPAA in compliant regions, including AWS `us-east-1`, `us-west-2` and GCP `us-central1`, `us-east1`. Customers wishing to onboard must sign a Business Associate Agreement (BAA) and deploy to the compliant version of the region. For more information on HIPAA, please refer to the [documentation](/cloud/security/compliance-overview). ### Scheduled upgrades {#scheduled-upgrades} @@ -789,12 +789,12 @@ This release upgrades the core database version, adds ability to set up private ### Integrations changes {#integrations-changes-4} * Kafka Connect - * Support async_insert for exactly once (disabled by default) + * Support async_insert for exactly once (disabled by default) * Golang client - * Fixed DateTime binding - * Improved batch insert performance + * Fixed DateTime binding + * Improved batch insert performance * Java client - * Fixed request compression problem + * Fixed request compression problem ### Settings changes {#settings-changes} * `use_mysql_types_in_show_columns` is no longer required. It will be automatically enabled when you connect through the MySQL interface. @@ -1442,7 +1442,7 @@ This release enables dictionaries from local ClickHouse table and HTTP sources, ### General changes {#general-changes-5} - Added support for [dictionaries](/sql-reference/dictionaries/index.md) from local ClickHouse table and HTTP sources -- Introduced support for the Mumbai [region](/cloud/reference/supported-regions.md) +- Introduced support for the Mumbai [region](/cloud/reference/supported-regions) ### Console changes {#console-changes-30} @@ -1510,4 +1510,4 @@ This release significantly lowers compute consumption for small workloads, lower ClickHouse Cloud began its public Beta on October 4th, 2022. Learn more in this [blog](https://clickhouse.com/blog/clickhouse-cloud-public-beta). -The ClickHouse Cloud version is based on ClickHouse core v22.10. For a list of compatible features, refer to the [Cloud Compatibility](/cloud/reference/cloud-compatibility.md) guide. +The ClickHouse Cloud version is based on ClickHouse core v22.10. For a list of compatible features, refer to the [Cloud Compatibility](/whats-new/cloud-compatibility) guide. diff --git a/docs/cloud/changelogs/24_02.md b/docs/cloud/reference/01_changelog/02_release_notes/24_02.md similarity index 100% rename from docs/cloud/changelogs/24_02.md rename to docs/cloud/reference/01_changelog/02_release_notes/24_02.md diff --git a/docs/cloud/changelogs/24_05.md b/docs/cloud/reference/01_changelog/02_release_notes/24_05.md similarity index 100% rename from docs/cloud/changelogs/24_05.md rename to docs/cloud/reference/01_changelog/02_release_notes/24_05.md diff --git a/docs/cloud/changelogs/24_06.md b/docs/cloud/reference/01_changelog/02_release_notes/24_06.md similarity index 100% rename from docs/cloud/changelogs/24_06.md rename to docs/cloud/reference/01_changelog/02_release_notes/24_06.md diff --git a/docs/cloud/changelogs/24_08.md b/docs/cloud/reference/01_changelog/02_release_notes/24_08.md similarity index 100% rename from docs/cloud/changelogs/24_08.md rename to docs/cloud/reference/01_changelog/02_release_notes/24_08.md diff --git a/docs/cloud/changelogs/24_10.md b/docs/cloud/reference/01_changelog/02_release_notes/24_10.md similarity index 100% rename from docs/cloud/changelogs/24_10.md rename to docs/cloud/reference/01_changelog/02_release_notes/24_10.md diff --git a/docs/cloud/changelogs/24_12.md b/docs/cloud/reference/01_changelog/02_release_notes/24_12.md similarity index 100% rename from docs/cloud/changelogs/24_12.md rename to docs/cloud/reference/01_changelog/02_release_notes/24_12.md diff --git a/docs/cloud/changelogs/25_04.md b/docs/cloud/reference/01_changelog/02_release_notes/25_04.md similarity index 100% rename from docs/cloud/changelogs/25_04.md rename to docs/cloud/reference/01_changelog/02_release_notes/25_04.md diff --git a/docs/cloud/changelogs/25_06.md b/docs/cloud/reference/01_changelog/02_release_notes/25_06.md similarity index 100% rename from docs/cloud/changelogs/25_06.md rename to docs/cloud/reference/01_changelog/02_release_notes/25_06.md diff --git a/docs/cloud/reference/01_changelog/02_release_notes/_category_.json b/docs/cloud/reference/01_changelog/02_release_notes/_category_.json new file mode 100644 index 00000000000..4eeae460788 --- /dev/null +++ b/docs/cloud/reference/01_changelog/02_release_notes/_category_.json @@ -0,0 +1,6 @@ +{ + "label": "Release notes", + "collapsible": true, + "collapsed": true, + "link": { "type": "doc", "id": "cloud/reference/changelog/release_notes/index" } +} \ No newline at end of file diff --git a/docs/cloud/reference/01_changelog/02_release_notes/index.md b/docs/cloud/reference/01_changelog/02_release_notes/index.md new file mode 100644 index 00000000000..ab87960c306 --- /dev/null +++ b/docs/cloud/reference/01_changelog/02_release_notes/index.md @@ -0,0 +1,24 @@ +--- +slug: /cloud/reference/changelogs/release-notes +title: 'Cloud Release Notes' +description: 'Landing page for Cloud release notes' +--- + + + + +| Page | Description | +|-----|-----| +| [v25.6 Changelog for Cloud](/changelogs/25.6) | Fast release changelog for v25.6 | +| [v25.4 Changelog for Cloud](/changelogs/25.4) | Fast release changelog for v25.4 | +| [v24.12 Changelog for Cloud](/changelogs/24.12) | Fast release changelog for v24.12 | +| [v24.10 Changelog for Cloud](/changelogs/24.10) | Fast release changelog for v24.10 | +| [v24.8 Changelog for Cloud](/changelogs/24.8) | Fast release changelog for v24.8 | +| [v24.6 Changelog for Cloud](/changelogs/24.6) | Fast release changelog for v24.6 | +| [v24.5 Changelog for Cloud](/changelogs/24.5) | Fast release changelog for v24.5 | +| [v24.2 Changelog](/whats-new/changelog/24.2-fast-release) | Fast release changelog for v24.2 | +| [Cloud Changelog](/whats-new/cloud) | ClickHouse Cloud changelog providing descriptions of what is new in each ClickHouse Cloud release | + diff --git a/docs/cloud/reference/01_changelog/_category_.json b/docs/cloud/reference/01_changelog/_category_.json new file mode 100644 index 00000000000..60a9e95ee7e --- /dev/null +++ b/docs/cloud/reference/01_changelog/_category_.json @@ -0,0 +1,6 @@ +{ + "label": "Change logs", + "collapsible": true, + "collapsed": true, + "link": { "type": "doc", "id": "cloud/reference/changelog/index" } +} \ No newline at end of file diff --git a/docs/cloud/reference/changelogs-index.md b/docs/cloud/reference/01_changelog/index.md similarity index 91% rename from docs/cloud/reference/changelogs-index.md rename to docs/cloud/reference/01_changelog/index.md index c23e70f4ea2..cfdb11087f8 100644 --- a/docs/cloud/reference/changelogs-index.md +++ b/docs/cloud/reference/01_changelog/index.md @@ -7,4 +7,4 @@ description: 'Landing page for Cloud changelogs' | Page | Description | |---------------------------------------------------------------|-------------------------------------------------| | [Cloud Changelog](/whats-new/cloud) | Changelog for ClickHouse Cloud | -| [Release Notes](/cloud/reference/changelogs/release-notes) | Release notes for all ClickHouse Cloud releases | +| [Release Notes](/cloud/reference/changelogs/release-notes) | Release notes for all ClickHouse Cloud releases | \ No newline at end of file diff --git a/docs/cloud/reference/architecture.md b/docs/cloud/reference/02_architecture.md similarity index 98% rename from docs/cloud/reference/architecture.md rename to docs/cloud/reference/02_architecture.md index 9c3d7cf5f56..6e3294d3a97 100644 --- a/docs/cloud/reference/architecture.md +++ b/docs/cloud/reference/02_architecture.md @@ -1,7 +1,7 @@ --- sidebar_label: 'Architecture' slug: /cloud/reference/architecture -title: 'ClickHouse Cloud Architecture' +title: 'ClickHouse Cloud architecture' description: 'This page describes the architecture of ClickHouse Cloud' --- diff --git a/docs/cloud/manage/billing.md b/docs/cloud/reference/03_billing/01_billing_overview.md similarity index 98% rename from docs/cloud/manage/billing.md rename to docs/cloud/reference/03_billing/01_billing_overview.md index 3745df1d2aa..cdbe6c40355 100644 --- a/docs/cloud/manage/billing.md +++ b/docs/cloud/reference/03_billing/01_billing_overview.md @@ -5,7 +5,7 @@ title: 'Pricing' description: 'Overview page for ClickHouse Cloud pricing' --- -import ClickPipesFAQ from './jan2025_faq/_snippets/_clickpipes_faq.md' +import ClickPipesFAQ from '../../_snippets/_clickpipes_faq.md' For pricing information, see the [ClickHouse Cloud Pricing](https://clickhouse.com/pricing#pricing-calculator) page. ClickHouse Cloud bills based on the usage of compute, storage, [data transfer](/cloud/manage/network-data-transfer) (egress over the internet and cross-region), and [ClickPipes](/integrations/clickpipes). @@ -15,7 +15,7 @@ To understand what can affect your bill, and ways that you can manage your spend :::note - Prices reflect AWS us-east-1 pricing. -- Explore applicable data transfer and ClickPipes charges [here](jan2025_faq/dimensions.md). +- Explore applicable data transfer and ClickPipes charges [here](/cloud/manage/network-data-transfer). ::: ### Basic: from $66.52 per month {#basic-from-6652-per-month} @@ -191,7 +191,7 @@ Storage costs are the same across tiers and vary by region and cloud service pro Storage and backups are counted towards storage costs and billed separately. All services will default to one backup, retained for a day. -Users who need additional backups can do so by configuring additional [backups](backups/overview.md) under the settings tab of the Cloud console. +Users who need additional backups can do so by configuring additional [backups](/cloud/manage/backups/overview) under the settings tab of the Cloud console. ### How do I estimate compression? {#how-do-i-estimate-compression} @@ -287,7 +287,7 @@ which are metered in the same way and billed accordingly. When creating a service in addition to an existing service, you can choose if this new service should share the same data with the existing one. -If yes, these two services now form a [warehouse](../reference/warehouses.md). +If yes, these two services now form a [warehouse](/cloud/reference/warehouses). A warehouse has the data stored in it with multiple compute services accessing this data. As the data is stored only once, you only pay for one copy of data, though multiple services are accessing it. diff --git a/docs/cloud/manage/billing/marketplace/aws-marketplace-committed.md b/docs/cloud/reference/03_billing/02_marketplace/aws-marketplace-committed.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/aws-marketplace-committed.md rename to docs/cloud/reference/03_billing/02_marketplace/aws-marketplace-committed.md diff --git a/docs/cloud/manage/billing/marketplace/aws-marketplace-payg.md b/docs/cloud/reference/03_billing/02_marketplace/aws-marketplace-payg.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/aws-marketplace-payg.md rename to docs/cloud/reference/03_billing/02_marketplace/aws-marketplace-payg.md diff --git a/docs/cloud/manage/billing/marketplace/azure-marketplace-committed.md b/docs/cloud/reference/03_billing/02_marketplace/azure-marketplace-committed.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/azure-marketplace-committed.md rename to docs/cloud/reference/03_billing/02_marketplace/azure-marketplace-committed.md diff --git a/docs/cloud/manage/billing/marketplace/azure-marketplace-payg.md b/docs/cloud/reference/03_billing/02_marketplace/azure-marketplace-payg.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/azure-marketplace-payg.md rename to docs/cloud/reference/03_billing/02_marketplace/azure-marketplace-payg.md diff --git a/docs/cloud/manage/billing/marketplace/gcp-marketplace-committed.md b/docs/cloud/reference/03_billing/02_marketplace/gcp-marketplace-committed.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/gcp-marketplace-committed.md rename to docs/cloud/reference/03_billing/02_marketplace/gcp-marketplace-committed.md diff --git a/docs/cloud/manage/billing/marketplace/gcp-marketplace-payg.md b/docs/cloud/reference/03_billing/02_marketplace/gcp-marketplace-payg.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/gcp-marketplace-payg.md rename to docs/cloud/reference/03_billing/02_marketplace/gcp-marketplace-payg.md diff --git a/docs/cloud/manage/billing/marketplace/index.md b/docs/cloud/reference/03_billing/02_marketplace/index.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/index.md rename to docs/cloud/reference/03_billing/02_marketplace/index.md diff --git a/docs/cloud/manage/billing/marketplace/migrate-marketplace-payg-committed.md b/docs/cloud/reference/03_billing/02_marketplace/migrate-marketplace-payg-committed.md similarity index 84% rename from docs/cloud/manage/billing/marketplace/migrate-marketplace-payg-committed.md rename to docs/cloud/reference/03_billing/02_marketplace/migrate-marketplace-payg-committed.md index 937a751b0fa..b3feb2aae06 100644 --- a/docs/cloud/manage/billing/marketplace/migrate-marketplace-payg-committed.md +++ b/docs/cloud/reference/03_billing/02_marketplace/migrate-marketplace-payg-committed.md @@ -32,12 +32,12 @@ If you want to use a different AWS Account ID for migrating your ClickHouse orga 1. **Go to the [AWS Marketplace](https://us-east-1.console.aws.amazon.com/marketplace)** 2. **Click on the "Manage Subscriptions" button** 3. **Navigate to "Your Subscriptions":** - - Click on "Manage Subscriptions" + - Click on "Manage Subscriptions" 4. **Find ClickHouse Cloud in the list:** - - Look and click on ClickHouse Cloud under "Your Subscriptions" + - Look and click on ClickHouse Cloud under "Your Subscriptions" 5. **Cancel the Subscription:** - - Under "Agreement" click on the "Actions" dropdown or button next to the ClickHouse Cloud listing - - Select "Cancel subscription" + - Under "Agreement" click on the "Actions" dropdown or button next to the ClickHouse Cloud listing + - Select "Cancel subscription" > **Note:** For help cancelling your subscription (e.g. if the cancel subscription button is not available) please contact [AWS support](https://support.console.aws.amazon.com/support/home#/). @@ -48,12 +48,12 @@ Next follow these [steps](/cloud/billing/marketplace/aws-marketplace-committed-c ### Steps to Cancel GCP PAYG Order {#cancel-gcp-payg} 1. **Go to your [Google Cloud Marketplace Console](https://console.cloud.google.com/marketplace):** - - Make sure you are logged in to the correct GCP account and have selected the appropriate project + - Make sure you are logged in to the correct GCP account and have selected the appropriate project 2. **Locate your ClickHouse order:** - - In the left menu, click "Your Orders" - - Find the correct ClickHouse order in the list of active orders + - In the left menu, click "Your Orders" + - Find the correct ClickHouse order in the list of active orders 3. **Cancel the order:** - - Find the three dots menu to the right of your order and follow the instructions to cancel the ClickHouse order + - Find the three dots menu to the right of your order and follow the instructions to cancel the ClickHouse order > **Note:** For help cancelling this order please contact [GCP support](https://cloud.google.com/support/docs/get-billing-support). @@ -67,12 +67,12 @@ Next follow these [steps](/cloud/billing/marketplace/gcp-marketplace-committed-c 2. **Navigate to "Subscriptions"** 3. **Locate the active ClickHouse subscription you want to cancel** 4. **Cancel the Subscription:** - - Click on the ClickHouse Cloud subscription to open the subscription details - - Select the "Cancel subscription" button + - Click on the ClickHouse Cloud subscription to open the subscription details + - Select the "Cancel subscription" button > **Note:** For help cancelling this order please open a support ticket in your Azure Portal. -Next follow these [steps](/cloud/billing/marketplace/azure-marketplace-committed-contract) to configure your ClickHouse organization to your new Azure committed spend contract. +Next, follow these [steps](/cloud/billing/marketplace/azure-marketplace-committed-contract) to configure your ClickHouse organization to your new Azure committed spend contract. ## Requirements for Linking to Committed Spend Contract {#linking-requirements} diff --git a/docs/cloud/manage/billing/marketplace/overview.md b/docs/cloud/reference/03_billing/02_marketplace/overview.md similarity index 100% rename from docs/cloud/manage/billing/marketplace/overview.md rename to docs/cloud/reference/03_billing/02_marketplace/overview.md diff --git a/docs/cloud/manage/billing/payment-thresholds.md b/docs/cloud/reference/03_billing/03_payment-thresholds.md similarity index 97% rename from docs/cloud/manage/billing/payment-thresholds.md rename to docs/cloud/reference/03_billing/03_payment-thresholds.md index 0c2b6948d0e..2d9ce5f188a 100644 --- a/docs/cloud/manage/billing/payment-thresholds.md +++ b/docs/cloud/reference/03_billing/03_payment-thresholds.md @@ -1,7 +1,7 @@ --- sidebar_label: 'Payment Thresholds' slug: /cloud/billing/payment-thresholds -title: 'Payment Thresholds' +title: 'Payment thresholds' description: 'Payment thresholds and automatic invoicing for ClickHouse Cloud.' keywords: ['billing', 'payment thresholds', 'automatic invoicing', 'invoice'] --- diff --git a/docs/cloud/reference/03_billing/04_network-data-transfer.mdx b/docs/cloud/reference/03_billing/04_network-data-transfer.mdx new file mode 100644 index 00000000000..4013e1477b7 --- /dev/null +++ b/docs/cloud/reference/03_billing/04_network-data-transfer.mdx @@ -0,0 +1,56 @@ +--- +sidebar_label: 'Data Transfer' +slug: /cloud/manage/network-data-transfer +title: 'Data Transfer' +description: 'Understand how ClickHouse Cloud meters data transferred ingress and egress' +--- + +import NetworkPricing from '@site/docs/cloud/reference/_snippets/_network_transfer_rates.md'; + +ClickHouse Cloud meters data transferred ingress and egress. +This includes any data in and out of ClickHouse Cloud as well as any intra-region +and cross-region data transfer. This usage is tracked at the service level. Based +on this usage, customers incur data transfer charges that are then added to their +monthly bill. + +ClickHouse Cloud charges for: +- Data egress from ClickHouse Cloud to the public Internet, including to other +regions of other cloud providers. +- Data egress to another region in the same cloud provider. + +There are no charges for intra-region data transfer or Private Link/Private +Service Connect use and data transfer.However, we reserve the right to implement +additional data transfer pricing dimensions if we see usage patterns that impact +our ability to charge users appropriately. + +Data transfer charges vary by Cloud Service Provider (CSP) and region. +Public internet egress pricing is based only on the origin region. +Inter-region (or cross-region) pricing depends on both the origin and destination +regions. + +**Best Practices to minimize Data Transfer Costs** + +There are some patterns to keep in mind when ingressing and egressing data to +minimize data transfer costs. + +1. When ingressing or egressing data from Clickhouse Cloud, use compression where +possible, to minimize the amount of data transferred and the associated cost. + +2. Be aware that when doing an INSERT over the native protocol with non-inlined +values (e.g. `INSERT INTO [TABLE] FROM INFILE [FILE] FORMAT NATIVE`), ClickHouse +clients pull metadata from servers to pack the data. If the metadata is larger +than the `INSERT` payload, you might counterintuitively see more egress than +there is ingress from the server perspective. If this is unacceptable, consider +inlining data with `VALUES` syntax or using the HTTP protocol. + +The tables below shows how data transfer charges for egress vary across public +internet or cross-region by cloud provider and region. + +:::note +ClickHouse Cloud meters inter-region usage in terms of tiers, Tier 1 through +Tier 4, depending on the origin and destination regions. The table below shows +the tier for each combination of inter-region data transfer. In the Billing usage +screen on ClickHouse Cloud you will see data transfer usage broken out by tiers. +::: + + diff --git a/docs/cloud/manage/troubleshooting-billing-issues.md b/docs/cloud/reference/03_billing/05_billing_compliance.md similarity index 100% rename from docs/cloud/manage/troubleshooting-billing-issues.md rename to docs/cloud/reference/03_billing/05_billing_compliance.md diff --git a/docs/cloud/manage/billing/index.md b/docs/cloud/reference/03_billing/index.md similarity index 87% rename from docs/cloud/manage/billing/index.md rename to docs/cloud/reference/03_billing/index.md index f940c75a034..1a47fc98417 100644 --- a/docs/cloud/manage/billing/index.md +++ b/docs/cloud/reference/03_billing/index.md @@ -12,4 +12,4 @@ This section of the documentation covers topics related to billing, and contains | [Overview](/cloud/marketplace/marketplace-billing) | Overview and FAQ pages for marketplace billing. | | [Payment Thresholds](/cloud/billing/payment-thresholds) | Learn more about how payment thresholds work and how to adjust them. | | [Troubleshooting Billing Issues](/manage/clickhouse-cloud-billing-compliance) | Troubleshoot common billing issues. | -| [Marketplace](/cloud/manage/) | Landing page for further marketplace related topics. | +| [Marketplace](/cloud/manage/marketplace/) | Landing page for further marketplace related topics. | diff --git a/docs/cloud/reference/supported-regions.md b/docs/cloud/reference/05_supported-regions.md similarity index 98% rename from docs/cloud/reference/supported-regions.md rename to docs/cloud/reference/05_supported-regions.md index f434b8786e1..4086227f4ab 100644 --- a/docs/cloud/reference/supported-regions.md +++ b/docs/cloud/reference/05_supported-regions.md @@ -1,6 +1,6 @@ --- title: 'Supported Cloud Regions' -sidebar_label: 'Supported Cloud Regions' +sidebar_label: 'Supported Cloud regions' keywords: ['aws', 'gcp', 'google cloud', 'azure', 'cloud', 'regions'] description: 'Supported regions for ClickHouse Cloud' slug: /cloud/reference/supported-regions diff --git a/docs/cloud/manage/service-uptime.md b/docs/cloud/reference/06_service-uptime.md similarity index 95% rename from docs/cloud/manage/service-uptime.md rename to docs/cloud/reference/06_service-uptime.md index 3a31e459eaf..33397a626be 100644 --- a/docs/cloud/manage/service-uptime.md +++ b/docs/cloud/reference/06_service-uptime.md @@ -1,7 +1,7 @@ --- sidebar_label: 'Service Uptime and SLA' slug: /cloud/manage/service-uptime -title: 'Service Uptime' +title: 'Service uptime' description: 'Users can now see regional uptimes on the status page and subscribe to alerts on service disruptions.' --- diff --git a/docs/cloud/manage/settings.md b/docs/cloud/reference/08_settings.md similarity index 94% rename from docs/cloud/manage/settings.md rename to docs/cloud/reference/08_settings.md index a766ef59c13..9926c5833cb 100644 --- a/docs/cloud/manage/settings.md +++ b/docs/cloud/reference/08_settings.md @@ -1,7 +1,7 @@ --- -sidebar_label: 'Configuring Settings' +sidebar_label: 'Configuring settings' slug: /manage/settings -title: 'Configuring Settings' +title: 'Configuring settings' description: 'How to configure settings for your ClickHouse Cloud service for a specific user or role' --- diff --git a/docs/cloud/reference/09_security/_category_.json b/docs/cloud/reference/09_security/_category_.json new file mode 100644 index 00000000000..aed26fa7f7a --- /dev/null +++ b/docs/cloud/reference/09_security/_category_.json @@ -0,0 +1,5 @@ +{ + "label": "Security", + "collapsible": true, + "collapsed": true, +} \ No newline at end of file diff --git a/docs/cloud/security/audit-logging.md b/docs/cloud/reference/09_security/audit-logging.md similarity index 100% rename from docs/cloud/security/audit-logging.md rename to docs/cloud/reference/09_security/audit-logging.md diff --git a/docs/cloud/reference/09_security/privacy_and_compliance/_category_.json b/docs/cloud/reference/09_security/privacy_and_compliance/_category_.json new file mode 100644 index 00000000000..99beeb3e924 --- /dev/null +++ b/docs/cloud/reference/09_security/privacy_and_compliance/_category_.json @@ -0,0 +1,6 @@ +{ + "label": "Privacy and compliance", + "collapsible": true, + "collapsed": true, + "link": { "type": "doc", "id": "cloud/reference/security/privacy_and_compliance/index" } +} \ No newline at end of file diff --git a/docs/cloud/security/compliance-overview.md b/docs/cloud/reference/09_security/privacy_and_compliance/compliance-overview.md similarity index 96% rename from docs/cloud/security/compliance-overview.md rename to docs/cloud/reference/09_security/privacy_and_compliance/compliance-overview.md index 4653c0f09c1..de8b7a9b90e 100644 --- a/docs/cloud/security/compliance-overview.md +++ b/docs/cloud/reference/09_security/privacy_and_compliance/compliance-overview.md @@ -1,8 +1,7 @@ --- -sidebar_label: 'Security and Compliance' -slug: /cloud/security/security-and-compliance -title: 'Security and Compliance' -description: 'This page describes the security and compliance measures implemented by ClickHouse Cloud to protect customer data.' +title: 'Security and compliance reports' +slug: /cloud/security/compliance-overview +description: 'Overview of ClickHouse Cloud security and compliance certifications including SOC 2, ISO 27001, U.S. DPF, and HIPAA' --- import BetaBadge from '@theme/badges/BetaBadge'; diff --git a/docs/cloud/security/privacy-compliance-overview.md b/docs/cloud/reference/09_security/privacy_and_compliance/index.md similarity index 83% rename from docs/cloud/security/privacy-compliance-overview.md rename to docs/cloud/reference/09_security/privacy_and_compliance/index.md index e47d422c0a8..9b0b8594fc8 100644 --- a/docs/cloud/security/privacy-compliance-overview.md +++ b/docs/cloud/reference/09_security/privacy_and_compliance/index.md @@ -11,5 +11,5 @@ This section contains the following pages: | Page | Description | |----------------------------------------------------------------------------|--------------------------------------------------------------| -| [Security and Compliance](/cloud/security/security-and-compliance) | Security reports and privacy compliance of ClickHouse Cloud. | +| [Security and Compliance](/cloud/security/compliance-overview) | Security reports and privacy compliance of ClickHouse Cloud. | | [Personal Data Access](/cloud/security/personal-data-access) | Information on how to access your personal data. | diff --git a/docs/cloud/security/personal-data-access.md b/docs/cloud/reference/09_security/privacy_and_compliance/personal-data-access.md similarity index 98% rename from docs/cloud/security/personal-data-access.md rename to docs/cloud/reference/09_security/privacy_and_compliance/personal-data-access.md index bcf4514b301..3bdc8ca3302 100644 --- a/docs/cloud/security/personal-data-access.md +++ b/docs/cloud/reference/09_security/privacy_and_compliance/personal-data-access.md @@ -1,7 +1,7 @@ --- -sidebar_label: 'Personal Data Access' +sidebar_label: 'Personal data access' slug: /cloud/security/personal-data-access -title: 'Personal Data Access' +title: 'Personal data access' description: 'As a registered user, ClickHouse allows you to view and manage your personal account data, including contact information.' --- diff --git a/docs/cloud/manage/account-close.md b/docs/cloud/reference/10_account-close.md similarity index 98% rename from docs/cloud/manage/account-close.md rename to docs/cloud/reference/10_account-close.md index ac9a79eeeea..021345d4a94 100644 --- a/docs/cloud/manage/account-close.md +++ b/docs/cloud/reference/10_account-close.md @@ -1,11 +1,12 @@ --- -sidebar_label: 'Delete Account' +sidebar_label: 'Account closure' slug: /cloud/manage/close_account -title: 'Account Close & Deletion' +title: 'Account closure and deletion' description: 'We know there are circumstances that sometimes necessitate account closure. This guide will help you through the process.' --- ## Account closure and deletion {#account-close--deletion} + Our goal is to help you be successful in your project. If you have questions that are not answered on this site or need help evaluating a unique use case, please contact us at [support@clickhouse.com](mailto:support@clickhouse.com). diff --git a/docs/cloud/manage/_snippets/_network_transfer_rates.md b/docs/cloud/reference/_snippets/_network_transfer_rates.md similarity index 100% rename from docs/cloud/manage/_snippets/_network_transfer_rates.md rename to docs/cloud/reference/_snippets/_network_transfer_rates.md diff --git a/docs/cloud/reference/release-notes-index.md b/docs/cloud/reference/release-notes-index.md deleted file mode 100644 index c7e32f843b5..00000000000 --- a/docs/cloud/reference/release-notes-index.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -slug: /cloud/reference/changelogs/release-notes -title: 'Cloud Release Notes' -description: 'Landing page for Cloud release notes' ---- - - - - - diff --git a/docs/cloud/security/_category_.yml b/docs/cloud/security/_category_.yml deleted file mode 100644 index b7253753fd5..00000000000 --- a/docs/cloud/security/_category_.yml +++ /dev/null @@ -1,6 +0,0 @@ -label: 'Cloud Security' -collapsible: true -collapsed: true -link: - type: generated-index - title: Cloud Security diff --git a/docs/cloud/security/index.md b/docs/cloud/security/index.md deleted file mode 100644 index b6a2d56ab1b..00000000000 --- a/docs/cloud/security/index.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -slug: /cloud/security -keywords: ['Cloud', 'Security'] -title: 'Overview' -hide_title: true -description: 'Landing page for ClickHouse Cloud Security' ---- - -# ClickHouse Cloud security - -This section delves into security in ClickHouse Cloud and contains the following pages: - -| Page | Description | -|---------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Shared Responsibility Model](shared-responsibility-model.md) | Information on the security features offered for each service type. | -| [Cloud Access Management](cloud-access-management/index.md) | Information on access control, authentication, SSO setup, common access management queries and how to invite new users. | -| [Connectivity](connectivity-overview.md) | Information on setting IP filters, private networking, secure access of S3 data and Cloud IP addresses. | -| [Enhanced Encryption](cmek.md) | Data at rest is encrypted by default using cloud provider-managed AES 256 keys. Customers may enable Transparent Data Encryption (TDE) to provide an additional layer of protection for service data. | -| [Audit Logging](audit-logging.md) | A guide to audit logging in ClickHouse Cloud. | -| [Privacy and Compliance](privacy-compliance-overview.md) | Information on security and compliance of ClickHouse Cloud, a guide on how to view and correct your personal information. | diff --git a/docs/faq/integration/index.md b/docs/faq/integration/index.md index 76939b7bfb3..bbe8e93f276 100644 --- a/docs/faq/integration/index.md +++ b/docs/faq/integration/index.md @@ -13,7 +13,7 @@ description: 'Landing page listing questions related to integrating ClickHouse w - [How to import JSON into ClickHouse?](/integrations/data-ingestion/data-formats/json/intro.md) - [How do I connect Kafka to ClickHouse?](/integrations/data-ingestion/kafka/index.md) - [Can I connect my Java application to ClickHouse?](/integrations/data-ingestion/dbms/jdbc-with-clickhouse.md) -- [Can ClickHouse read tables from MySQL?](/integrations/data-ingestion/dbms/mysql/index.md) +- [Can ClickHouse read tables from MySQL?](/integrations/mysql) - [Can ClickHouse read tables from PostgreSQL](/integrations/data-ingestion/dbms/postgresql/connecting-to-postgresql.md) - [What if I have a problem with encodings when connecting to Oracle via ODBC?](/faq/integration/oracle-odbc.md) diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index ca8b2ee1c18..02ff69186f1 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -23,4 +23,34 @@ functions in ClickHouse. The sample datasets include: by https://github.com/ClickHouse/clickhouse-docs/blob/main/scripts/autogenerate-table-of-contents.sh --> +| Page | Description | +|-----|-----| +| [NOAA Global Historical Climatology Network](/getting-started/example-datasets/noaa) | 2.5 billion rows of climate data for the last 120 yrs | +| [Writing Queries in ClickHouse using GitHub Data](/getting-started/example-datasets/github) | Dataset containing all of the commits and changes for the ClickHouse repository | +| [Analyzing Stack Overflow data with ClickHouse](/getting-started/example-datasets/stackoverflow) | Analyzing Stack Overflow data with ClickHouse | +| [The UK property prices dataset](/getting-started/example-datasets/uk-price-paid) | Learn how to use projections to improve the performance of queries that you run frequently using the UK property dataset, which contains data about prices paid for real-estate property in England and Wales | +| [Taiwan Historical Weather Datasets](/getting-started/example-datasets/tw-weather) | 131 million rows of weather observation data for the last 128 yrs | +| [New York Taxi Data](/getting-started/example-datasets/nyc-taxi) | Data for billions of taxi and for-hire vehicle (Uber, Lyft, etc.) trips originating in New York City since 2009 | +| [Geo Data using the Cell Tower Dataset](/getting-started/example-datasets/cell-towers) | Learn how to load OpenCelliD data into ClickHouse, connect Apache Superset to ClickHouse and build a dashboard based on data | +| [Amazon Customer Review](/getting-started/example-datasets/amazon-reviews) | Over 150M customer reviews of Amazon products | +| [AMPLab Big Data Benchmark](/getting-started/example-datasets/amplab-benchmark) | A benchmark dataset used for comparing the performance of data warehousing solutions. | +| [Anonymized Web Analytics](/getting-started/example-datasets/metrica) | Dataset consisting of two tables containing anonymized web analytics data with hits and visits | +| [Brown University Benchmark](/getting-started/example-datasets/brown-benchmark) | A new analytical benchmark for machine-generated log data | +| [COVID-19 Open-Data](/getting-started/example-datasets/covid19) | COVID-19 Open-Data is a large, open-source database of COVID-19 epidemiological data and related factors like demographics, economics, and government responses | +| [dbpedia dataset](/getting-started/example-datasets/dbpedia-dataset) | Dataset containing 1 million articles from Wikipedia and their vector embeddings | +| [Environmental Sensors Data](/getting-started/example-datasets/environmental-sensors) | Over 20 billion records of data from Sensor.Community, a contributors-driven global sensor network that creates Open Environmental Data. | +| [Foursquare places](/getting-started/example-datasets/foursquare-places) | Dataset with over 100 million records containing information about places on a map, such as shops, restaurants, parks, playgrounds, and monuments. | +| [GitHub Events Dataset](/getting-started/example-datasets/github-events) | Dataset containing all events on GitHub from 2011 to Dec 6 2020, with a size of 3.1 billion records. | +| [Hacker News dataset](/getting-started/example-datasets/hacker-news) | Dataset containing 28 million rows of hacker news data. | +| [LAION 5B dataset](/getting-started/example-datasets/laion-5b-dataset) | Dataset containing 100 million vectors from the LAION 5B dataset | +| [Laion-400M dataset](/getting-started/example-datasets/laion-400m-dataset) | Dataset containing 400 million images with English image captions | +| [New York Public Library "What's on the Menu?" Dataset](/getting-started/example-datasets/menus) | Dataset containing 1.3 million records of historical data on the menus of hotels, restaurants and cafes with the dishes along with their prices. | +| [NYPD Complaint Data](/getting-started/example-datasets/nypd_complaint_data) | Ingest and query Tab Separated Value data in 5 steps | +| [OnTime](/getting-started/example-datasets/ontime) | Dataset containing the on-time performance of airline flights | +| [Star Schema Benchmark (SSB, 2009)](/getting-started/example-datasets/star-schema) | The Star Schema Benchmark (SSB) data set and queries | +| [Terabyte Click Logs from Criteo](/getting-started/example-datasets/criteo) | A terabyte of Click Logs from Criteo | +| [TPC-DS (2012)](/getting-started/example-datasets/tpcds) | The TPC-DS benchmark data set and queries. | +| [TPC-H (1999)](/getting-started/example-datasets/tpch) | The TPC-H benchmark data set and queries. | +| [WikiStat](/getting-started/example-datasets/wikistat) | Explore the WikiStat dataset containing 0.5 trillion records. | +| [YouTube dataset of dislikes](/getting-started/example-datasets/youtube-dislikes) | A collection is dislikes of YouTube videos. | diff --git a/docs/guides/best-practices/_snippets/_performance_optimizations_table_of_contents.md b/docs/guides/best-practices/_snippets/_performance_optimizations_table_of_contents.md new file mode 100644 index 00000000000..3cbd396bb55 --- /dev/null +++ b/docs/guides/best-practices/_snippets/_performance_optimizations_table_of_contents.md @@ -0,0 +1,17 @@ +| Topic | Description | +|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Query optimization guide](/optimize/query-optimization) | Start here for query optimization fundamentals, covering common scenarios and performance techniques to improve query execution speed. | +| [Primary indexes advanced guide](/guides/best-practices/sparse-primary-indexes) | Deep dive into ClickHouse's unique sparse primary indexing system, how it differs from traditional databases, and best practices for optimal indexing strategies. | +| [Query parallelism](/optimize/query-parallelism) | Learn how ClickHouse parallelizes query execution using processing lanes and `max_threads` settings, including how to inspect and optimize parallel execution. | +| [Partitioning key](/optimize/partitioning-key) | Master partition key selection to dramatically improve query performance by enabling efficient data segment pruning and avoiding common partitioning pitfalls. | +| [Data skipping indexes](/optimize/skipping-indexes) | Apply secondary indexes strategically to skip irrelevant data blocks and accelerate filtered queries on non-primary key columns. | +| [`PREWHERE` optimization](/optimize/prewhere) | Understand how `PREWHERE` automatically reduces I/O by filtering data before reading unnecessary columns, plus how to monitor its effectiveness. | +| [Bulk inserts](/optimize/bulk-inserts) | Maximize ingestion throughput and reduce resource overhead by batching data insertions effectively. | +| [Asynchronous inserts](/optimize/asynchronous-inserts) | Improve insert performance by leveraging server-side batching to reduce client-side complexity and increase throughput for high-frequency insertions. | +| [Avoid mutations](/optimize/avoid-mutations) | Design append-only workflows that eliminate costly `UPDATE` and `DELETE` operations while maintaining data accuracy and performance. | +| [Avoid nullable columns](/optimize/avoid-nullable-columns) | Reduce storage overhead and improve query performance by using default values instead of nullable columns where possible. | +| [Avoid `OPTIMIZE FINAL`](/optimize/avoidoptimizefinal) | Understand when you should and should not use `OPTIMIZE TABLE FINAL` | +| [Analyzer](/operations/analyzer) | Leverage ClickHouse's new query analyzer to identify performance bottlenecks and optimize query execution plans for better efficiency. | +| [Query profiling](/operations/optimizing-performance/sampling-query-profiler) | Use the sampling query profiler to analyze query execution patterns, identify performance hot spots, and optimize resource usage. | +| [Query cache](/operations/query-cache) | Accelerate frequently executed `SELECT` queries by enabling and configuring ClickHouse's built-in query result caching. | +| [Testing hardware](/operations/performance-test) | Run ClickHouse performance benchmarks on any server without installation to evaluate hardware capabilities. (Not applicable to ClickHouse Cloud) | \ No newline at end of file diff --git a/docs/guides/best-practices/index.md b/docs/guides/best-practices/index.md index 0c52281492f..ef320eaf03c 100644 --- a/docs/guides/best-practices/index.md +++ b/docs/guides/best-practices/index.md @@ -5,26 +5,12 @@ description: 'Overview page of Performance and Optimizations' title: 'Performance and Optimizations' --- -# Performance and optimizations +import TableOfContents from '@site/docs/guides/best-practices/_snippets/_performance_optimizations_table_of_contents.md'; + +# Performance and Optimizations This section contains tips and best practices for improving performance with ClickHouse. We recommend users read [Core Concepts](/parts) as a precursor to this section, which covers the main concepts required to improve performance. -| Topic | Description | -|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Query Optimization Guide](/optimize/query-optimization) | A good place to start for query optimization, this simple guide describes common scenarios of how to use different performance and optimization techniques to improve query performance. | -| [Primary Indexes Advanced Guide](/guides/best-practices/sparse-primary-indexes) | A deep dive into ClickHouse indexing including how it differs from other DB systems, how ClickHouse builds and uses a table's spare primary index and what some of the best practices are for indexing in ClickHouse. | -| [Query Parallelism](/optimize/query-parallelism) | Explains how ClickHouse parallelizes query execution using processing lanes and the max_threads setting. Covers how data is distributed across lanes, how max_threads is applied, when it isn't fully used, and how to inspect execution with tools like EXPLAIN and trace logs. | -| [Partitioning Key](/optimize/partitioning-key) | Delves into ClickHouse partition key optimization. Explains how choosing the right partition key can significantly improve query performance by allowing ClickHouse to quickly locate relevant data segments. Covers best practices for selecting efficient partition keys and potential pitfalls to avoid. | -| [Data Skipping Indexes](/optimize/skipping-indexes) | Explains data skipping indexes as a way to optimize performance. | -| [PREWHERE Optimization](/optimize/prewhere) | Explains how PREWHERE reduces I/O by avoiding reading unnecessary column data. Shows how it's applied automatically, how the filtering order is chosen, and how to monitor it using EXPLAIN and logs. | -| [Bulk Inserts](/optimize/bulk-inserts) | Explains the benefits of using bulk inserts in ClickHouse. | -| [Asynchronous Inserts](/optimize/asynchronous-inserts) | Focuses on ClickHouse's asynchronous inserts feature. It likely explains how asynchronous inserts work (batching data on the server for efficient insertion) and their benefits (improved performance by offloading insert processing). It might also cover enabling asynchronous inserts and considerations for using them effectively in your ClickHouse environment. | -| [Avoid Mutations](/optimize/avoid-mutations) | Discusses the importance of avoiding mutations (updates and deletes) in ClickHouse. It recommends using append-only inserts for optimal performance and suggests alternative approaches for handling data changes. | -| [Avoid nullable columns](/optimize/avoid-nullable-columns) | Discusses why you may want to avoid nullable columns to save space and increase performance. Demonstrates how to set a default value for a column. | -| [Avoid `OPTIMIZE FINAL`](/optimize/avoidoptimizefinal) | Explains how the `OPTIMIZE TABLE ... FINAL` query is resource-intensive and suggests alternative approaches to optimize ClickHouse performance. | -| [Analyzer](/operations/analyzer) | Looks at the ClickHouse Analyzer, a tool for analyzing and optimizing queries. Discusses how the Analyzer works, its benefits (e.g., identifying performance bottlenecks), and how to use it to improve your ClickHouse queries' efficiency. | -| [Query Profiling](/operations/optimizing-performance/sampling-query-profiler) | Explains ClickHouse's Sampling Query Profiler, a tool that helps analyze query execution. | -| [Query Cache](/operations/query-cache) | Details ClickHouse's Query Cache, a feature that aims to improve performance by caching the results of frequently executed `SELECT` queries. | -| [Testing Hardware](/operations/performance-test) | How to run a basic ClickHouse performance test on any server without installation of ClickHouse packages. (Not applicable to ClickHouse Cloud) | + \ No newline at end of file diff --git a/docs/integrations/data-ingestion/clickpipes/kafka/index.md b/docs/integrations/data-ingestion/clickpipes/kafka/index.md index 830168afc93..e8b4e027cc2 100644 --- a/docs/integrations/data-ingestion/clickpipes/kafka/index.md +++ b/docs/integrations/data-ingestion/clickpipes/kafka/index.md @@ -6,4 +6,11 @@ title: 'Kafka ClickPipes' --- +| Page | Description | +|-----|-----| +| [Reference](/integrations/clickpipes/kafka/reference) | Details supported formats, sources, delivery semantics, authentication and experimental features supported by Kafka ClickPipes | +| [Schema registries for Kafka ClickPipe](/integrations/clickpipes/kafka/schema-registries) | How to integrate for ClickPipes with a schema registry for schema management | +| [Creating your first Kafka ClickPipe](/integrations/clickpipes/kafka/create-your-first-kafka-clickpipe) | Step-by-step guide to creating your first Kafka ClickPipe. | +| [Kafka ClickPipes FAQ](/integrations/clickpipes/kafka/faq) | Frequently asked questions about ClickPipes for Kafka | +| [Best practices](/integrations/clickpipes/kafka/best-practices) | Details best practices to follow when working with Kafka ClickPipes | \ No newline at end of file diff --git a/docs/integrations/data-ingestion/dbms/jdbc-with-clickhouse.md b/docs/integrations/data-ingestion/dbms/jdbc-with-clickhouse.md index 981a11bad97..8ac322ecd5d 100644 --- a/docs/integrations/data-ingestion/dbms/jdbc-with-clickhouse.md +++ b/docs/integrations/data-ingestion/dbms/jdbc-with-clickhouse.md @@ -17,7 +17,7 @@ import Jdbc03 from '@site/static/images/integrations/data-ingestion/dbms/jdbc-03 # Connecting ClickHouse to external data sources with JDBC :::note -Using JDBC requires the ClickHouse JDBC bridge, so you will need to use `clickhouse-local` on a local machine to stream the data from your database to ClickHouse Cloud. Visit the [**Using clickhouse-local**](/integrations/migration/clickhouse-local-etl.md#example-2-migrating-from-mysql-to-clickhouse-cloud-with-the-jdbc-bridge) page in the **Migrate** section of the docs for details. +Using JDBC requires the ClickHouse JDBC bridge, so you will need to use `clickhouse-local` on a local machine to stream the data from your database to ClickHouse Cloud. Visit the [**Using clickhouse-local**](/cloud/migration/clickhouse-local#example-2-migrating-from-mysql-to-clickhouse-cloud-with-the-jdbc-bridge) page in the **Migrate** section of the docs for details. ::: **Overview:** The ClickHouse JDBC Bridge in combination with the [jdbc table function](/sql-reference/table-functions/jdbc.md) or the [JDBC table engine](/engines/table-engines/integrations/jdbc.md) allows ClickHouse to access data from any external data source for which a JDBC driver is available: diff --git a/docs/integrations/data-ingestion/dbms/mysql/index.md b/docs/integrations/data-ingestion/dbms/mysql/index.md deleted file mode 100644 index 47594a4417c..00000000000 --- a/docs/integrations/data-ingestion/dbms/mysql/index.md +++ /dev/null @@ -1,152 +0,0 @@ ---- -sidebar_label: 'MySQL' -sidebar_position: 10 -slug: /integrations/connecting-to-mysql -description: 'The MySQL table engine allows you to connect ClickHouse to MySQL.' -keywords: ['mysql'] -title: 'Integrating MySQL with ClickHouse' -show_related_blogs: true ---- - -import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; -import ExperimentalBadge from '@theme/badges/ExperimentalBadge'; - -# Integrating MySQL with ClickHouse - -This page covers using the `MySQL` table engine, for reading from a MySQL table. - -:::note -For ClickHouse Cloud, you can also use the [MySQL ClickPipe](/integrations/clickpipes/mysql) (currently in public beta) to easily move data from your MySQL tables to ClickHouse. -::: - -## Connecting ClickHouse to MySQL using the MySQL Table Engine {#connecting-clickhouse-to-mysql-using-the-mysql-table-engine} - -The `MySQL` table engine allows you to connect ClickHouse to MySQL. **SELECT** and **INSERT** statements can be made in either ClickHouse or in the MySQL table. This article illustrates the basic methods of how to use the `MySQL` table engine. - -### 1. Configure MySQL {#1-configure-mysql} - -1. Create a database in MySQL: - ```sql - CREATE DATABASE db1; - ``` - -2. Create a table: - ```sql - CREATE TABLE db1.table1 ( - id INT, - column1 VARCHAR(255) - ); - ``` - -3. Insert sample rows: - ```sql - INSERT INTO db1.table1 - (id, column1) - VALUES - (1, 'abc'), - (2, 'def'), - (3, 'ghi'); - ``` - -4. Create a user to connect from ClickHouse: - ```sql - CREATE USER 'mysql_clickhouse'@'%' IDENTIFIED BY 'Password123!'; - ``` - -5. Grant privileges as needed. (For demonstration purposes, the `mysql_clickhouse` user is granted admin privileges.) - ```sql - GRANT ALL PRIVILEGES ON *.* TO 'mysql_clickhouse'@'%'; - ``` - -:::note -If you are using this feature in ClickHouse Cloud, you may need the to allow the ClickHouse Cloud IP addresses to access your MySQL instance. -Check the ClickHouse [Cloud Endpoints API](//cloud/get-started/query-endpoints.md) for egress traffic details. -::: - -### 2. Define a Table in ClickHouse {#2-define-a-table-in-clickhouse} - -1. Now let's create a ClickHouse table that uses the `MySQL` table engine: - ```sql - CREATE TABLE mysql_table1 ( - id UInt64, - column1 String - ) - ENGINE = MySQL('mysql-host.domain.com','db1','table1','mysql_clickhouse','Password123!') - ``` - - The minimum parameters are: - - |parameter|Description |example | - |---------|----------------------------|---------------------| - |host |hostname or IP |mysql-host.domain.com| - |database |mysql database name |db1 | - |table |mysql table name |table1 | - |user |username to connect to mysql|mysql_clickhouse | - |password |password to connect to mysql|Password123! | - - :::note - View the [MySQL table engine](/engines/table-engines/integrations/mysql.md) doc page for a complete list of parameters. - ::: - -### 3. Test the Integration {#3-test-the-integration} - -1. In MySQL, insert a sample row: - ```sql - INSERT INTO db1.table1 - (id, column1) - VALUES - (4, 'jkl'); - ``` - -2. Notice the existing rows from the MySQL table are in the ClickHouse table, along with the new row you just added: - ```sql - SELECT - id, - column1 - FROM mysql_table1 - ``` - - You should see 4 rows: - ```response - Query id: 6d590083-841e-4e95-8715-ef37d3e95197 - - ┌─id─┬─column1─┐ - │ 1 │ abc │ - │ 2 │ def │ - │ 3 │ ghi │ - │ 4 │ jkl │ - └────┴─────────┘ - - 4 rows in set. Elapsed: 0.044 sec. - ``` - -3. Let's add a row to the ClickHouse table: - ```sql - INSERT INTO mysql_table1 - (id, column1) - VALUES - (5,'mno') - ``` - -4. Notice the new row appears in MySQL: - ```bash - mysql> select id,column1 from db1.table1; - ``` - - You should see the new row: - ```response - +------+---------+ - | id | column1 | - +------+---------+ - | 1 | abc | - | 2 | def | - | 3 | ghi | - | 4 | jkl | - | 5 | mno | - +------+---------+ - 5 rows in set (0.01 sec) - ``` - -### Summary {#summary} - -The `MySQL` table engine allows you to connect ClickHouse to MySQL to exchange data back and forth. For more details, be sure to check out the documentation page for the [MySQL table engine](/sql-reference/table-functions/mysql.md). diff --git a/docs/integrations/data-ingestion/kafka/index.md b/docs/integrations/data-ingestion/kafka/index.md index bde0ca73608..ef85ea20227 100644 --- a/docs/integrations/data-ingestion/kafka/index.md +++ b/docs/integrations/data-ingestion/kafka/index.md @@ -37,7 +37,7 @@ This is the recommended option if you're a ClickHouse Cloud user. ClickPipes is * Optimized for ClickHouse Cloud, delivering blazing-fast performance * Horizontal and vertical scalability for high-throughput workloads * Built-in fault tolerance with configurable replicas and automatic retries -* Deployment and management via ClickHouse Cloud UI, [Open API](../../../cloud/manage/api/api-overview.md), or [Terraform](https://registry.terraform.io/providers/ClickHouse/clickhouse/3.3.3-alpha2/docs/resources/clickpipe) +* Deployment and management via ClickHouse Cloud UI, [Open API](/cloud/manage/api/api-overview), or [Terraform](https://registry.terraform.io/providers/ClickHouse/clickhouse/3.3.3-alpha2/docs/resources/clickpipe) * Enterprise-grade security with support for cloud-native authorization (IAM) and private connectivity (PrivateLink) * Supports a wide range of [data sources](/integrations/clickpipes/kafka/reference/), including Confluent Cloud, Amazon MSK, Redpanda Cloud, and Azure Event Hubs * Supports most common serialization formats (JSON, Avro, Protobuf coming soon!) @@ -100,6 +100,6 @@ To get started using the Kafka table engine, see the [reference documentation](. * **Custom code** - Custom code using Kafka and ClickHouse [client libraries](../../language-clients/index.md) may be appropriate in cases where custom processing of events is required. -[BYOC]: ../../../cloud/reference/byoc.md -[Cloud]: ../../../cloud-index.md +[BYOC]: /cloud/reference/byoc +[Cloud]: /cloud/get-started [Self-hosted]: ../../../intro.md diff --git a/docs/integrations/data-ingestion/redshift/index.md b/docs/integrations/data-ingestion/redshift/_snippets/_migration_guide.md similarity index 50% rename from docs/integrations/data-ingestion/redshift/index.md rename to docs/integrations/data-ingestion/redshift/_snippets/_migration_guide.md index 3e936cec37b..960120aa751 100644 --- a/docs/integrations/data-ingestion/redshift/index.md +++ b/docs/integrations/data-ingestion/redshift/_snippets/_migration_guide.md @@ -1,12 +1,3 @@ ---- -sidebar_label: 'Redshift' -slug: /integrations/redshift -description: 'Migrating Data from Redshift to ClickHouse' -title: 'Migrating data from Redshift to ClickHouse' -keywords: ['Redshift'] -show_related_blogs: true ---- - import redshiftToClickhouse from '@site/static/images/integrations/data-ingestion/redshift/redshift-to-clickhouse.png'; import push from '@site/static/images/integrations/data-ingestion/redshift/push.png'; import pull from '@site/static/images/integrations/data-ingestion/redshift/pull.png'; @@ -15,27 +6,11 @@ import s3_1 from '@site/static/images/integrations/data-ingestion/redshift/s3-1. import s3_2 from '@site/static/images/integrations/data-ingestion/redshift/s3-2.png'; import Image from '@theme/IdealImage'; -# Migrating data from Redshift to ClickHouse - -## Related content {#related-content} - -
- -
- ## Introduction {#introduction} [Amazon Redshift](https://aws.amazon.com/redshift/) is a popular cloud data warehousing solution that is part of the Amazon Web Services offerings. This guide presents different approaches to migrating data from a Redshift instance to ClickHouse. We will cover three options: -Redshift to ClickHouse Migration Options +Redshift to ClickHouse Migration Options From the ClickHouse instance standpoint, you can either: @@ -49,11 +24,11 @@ From the ClickHouse instance standpoint, you can either: We used Redshift as a data source in this tutorial. However, the migration approaches presented here are not exclusive to Redshift, and similar steps can be derived for any compatible data source. ::: -## Push data from Redshift to ClickHouse {#push-data-from-redshift-to-clickhouse} +## Push Data from Redshift to ClickHouse {#push-data-from-redshift-to-clickhouse} In the push scenario, the idea is to leverage a third-party tool or service (either custom code or an [ETL/ELT](https://en.wikipedia.org/wiki/Extract,_transform,_load#ETL_vs._ELT)) to send your data to your ClickHouse instance. For example, you can use a software like [Airbyte](https://www.airbyte.com/) to move data between your Redshift instance (as a source) and ClickHouse as a destination ([see our integration guide for Airbyte](/integrations/data-ingestion/etl-tools/airbyte-and-clickhouse.md)) -PUSH Redshift to ClickHouse +PUSH Redshift to ClickHouse ### Pros {#pros} @@ -66,11 +41,11 @@ In the push scenario, the idea is to leverage a third-party tool or service (eit * Users need to set up and maintain an ETL/ELT infrastructure. * Introduces a third-party element in the architecture which can turn into a potential scalability bottleneck. -## Pull data from Redshift to ClickHouse {#pull-data-from-redshift-to-clickhouse} +## Pull Data from Redshift to ClickHouse {#pull-data-from-redshift-to-clickhouse} In the pull scenario, the idea is to leverage the ClickHouse JDBC Bridge to connect to a Redshift cluster directly from a ClickHouse instance and perform `INSERT INTO ... SELECT` queries: -PULL from Redshift to ClickHouse +PULL from Redshift to ClickHouse ### Pros {#pros-1} @@ -89,109 +64,120 @@ Even though Redshift is based on PostgreSQL, using the ClickHouse PostgreSQL tab To use this option, you need to set up a ClickHouse JDBC Bridge. ClickHouse JDBC Bridge is a standalone Java application that handles JDBC connectivity and acts as a proxy between the ClickHouse instance and the data sources. For this tutorial, we used a pre-populated Redshift instance with a [sample database](https://docs.aws.amazon.com/redshift/latest/dg/c_sampledb.html). -1. Deploy the ClickHouse JDBC Bridge. For more details, see our user guide on [JDBC for External Data sources](/integrations/data-ingestion/dbms/jdbc-with-clickhouse.md) + + +#### Deploy ClickHouse JDBC Bridge {#deploy-clickhouse-jdbc-bridge} + +Deploy the ClickHouse JDBC Bridge. For more details, see our user guide on [JDBC for External Data sources](/integrations/data-ingestion/dbms/jdbc-with-clickhouse.md) :::note If you are using ClickHouse Cloud, you will need to run your ClickHouse JDBC Bridge on a separate environment and connect to ClickHouse Cloud using the [remoteSecure](/sql-reference/table-functions/remote/) function ::: -2. Configure your Redshift datasource for ClickHouse JDBC Bridge. For example, `/etc/clickhouse-jdbc-bridge/config/datasources/redshift.json ` - - ```json - { - "redshift-server": { - "aliases": [ - "redshift" - ], - "driverUrls": [ - "https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/2.1.0.4/redshift-jdbc42-2.1.0.4.jar" - ], - "driverClassName": "com.amazon.redshift.jdbc.Driver", - "jdbcUrl": "jdbc:redshift://redshift-cluster-1.ckubnplpz1uv.us-east-1.redshift.amazonaws.com:5439/dev", - "username": "awsuser", - "password": "", - "maximumPoolSize": 5 - } - } - ``` - -3. Once ClickHouse JDBC Bridge deployed and running, you can start querying your Redshift instance from ClickHouse - - ```sql - SELECT * - FROM jdbc('redshift', 'select username, firstname, lastname from users limit 5') - ``` - - ```response - Query id: 1b7de211-c0f6-4117-86a2-276484f9f4c0 - - ┌─username─┬─firstname─┬─lastname─┐ - │ PGL08LJI │ Vladimir │ Humphrey │ - │ XDZ38RDD │ Barry │ Roy │ - │ AEB55QTM │ Reagan │ Hodge │ - │ OWY35QYB │ Tamekah │ Juarez │ - │ MSD36KVR │ Mufutau │ Watkins │ - └──────────┴───────────┴──────────┘ - - 5 rows in set. Elapsed: 0.438 sec. - ``` - - ```sql - SELECT * - FROM jdbc('redshift', 'select count(*) from sales') - ``` - - ```response - Query id: 2d0f957c-8f4e-43b2-a66a-cc48cc96237b - - ┌──count─┐ - │ 172456 │ - └────────┘ - - 1 rows in set. Elapsed: 0.304 sec. - ``` - -4. In the following, we display importing data using an `INSERT INTO ... SELECT` statement - - ```sql - # TABLE CREATION with 3 columns - CREATE TABLE users_imported - ( - `username` String, - `firstname` String, - `lastname` String - ) - ENGINE = MergeTree - ORDER BY firstname - ``` - - ```response - Query id: c7c4c44b-cdb2-49cf-b319-4e569976ab05 - - Ok. - - 0 rows in set. Elapsed: 0.233 sec. - ``` - - ```sql - # IMPORTING DATA - INSERT INTO users_imported (*) SELECT * - FROM jdbc('redshift', 'select username, firstname, lastname from users') - ``` - - ```response - Query id: 9d3a688d-b45a-40f4-a7c7-97d93d7149f1 - - Ok. - - 0 rows in set. Elapsed: 4.498 sec. Processed 49.99 thousand rows, 2.49 MB (11.11 thousand rows/s., 554.27 KB/s.) - ``` - -## Pivot data from Redshift to ClickHouse using S3 {#pivot-data-from-redshift-to-clickhouse-using-s3} +#### Configure your Redshift datasource {#configure-your-redshift-datasource} + +Configure your Redshift datasource for ClickHouse JDBC Bridge. For example, `/etc/clickhouse-jdbc-bridge/config/datasources/redshift.json ` + +```json +{ + "redshift-server": { + "aliases": [ + "redshift" + ], + "driverUrls": [ + "https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/2.1.0.4/redshift-jdbc42-2.1.0.4.jar" + ], + "driverClassName": "com.amazon.redshift.jdbc.Driver", + "jdbcUrl": "jdbc:redshift://redshift-cluster-1.ckubnplpz1uv.us-east-1.redshift.amazonaws.com:5439/dev", + "username": "awsuser", + "password": "", + "maximumPoolSize": 5 + } +} +``` + +#### Query your Redshift instance from ClickHouse {#query-your-redshift-instance-from-clickhouse} + +Once ClickHouse JDBC Bridge deployed and running, you can start querying your Redshift instance from ClickHouse + +```sql +SELECT * +FROM jdbc('redshift', 'select username, firstname, lastname from users limit 5') +``` + +```response +Query id: 1b7de211-c0f6-4117-86a2-276484f9f4c0 + +┌─username─┬─firstname─┬─lastname─┐ +│ PGL08LJI │ Vladimir │ Humphrey │ +│ XDZ38RDD │ Barry │ Roy │ +│ AEB55QTM │ Reagan │ Hodge │ +│ OWY35QYB │ Tamekah │ Juarez │ +│ MSD36KVR │ Mufutau │ Watkins │ +└──────────┴───────────┴──────────┘ + +5 rows in set. Elapsed: 0.438 sec. +``` + +```sql +SELECT * +FROM jdbc('redshift', 'select count(*) from sales') +``` + +```response +Query id: 2d0f957c-8f4e-43b2-a66a-cc48cc96237b + +┌──count─┐ +│ 172456 │ +└────────┘ + +1 rows in set. Elapsed: 0.304 sec. +``` + +#### Import Data from Redshift to ClickHouse {#import-data-from-redshift-to-clickhouse} + +In the following, we display importing data using an `INSERT INTO ... SELECT` statement + +```sql +# TABLE CREATION with 3 columns +CREATE TABLE users_imported +( + `username` String, + `firstname` String, + `lastname` String +) +ENGINE = MergeTree +ORDER BY firstname +``` + +```response +Query id: c7c4c44b-cdb2-49cf-b319-4e569976ab05 + +Ok. + +0 rows in set. Elapsed: 0.233 sec. +``` + +```sql +INSERT INTO users_imported (*) SELECT * +FROM jdbc('redshift', 'select username, firstname, lastname from users') +``` + +```response +Query id: 9d3a688d-b45a-40f4-a7c7-97d93d7149f1 + +Ok. + +0 rows in set. Elapsed: 4.498 sec. Processed 49.99 thousand rows, 2.49 MB (11.11 thousand rows/s., 554.27 KB/s.) +``` + + + +## Pivot Data from Redshift to ClickHouse using S3 {#pivot-data-from-redshift-to-clickhouse-using-s3} In this scenario, we export data to S3 in an intermediary pivot format and, in a second step, load the data from S3 into ClickHouse. -PIVOT from Redshift using S3 +PIVOT from Redshift using S3 ### Pros {#pros-2} @@ -206,52 +192,63 @@ In this scenario, we export data to S3 in an intermediary pivot format and, in a ### Tutorial {#tutorial-1} -1. Using Redshift's [UNLOAD](https://docs.aws.amazon.com/redshift/latest/dg/r_UNLOAD.html) feature, export the data into a an existing private S3 bucket: + + +#### Export data into an S3 bucket using UNLOAD {#export-data-into-an-s3-bucket-using-unload} + +Using Redshift's [UNLOAD](https://docs.aws.amazon.com/redshift/latest/dg/r_UNLOAD.html) feature, export the data into an existing private S3 bucket: - UNLOAD from Redshift to S3 +UNLOAD from Redshift to S3 - It will generate part files containing the raw data in S3 +It will generate part files containing the raw data in S3 - Data in S3 +Data in S3 -2. Create the table in ClickHouse: +#### Create the table in ClickHouse {#create-the-table-in-clickhouse} - ```sql - CREATE TABLE users - ( - username String, - firstname String, - lastname String - ) - ENGINE = MergeTree - ORDER BY username - ``` +Create the table in ClickHouse: - Alternatively, ClickHouse can try to infer the table structure using `CREATE TABLE ... EMPTY AS SELECT`: +```sql +CREATE TABLE users +( + username String, + firstname String, + lastname String +) +ENGINE = MergeTree +ORDER BY username +``` - ```sql - CREATE TABLE users - ENGINE = MergeTree ORDER BY username - EMPTY AS - SELECT * FROM s3('https://your-bucket.s3.amazonaws.com/unload/users/*', '', '', 'CSV') - ``` +Alternatively, ClickHouse can try to infer the table structure using `CREATE TABLE ... EMPTY AS SELECT`: - This works especially well when the data is in a format that contains information about data types, like Parquet. +```sql +CREATE TABLE users +ENGINE = MergeTree ORDER BY username +EMPTY AS +SELECT * FROM s3('https://your-bucket.s3.amazonaws.com/unload/users/*', '', '', 'CSV') +``` -3. Load the S3 files into ClickHouse using an `INSERT INTO ... SELECT` statement: - ```sql - INSERT INTO users SELECT * - FROM s3('https://your-bucket.s3.amazonaws.com/unload/users/*', '', '', 'CSV') - ``` +This works especially well when the data is in a format that contains information about data types, like Parquet. - ```response - Query id: 2e7e219a-6124-461c-8d75-e4f5002c8557 +#### Load S3 files into ClickHouse {#load-s3-files-into-clickhouse} - Ok. +Load the S3 files into ClickHouse using an `INSERT INTO ... SELECT` statement: - 0 rows in set. Elapsed: 0.545 sec. Processed 49.99 thousand rows, 2.34 MB (91.72 thousand rows/s., 4.30 MB/s.) - ``` +```sql +INSERT INTO users SELECT * +FROM s3('https://your-bucket.s3.amazonaws.com/unload/users/*', '', '', 'CSV') +``` + +```response +Query id: 2e7e219a-6124-461c-8d75-e4f5002c8557 + +Ok. + +0 rows in set. Elapsed: 0.545 sec. Processed 49.99 thousand rows, 2.34 MB (91.72 thousand rows/s., 4.30 MB/s.) +``` :::note This example used CSV as the pivot format. However, for production workloads we recommend Apache Parquet as the best option for large migrations since it comes with compression and can save some storage costs while reducing transfer times. (By default, each row group is compressed using SNAPPY). ClickHouse also leverages Parquet's column orientation to speed up data ingestion. ::: + + \ No newline at end of file diff --git a/docs/integrations/data-sources/mysql.md b/docs/integrations/data-sources/mysql.md index 7d60df1a874..56eebe31735 100644 --- a/docs/integrations/data-sources/mysql.md +++ b/docs/integrations/data-sources/mysql.md @@ -6,6 +6,145 @@ hide_title: true description: 'Page describing MySQL integration' --- -import MySQL from '@site/docs/integrations/data-ingestion/dbms/mysql/index.md'; +import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge'; +import ExperimentalBadge from '@theme/badges/ExperimentalBadge'; - +# Integrating MySQL with ClickHouse + +This page covers using the `MySQL` table engine, for reading from a MySQL table. + +:::note +For ClickHouse Cloud, you can also use the [MySQL ClickPipe](/integrations/clickpipes/mysql) (currently in public beta) to easily move data from your MySQL tables to ClickHouse. +::: + +## Connecting ClickHouse to MySQL using the MySQL Table Engine {#connecting-clickhouse-to-mysql-using-the-mysql-table-engine} + +The `MySQL` table engine allows you to connect ClickHouse to MySQL. **SELECT** and **INSERT** statements can be made in either ClickHouse or in the MySQL table. This article illustrates the basic methods of how to use the `MySQL` table engine. + +### 1. Configure MySQL {#1-configure-mysql} + +1. Create a database in MySQL: + ```sql + CREATE DATABASE db1; + ``` + +2. Create a table: + ```sql + CREATE TABLE db1.table1 ( + id INT, + column1 VARCHAR(255) + ); + ``` + +3. Insert sample rows: + ```sql + INSERT INTO db1.table1 + (id, column1) + VALUES + (1, 'abc'), + (2, 'def'), + (3, 'ghi'); + ``` + +4. Create a user to connect from ClickHouse: + ```sql + CREATE USER 'mysql_clickhouse'@'%' IDENTIFIED BY 'Password123!'; + ``` + +5. Grant privileges as needed. (For demonstration purposes, the `mysql_clickhouse` user is granted admin privileges.) + ```sql + GRANT ALL PRIVILEGES ON *.* TO 'mysql_clickhouse'@'%'; + ``` + +:::note +If you are using this feature in ClickHouse Cloud, you may need the to allow the ClickHouse Cloud IP addresses to access your MySQL instance. +Check the ClickHouse [Cloud Endpoints API](//cloud/get-started/query-endpoints.md) for egress traffic details. +::: + +### 2. Define a Table in ClickHouse {#2-define-a-table-in-clickhouse} + +1. Now let's create a ClickHouse table that uses the `MySQL` table engine: + ```sql + CREATE TABLE mysql_table1 ( + id UInt64, + column1 String + ) + ENGINE = MySQL('mysql-host.domain.com','db1','table1','mysql_clickhouse','Password123!') + ``` + +The minimum parameters are: + +|parameter|Description |example | + |---------|----------------------------|---------------------| +|host |hostname or IP |mysql-host.domain.com| +|database |mysql database name |db1 | +|table |mysql table name |table1 | +|user |username to connect to mysql|mysql_clickhouse | +|password |password to connect to mysql|Password123! | + +:::note +View the [MySQL table engine](/engines/table-engines/integrations/mysql.md) doc page for a complete list of parameters. +::: + +### 3. Test the Integration {#3-test-the-integration} + +1. In MySQL, insert a sample row: + ```sql + INSERT INTO db1.table1 + (id, column1) + VALUES + (4, 'jkl'); + ``` + +2. Notice the existing rows from the MySQL table are in the ClickHouse table, along with the new row you just added: + ```sql + SELECT + id, + column1 + FROM mysql_table1 + ``` + +You should see 4 rows: + ```response + Query id: 6d590083-841e-4e95-8715-ef37d3e95197 + + ┌─id─┬─column1─┐ + │ 1 │ abc │ + │ 2 │ def │ + │ 3 │ ghi │ + │ 4 │ jkl │ + └────┴─────────┘ + + 4 rows in set. Elapsed: 0.044 sec. + ``` + +3. Let's add a row to the ClickHouse table: + ```sql + INSERT INTO mysql_table1 + (id, column1) + VALUES + (5,'mno') + ``` + +4. Notice the new row appears in MySQL: + ```bash + mysql> select id,column1 from db1.table1; + ``` + +You should see the new row: + ```response + +------+---------+ + | id | column1 | + +------+---------+ + | 1 | abc | + | 2 | def | + | 3 | ghi | + | 4 | jkl | + | 5 | mno | + +------+---------+ + 5 rows in set (0.01 sec) + ``` + +### Summary {#summary} + +The `MySQL` table engine allows you to connect ClickHouse to MySQL to exchange data back and forth. For more details, be sure to check out the documentation page for the [MySQL table engine](/sql-reference/table-functions/mysql.md). diff --git a/docs/integrations/migration/_category_.yml b/docs/integrations/migration/_category_.yml deleted file mode 100644 index a0f40febf2e..00000000000 --- a/docs/integrations/migration/_category_.yml +++ /dev/null @@ -1,8 +0,0 @@ -position: 500 -label: 'Migration' -collapsible: true -collapsed: true -link: - type: generated-index - title: Migration - slug: /cloud/migration diff --git a/docs/integrations/migration/index.md b/docs/integrations/migration/index.md deleted file mode 100644 index f58f3bb14f8..00000000000 --- a/docs/integrations/migration/index.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -sidebar_label: 'Overview' -sidebar_position: 1 -keywords: ['clickhouse', 'migrate', 'migration', 'migrating', 'data'] -description: 'Migrating To Cloud Table Of Contents' -title: 'Migrating To Cloud' -slug: /integrations/migration ---- - -This section of the docs explores how you can migrate from ClickHouse to ClickHouse Cloud. -Take a look at the pages below for further information. - -| Page | Description | -|--------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Overview](/integrations/migration/overview) | Provides an overview of the available ways to migrate from ClickHouse to ClickHouse Cloud | -| [Migrating to ClickHouse using clickhouse-local](/cloud/migration/clickhouse-local) | Learn how to migrate to ClickHouse using clickhouse-local. | | -| [ClickHouse to ClickHouse Cloud](/cloud/migration/clickhouse-to-cloud) | This guide will show how to migrate from a self-managed ClickHouse server to ClickHouse Cloud, and also how to migrate between ClickHouse Cloud services. | -| [Using a 3rd-party ETL Tool](/cloud/migration/etl-tool-to-clickhouse) | Learn more about the popular ETL and ELT tools for moving data from an external data source into ClickHouse. | -| [Object Storage to ClickHouse Cloud](/integrations/migration/object-storage-to-clickhouse) | Learn how to use various table functions to import data from Cloud Object Storage into ClickHouse Cloud. | -| [Upload a CSV File](/cloud/migrate/upload-a-csv-file) | Learn how to upload data to ClickHouse Cloud using a CSV or TSV file. | diff --git a/docs/migrations/index.md b/docs/migrations/index.md deleted file mode 100644 index 3404415cac1..00000000000 --- a/docs/migrations/index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -slug: migrations -title: 'Migrations' -pagination_prev: null -pagination_next: null -description: 'Landing page for the migrations section' ---- - -| Page | Description | -|------------------------------------------------------------------------|-------------------------------------------------| -| [BigQuery](bigquery/index.md) | Migration guide for BigQuery | -| [Snowflake](./snowflake.md) | Migration guide for Snowflake | -| [PostgreSQL](postgres/index.md) | Migration guide for PostgreSQL | -| [MySQL](../integrations/data-ingestion/dbms/mysql/index.md) | Migration guide for MySQL | -| [Redshift](../integrations/data-ingestion/redshift/index.md) | Migration guide for Redshift | -| [DynamoDB](../integrations/data-ingestion/dbms/dynamodb/index.md) | Migration guide for DynamoDB | -| [Elasticsearch](/use-cases/observability/clickstack/migration/elastic) | Migration guide for Elasticsearch to ClickStack | diff --git a/docs/use-cases/AI_ML/MCP/index.md b/docs/use-cases/AI_ML/MCP/index.md index 353d777de53..4e98d9531e6 100644 --- a/docs/use-cases/AI_ML/MCP/index.md +++ b/docs/use-cases/AI_ML/MCP/index.md @@ -62,6 +62,7 @@ Below are some guides showing how to use the ClickHouse MCP Server. | [How to build an OpenAI agent using ClickHouse MCP Server.](/use-cases/AI/MCP/ai-agent-libraries/openai-agents) | Learn how to build an OpenAI agent that can interact with ClickHouse MCP Server. | | [Set Up ClickHouse MCP Server with AnythingLLM and ClickHouse Cloud](/use-cases/AI/MCP/anythingllm) | This guide explains how to set up AnythingLLM with a ClickHouse MCP server using Docker. | | [Set Up ClickHouse MCP Server with Claude Desktop](/use-cases/AI/MCP/claude-desktop) | This guide explains how to set up Claude Desktop with a ClickHouse MCP server. | +| [Set Up ClickHouse MCP Server with Jan.ai](/use-cases/AI/MCP/janai) | This guide explains how to set up Jan.ai with a ClickHouse MCP server. | | [Set Up ClickHouse MCP Server with LibreChat and ClickHouse Cloud](/use-cases/AI/MCP/librechat) | This guide explains how to set up LibreChat with a ClickHouse MCP server using Docker. | | [Set Up ClickHouse MCP Server with Ollama](/use-cases/AI/MCP/ollama) | This guide explains how to set up Ollama with a ClickHouse MCP server. | | [Set Up ClickHouse MCP Server with Open WebUI and ClickHouse Cloud](/use-cases/AI/MCP/open-webui) | This guide explains how to set up Open WebUI with a ClickHouse MCP server using Docker. | diff --git a/docs/use-cases/observability/clickstack/production.md b/docs/use-cases/observability/clickstack/production.md index 4fd11eadf10..3fa9c0fe0b8 100644 --- a/docs/use-cases/observability/clickstack/production.md +++ b/docs/use-cases/observability/clickstack/production.md @@ -85,7 +85,7 @@ Additionally, we recommend enabling TLS for OTLP endpoints and creating a [dedic ## ClickHouse {#clickhouse} -For production deployments, we recommend using [ClickHouse Cloud](https://clickhouse.com/cloud), which applies industry-standard [security practices](/cloud/security) by default - including [enhanced encryption](/cloud/security/cmek), [authentication and connectivity](/cloud/security/connectivity), and [managed access controls](/cloud/security/cloud-access-management). See ["ClickHouse Cloud"](#clickhouse-cloud-production) for a step-by-step guide of using ClickHouse Cloud with best practices. +For production deployments, we recommend using [ClickHouse Cloud](https://clickhouse.com/cloud), which applies industry-standard [security practices](/cloud/security/shared-responsibility-model) by default - including [enhanced encryption](/cloud/security/cmek), [authentication and connectivity](/cloud/security/connectivity), and [managed access controls](/cloud/security/cloud-access-management). See ["ClickHouse Cloud"](#clickhouse-cloud-production) for a step-by-step guide of using ClickHouse Cloud with best practices. ### User permissions {#user-permissions} diff --git a/docs/whats-new/changelog/cloud.md b/docs/whats-new/changelog/cloud.md index cbc3f51a2f8..dd3faf69c73 100644 --- a/docs/whats-new/changelog/cloud.md +++ b/docs/whats-new/changelog/cloud.md @@ -8,6 +8,6 @@ description: 'Learn about Cloud Changelog' # Cloud Changelog -import CloudChangelog from '@site/docs/cloud/reference/changelog.md'; +import CloudChangelog from '@site/docs/cloud/reference/01_changelog/01_changelog.md'; diff --git a/docs/whats-new/changelog/index.md b/docs/whats-new/changelog/index.md index 5e70c9ecc81..c6b7ea36d6f 100644 --- a/docs/whats-new/changelog/index.md +++ b/docs/whats-new/changelog/index.md @@ -8,7 +8,8 @@ title: '2025 Changelog' --- ### Table of Contents -**[ClickHouse release v25.7, 2025-07-24](#256)**
+**[ClickHouse release v25.8, 2025-08-28](#258)**
+**[ClickHouse release v25.7, 2025-07-24](#257)**
**[ClickHouse release v25.6, 2025-06-26](#256)**
**[ClickHouse release v25.5, 2025-05-22](#255)**
**[ClickHouse release v25.4, 2025-04-22](#254)**
@@ -25,143 +26,452 @@ title: '2025 Changelog' **[Changelog for 2017](https://clickhouse.com/docs/whats-new/changelog/2017/)**
+### ClickHouse release 25.8, 2025-08-28 {#258} + +#### Backward Incompatible Change +* All the allocations done by external libraries are now visible to ClickHouse's memory tracker and accounted properly. This may result in "increased" reported memory usage for certain queries or failures with `MEMORY_LIMIT_EXCEEDED`. [#84082](https://github.com/ClickHouse/ClickHouse/pull/84082) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Infer `Array(Dynamic)` instead of unnamed `Tuple` for arrays of values with different types in JSON. To use the previous behaviour, disable setting `input_format_json_infer_array_of_dynamic_from_array_of_different_types`. [#80859](https://github.com/ClickHouse/ClickHouse/pull/80859) ([Pavel Kruglov](https://github.com/Avogar)). +* Move S3 latency metrics to histograms for homogeneity and simplicity. [#82305](https://github.com/ClickHouse/ClickHouse/pull/82305) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* Require backticks around identifiers with dots in default expressions to prevent them from being parsed as compound identifiers. [#83162](https://github.com/ClickHouse/ClickHouse/pull/83162) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* Lazy materialization is enabled only with analyzer (which is the default) to avoid maintenance without analyzer, — which, in our experience, have some issues (for example, when using `indexHint()` in conditions). [#83791](https://github.com/ClickHouse/ClickHouse/pull/83791) ([Igor Nikonov](https://github.com/devcrafter)). +* Write values of `Enum` type as `BYTE_ARRAY` with `ENUM` logical type in Parquet output format by default. [#84169](https://github.com/ClickHouse/ClickHouse/pull/84169) ([Pavel Kruglov](https://github.com/Avogar)). +* Enable MergeTree setting `write_marks_for_substreams_in_compact_parts` by default. It significantly improves performance of subcolumns reading from newly created Compact parts. Servers with version less then 25.5 won't be able to read new Compact parts. [#84171](https://github.com/ClickHouse/ClickHouse/pull/84171) ([Pavel Kruglov](https://github.com/Avogar)). +* The previous `concurrent_threads_scheduler` default value was `round_robin`, which proved unfair in the presence of a high number of single-threaded queries (e.g., INSERTs). This change makes a safer alternative `fair_round_robin` scheduler, the default. [#84747](https://github.com/ClickHouse/ClickHouse/pull/84747) ([Sergei Trifonov](https://github.com/serxa)). +* ClickHouse supports PostgreSQL-style heredoc syntax: `$tag$ string contents... $tag$`, also known as dollar-quoted string literals. In previous versions, there were fewer restrictions on tags: they could contain arbitrary characters, including punctuation and whitespace. This introduces parsing ambiguity with identifiers that can also start with a dollar character. At the same time, PostgreSQL only allows word characters for tags. To resolve the problem, we now restrict heredoc tags only to contain word characters. Closes [#84731](https://github.com/ClickHouse/ClickHouse/issues/84731). [#84846](https://github.com/ClickHouse/ClickHouse/pull/84846) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The functions `azureBlobStorage`, `deltaLakeAzure`, and `icebergAzure` have been updated to properly validate `AZURE` permissions. All cluster-variant functions (`-Cluster` functions) now verify permissions against their corresponding non-clustered counterparts. Additionally, the `icebergLocal` and `deltaLakeLocal` functions now enforce `FILE` permission checks. [#84938](https://github.com/ClickHouse/ClickHouse/pull/84938) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Enables `allow_dynamic_metadata_for_data_lakes` setting (Table Engine level setting) by default. [#85044](https://github.com/ClickHouse/ClickHouse/pull/85044) ([Daniil Ivanik](https://github.com/divanik)). + +#### New Feature +* Basic support for the PromQL dialect is added. To use it, set `dialect='promql'` in clickhouse-client, point it to the TimeSeries table using the setting `promql_table_name='X'` and execute queries like `rate(ClickHouseProfileEvents_ReadCompressedBytes[1m])[5m:1m]`. In addition you can wrap the PromQL query with SQL: `SELECT * FROM prometheusQuery('up', ...);`. So far only functions `rate`, `delta` and `increase` are supported. No unary/binary operators. No HTTP API. [#75036](https://github.com/ClickHouse/ClickHouse/pull/75036) ([Vitaly Baranov](https://github.com/vitlibar)). +* AI Powered SQL generation can now infer from env ANTHROPIC_API_KEY and OPENAI_API_KEY if available, this is to make it so that we can have a zero config option to use this feature. [#83787](https://github.com/ClickHouse/ClickHouse/pull/83787) ([Kaushik Iska](https://github.com/iskakaushik)). +* Implement support for [ArrowFlight RPC](https://arrow.apache.org/docs/format/Flight.html) protocol by adding: - new table function `arrowflight`. [#74184](https://github.com/ClickHouse/ClickHouse/pull/74184) ([zakr600](https://github.com/zakr600)). +* Now all tables support the `_table` virtual column (not only tables with the `Merge` engine), which is especially useful for queries with UNION ALL. [#63665](https://github.com/ClickHouse/ClickHouse/pull/63665) ([Xiaozhe Yu](https://github.com/wudidapaopao)). +* Allow to use any storage policy (i.e. object storage, such as S3) for external aggregation/sorting. [#84734](https://github.com/ClickHouse/ClickHouse/pull/84734) ([Azat Khuzhin](https://github.com/azat)). +* Implement AWS S3 authentication with an explicitly provided IAM role. Implement OAuth for GCS. These features were recently only available in ClickHouse Cloud and are now open-sourced. Synchronize some interfaces such as serialization of the connection parameters for object storages. [#84011](https://github.com/ClickHouse/ClickHouse/pull/84011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support position deletes for Iceberg TableEngine. [#83094](https://github.com/ClickHouse/ClickHouse/pull/83094) ([Daniil Ivanik](https://github.com/divanik)). +* Support Iceberg Equality Deletes. [#85843](https://github.com/ClickHouse/ClickHouse/pull/85843) ([Han Fei](https://github.com/hanfei1991)). +* Iceberg writes for create. Closes [#83927](https://github.com/ClickHouse/ClickHouse/issues/83927). [#83983](https://github.com/ClickHouse/ClickHouse/pull/83983) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Glue catalogs for writes. [#84136](https://github.com/ClickHouse/ClickHouse/pull/84136) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Iceberg Rest catalogs for writes. [#84684](https://github.com/ClickHouse/ClickHouse/pull/84684) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Merge all iceberg position delete files into data files. This will reduce amount and sizes of parquet files in iceberg storage. Syntax: `OPTIMIZE TABLE table_name`. [#85250](https://github.com/ClickHouse/ClickHouse/pull/85250) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Support `drop table` for iceberg (Removing from REST/Glue catalogs + removing metadata about table). [#85395](https://github.com/ClickHouse/ClickHouse/pull/85395) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Support alter delete mutations for iceberg in merge-on-read format. [#85549](https://github.com/ClickHouse/ClickHouse/pull/85549) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Support writes into DeltaLake. Closes [#79603](https://github.com/ClickHouse/ClickHouse/issues/79603). [#85564](https://github.com/ClickHouse/ClickHouse/pull/85564) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added setting `delta_lake_snapshot_version` to allow reading specific snapshot version in table engine `DeltaLake`. [#85295](https://github.com/ClickHouse/ClickHouse/pull/85295) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Write more iceberg statistics (column sizes, lower and upper bounds) in metadata (manifest entries) for min-max pruning. [#85746](https://github.com/ClickHouse/ClickHouse/pull/85746) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Support add/drop/modify columns in iceberg for simple types. [#85769](https://github.com/ClickHouse/ClickHouse/pull/85769) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Iceberg: support writing version-hint file. This closes [#85097](https://github.com/ClickHouse/ClickHouse/issues/85097). [#85130](https://github.com/ClickHouse/ClickHouse/pull/85130) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Views, created by ephemeral users, will now store a copy of an actual user and will no longer be invalidated after the ephemeral user is deleted. [#84763](https://github.com/ClickHouse/ClickHouse/pull/84763) ([pufit](https://github.com/pufit)). +* The vector similarity index now supports binary quantization. Binary quantization significantly reduces the memory consumption and speeds up the process of building a vector index (due to faster distance calculation). Also, the existing setting `vector_search_postfilter_multiplier `was made obsolete and replaced by a more general setting : `vector_search_index_fetch_multiplier`. [#85024](https://github.com/ClickHouse/ClickHouse/pull/85024) ([Shankar Iyer](https://github.com/shankar-iyer)). +* Allow key value arguments in `s3` or `s3Cluster` table engine/function, e.g. for example `s3('url', CSV, structure = 'a Int32', compression_method = 'gzip')`. [#85134](https://github.com/ClickHouse/ClickHouse/pull/85134) ([Kseniia Sumarokova](https://github.com/kssenii)). +* A new system table to keep erroneous incoming messages from engines like kafka ("dead letter queue"). [#68873](https://github.com/ClickHouse/ClickHouse/pull/68873) ([Ilya Golshtein](https://github.com/ilejn)). +* The new SYSTEM RESTORE DATABASE REPLICA for Replicated databases, similar to the existing functionality for restore in ReplicatedMergeTree. [#73100](https://github.com/ClickHouse/ClickHouse/pull/73100) ([Konstantin Morozov](https://github.com/k-morozov)). +* PostgreSQL protocol now supports the `COPY` command. [#74344](https://github.com/ClickHouse/ClickHouse/pull/74344) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Support C# client for mysql protocol. This closes [#83992](https://github.com/ClickHouse/ClickHouse/issues/83992). [#84397](https://github.com/ClickHouse/ClickHouse/pull/84397) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Add support for hive partition style reads and writes. [#76802](https://github.com/ClickHouse/ClickHouse/pull/76802) ([Arthur Passos](https://github.com/arthurpassos)). +* Add `zookeeper_connection_log` system table to store historical information about ZooKeeper connections. [#79494](https://github.com/ClickHouse/ClickHouse/pull/79494) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Server setting `cpu_slot_preemption` enables preemptive CPU scheduling for workloads and ensures max-min fair allocation of CPU time among workloads. New workload settings for CPU throttling are added: `max_cpus`, `max_cpu_share` and `max_burst_cpu_seconds`. More details: https://clickhouse.com/docs/operations/workload-scheduling#cpu_scheduling. [#80879](https://github.com/ClickHouse/ClickHouse/pull/80879) ([Sergei Trifonov](https://github.com/serxa)). +* Drop TCP connection after a configured number of queries or time threshold. This makes sense for a more uniform connection distribution between cluster nodes behind a load balancer. Resolves [#68000](https://github.com/ClickHouse/ClickHouse/issues/68000). [#81472](https://github.com/ClickHouse/ClickHouse/pull/81472) ([Kenny Sun](https://github.com/hwabis)). +* Parallel replicas now support using projections for queries. [#82659](https://github.com/ClickHouse/ClickHouse/issues/82659). [#82807](https://github.com/ClickHouse/ClickHouse/pull/82807) ([zoomxi](https://github.com/zoomxi)). +* Support DESCRIBE SELECT in addition to DESCRIBE (SELECT ...). [#82947](https://github.com/ClickHouse/ClickHouse/pull/82947) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Force secure connection for mysql_port and postgresql_port. [#82962](https://github.com/ClickHouse/ClickHouse/pull/82962) ([tiandiwonder](https://github.com/tiandiwonder)). +* Users can now do case-insensitive JSON key lookups using `JSONExtractCaseInsensitive` (and other variants of `JSONExtract`). [#83770](https://github.com/ClickHouse/ClickHouse/pull/83770) ([Alistair Evans](https://github.com/alistairjevans)). +* Introduction of `system.completions` table. Closes [#81889](https://github.com/ClickHouse/ClickHouse/issues/81889). [#83833](https://github.com/ClickHouse/ClickHouse/pull/83833) ([|2ustam](https://github.com/RuS2m)). +* Added a new function `nowInBlock64`. Example usage: `SELECT nowInBlock64(6)` returns `2025-07-29 17:09:37.775725`. [#84178](https://github.com/ClickHouse/ClickHouse/pull/84178) ([Halersson Paris](https://github.com/halersson)). +* Add extra_credentials to AzureBlobStorage to authenticate with client_id and tenant_id. [#84235](https://github.com/ClickHouse/ClickHouse/pull/84235) ([Pablo Marcos](https://github.com/pamarcos)). +* Added function `dateTimeToUUIDv7` to convert a DateTime value to a UUIDv7. Example usage: `SELECT dateTimeToUUIDv7(toDateTime('2025-08-15 18:57:56'))` returns `0198af18-8320-7a7d-abd3-358db23b9d5c`. [#84319](https://github.com/ClickHouse/ClickHouse/pull/84319) ([samradovich](https://github.com/samradovich)). +* `timeSeriesDerivToGrid` and `timeSeriesPredictLinearToGrid` aggregate functions to re-sample data to a time grid defined by the specified start timestamp, end timestamp, and step; calculates PromQL-like `deriv` and `predict_linear`, respectively. [#84328](https://github.com/ClickHouse/ClickHouse/pull/84328) ([Stephen Chi](https://github.com/stephchi0)). +* Add two new TimeSeries functions: - `timeSeriesRange(start_timestamp, end_timestamp, step)`, - `timeSeriesFromGrid(start_timestamp, end_timestamp, step, values)`,. [#85435](https://github.com/ClickHouse/ClickHouse/pull/85435) ([Vitaly Baranov](https://github.com/vitlibar)). +* New syntax added `GRANT READ ON S3('s3://foo/.*') TO user`. [#84503](https://github.com/ClickHouse/ClickHouse/pull/84503) ([pufit](https://github.com/pufit)). +* Added `Hash` as a new output format. It calculates a single hash value for all columns and rows of the result. This is useful for calculating a "fingerprint" of the result, for example, in use cases where data transfer is a bottleneck. Example: `SELECT arrayJoin(['abc', 'def']), 42 FORMAT Hash` returns `e5f9e676db098fdb9530d2059d8c23ef`. [#84607](https://github.com/ClickHouse/ClickHouse/pull/84607) ([Robert Schulze](https://github.com/rschu1ze)). +* Add the ability to set up arbitrary watches in Keeper Multi queries. [#84964](https://github.com/ClickHouse/ClickHouse/pull/84964) ([Mikhail Artemenko](https://github.com/Michicosun)). +* Adds an option `--max-concurrency` for the `clickhouse-benchmark` tool that enables a mode with a gradual increase in the number of parallel queries. [#85623](https://github.com/ClickHouse/ClickHouse/pull/85623) ([Sergei Trifonov](https://github.com/serxa)). +* TODO: what's that? Support partially aggregated metrics. [#85328](https://github.com/ClickHouse/ClickHouse/pull/85328) ([Mikhail Artemenko](https://github.com/Michicosun)). + +#### Experimental Feature +* Enable correlated subqueries support by default, they are no longer experimental. [#85107](https://github.com/ClickHouse/ClickHouse/pull/85107) ([Dmitry Novik](https://github.com/novikd)). +* Unity, Glue, Rest, and Hive Metastore data lake catalogs are promoted from experimental to beta. [#85848](https://github.com/ClickHouse/ClickHouse/pull/85848) ([Melvyn Peignon](https://github.com/melvynator)). +* Lightweight updates and deletes are promoted from experimental to beta. +* Approximate vector search with vector similarity indexes is now GA. [#85888](https://github.com/ClickHouse/ClickHouse/pull/85888) ([Robert Schulze](https://github.com/rschu1ze)). +* Ytsaurus table engine and table function. [#77606](https://github.com/ClickHouse/ClickHouse/pull/77606) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Previously, the text index data would be separated into multiple segments (each segment size by default was 256 MiB). This might reduce the memory consumption while building the text index, however this increases the space requirement on the disk and increase the query response time. [#84590](https://github.com/ClickHouse/ClickHouse/pull/84590) ([Elmi Ahmadov](https://github.com/ahmadov)). + +#### Performance Improvement +* New parquet reader implementation. It's generally faster and supports page-level filter pushdown and PREWHERE. Currently experimental. Use setting `input_format_parquet_use_native_reader_v3` to enable. [#82789](https://github.com/ClickHouse/ClickHouse/pull/82789) ([Michael Kolupaev](https://github.com/al13n321)). +* Replaced the official HTTP transport in Azure library with our own HTTP client implementation for Azure Blob Storage. Introduced multiple settings for this clients which mirror settings from S3. Introduced aggressive connection timeouts for both Azure and S3. Improved introspection into Azure profile events and metrics. New client is enabled by default, provide much better latencies for cold queries on top of Azure Blob Storage. Old `Curl` client can be returned back by setting `azure_sdk_use_native_client=false`. [#83294](https://github.com/ClickHouse/ClickHouse/pull/83294) ([alesapin](https://github.com/alesapin)). The previous, official implementation of Azure client was unsuitable for production due to terrible latency spikes, ranging from five seconds to minutes. We have ditched that terrible implementation and are very proud of that. +* Processes indexes in increasing order of file size. The net index ordering prioritizes minmax and vector indexes (due to simplicity and selectivity respectively), and small indexes thereafter. Within the minmax/vector indexes smaller indexes are also preferred. [#84094](https://github.com/ClickHouse/ClickHouse/pull/84094) ([Maruth Goyal](https://github.com/maruthgoyal)). +* Enable MergeTree setting `write_marks_for_substreams_in_compact_parts` by default. It significantly improves performance of subcolumns reading from newly created Compact parts. Servers with version less then 25.5 won't be able to read new Compact parts. [#84171](https://github.com/ClickHouse/ClickHouse/pull/84171) ([Pavel Kruglov](https://github.com/Avogar)). +* `azureBlobStorage` table engine: cache and reuse managed identity authentication tokens when possible to avoid throttling. [#79860](https://github.com/ClickHouse/ClickHouse/pull/79860) ([Nick Blakely](https://github.com/niblak)). +* `ALL` `LEFT/INNER` JOINs will be automatically converted to `RightAny` if the right side is functionally determined by the join key columns (all rows have unique join key values). [#84010](https://github.com/ClickHouse/ClickHouse/pull/84010) ([Nikita Taranov](https://github.com/nickitat)). +* Add `max_joined_block_size_bytes` in addition to `max_joined_block_size_rows` to limit the memory usage of JOINs with heavy columns. [#83869](https://github.com/ClickHouse/ClickHouse/pull/83869) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Added new logic (controlled by the setting `enable_producing_buckets_out_of_order_in_aggregation`, enabled by default) that allows sending some buckets out of order during memory-efficient aggregation. When some aggregation buckets take significantly longer to merge than others, it improves performance by allowing the initiator to merge buckets with higher bucket id-s in the meantime. The downside is potentially higher memory usage (shouldn't be significant). [#80179](https://github.com/ClickHouse/ClickHouse/pull/80179) ([Nikita Taranov](https://github.com/nickitat)). +* Introduced the `optimize_rewrite_regexp_functions` setting (enabled by default), which allows the optimizer to rewrite certain `replaceRegexpAll`, `replaceRegexpOne`, and `extract` calls into simpler and more efficient forms when specific regular expression patterns are detected. (issue [#81981](https://github.com/ClickHouse/ClickHouse/issues/81981)). [#81992](https://github.com/ClickHouse/ClickHouse/pull/81992) ([Amos Bird](https://github.com/amosbird)). +* Process `max_joined_block_rows` outside of hash JOIN main loop. Slightly better performance for ALL JOIN. [#83216](https://github.com/ClickHouse/ClickHouse/pull/83216) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Process higher granularity min-max indexes first. Closes [#75381](https://github.com/ClickHouse/ClickHouse/issues/75381). [#83798](https://github.com/ClickHouse/ClickHouse/pull/83798) ([Maruth Goyal](https://github.com/maruthgoyal)). +* Make `DISTINCT` window aggregates run in linear time and fix a bug in `sumDistinct`. Closes [#79792](https://github.com/ClickHouse/ClickHouse/issues/79792). Closes [#52253](https://github.com/ClickHouse/ClickHouse/issues/52253). [#79859](https://github.com/ClickHouse/ClickHouse/pull/79859) ([Nihal Z. Miaji](https://github.com/nihalzp)). +* Vector search queries using a vector similarity index complete with lower latency due to reduced storage reads and reduced CPU usage. [#83803](https://github.com/ClickHouse/ClickHouse/pull/83803) ([Shankar Iyer](https://github.com/shankar-iyer)). +* Rendezvous hashing for improve cache locality of workload distribution among parallel replicas. [#82511](https://github.com/ClickHouse/ClickHouse/pull/82511) ([Anton Ivashkin](https://github.com/ianton-ru)). +* Implement addManyDefaults for If combinators, so now aggregate functions with If combinators work faster. [#83870](https://github.com/ClickHouse/ClickHouse/pull/83870) ([Raúl Marín](https://github.com/Algunenano)). +* Calculate serialized key columnarly when group by multiple string or number columns. [#83884](https://github.com/ClickHouse/ClickHouse/pull/83884) ([李扬](https://github.com/taiyang-li)). +* Eliminated full scans for the cases when index analysis results in empty ranges for parallel replicas reading. [#84971](https://github.com/ClickHouse/ClickHouse/pull/84971) ([Eduard Karacharov](https://github.com/korowa)). +* Try -falign-functions=64 in attempt for more stable perf tests. [#83920](https://github.com/ClickHouse/ClickHouse/pull/83920) ([Azat Khuzhin](https://github.com/azat)). +* The bloom filter index is now used for conditions like `has([c1, c2, ...], column)`, where `column` is not of an `Array` type. This improves performance for such queries, making them as efficient as the `IN` operator. [#83945](https://github.com/ClickHouse/ClickHouse/pull/83945) ([Doron David](https://github.com/dorki)). +* Reduce unnecessary memcpy calls in CompressedReadBufferBase::readCompressedData. [#83986](https://github.com/ClickHouse/ClickHouse/pull/83986) ([Raúl Marín](https://github.com/Algunenano)). +* Optimize `largestTriangleThreeBuckets` by removing temporary data. [#84479](https://github.com/ClickHouse/ClickHouse/pull/84479) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimize string deserialization by simplifying the code. Closes [#38564](https://github.com/ClickHouse/ClickHouse/issues/38564). [#84561](https://github.com/ClickHouse/ClickHouse/pull/84561) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed the calculation of the minimal task size for parallel replicas. [#84752](https://github.com/ClickHouse/ClickHouse/pull/84752) ([Nikita Taranov](https://github.com/nickitat)). +* Improved performance of applying patch parts in `Join` mode. [#85040](https://github.com/ClickHouse/ClickHouse/pull/85040) ([Anton Popov](https://github.com/CurtizJ)). +* Remove zero byte. Closes [#85062](https://github.com/ClickHouse/ClickHouse/issues/85062). A few minor bugs were fixed. Functions `structureToProtobufSchema`, `structureToCapnProtoSchema` didn't correctly put a zero-terminating byte and were using a newline instead of it. That was leading to a missing newline in the output, and could lead to buffer overflows while using other functions that depend on the zero byte (such as `logTrace`, `demangle`, `extractURLParameter`, `toStringCutToZero`, and `encrypt`/`decrypt`). The `regexp_tree` dictionary layout didn't support processing strings with zero bytes. The `formatRowNoNewline` function, called with `Values` format or with any other format without a newline at the end of rows, erroneously cuts the last character of the output. Function `stem` contained an exception-safety error that could lead to a memory leak in a very rare scenario. The `initcap` function worked in the wrong way for `FixedString` arguments: it didn't recognize the start of the word at the start of the string if the previous string in a block ended with a word character. Fixed a security vulnerability of the Apache `ORC` format, which could lead to the exposure of uninitialized memory. Changed behavior of the function `replaceRegexpAll` and the corresponding alias, `REGEXP_REPLACE`: now it can do an empty match at the end of the string even if the previous match processed the whole string, such as in the case of `^a*|a*$` or `^|.*` - this corresponds to the semantic of JavaScript, Perl, Python, PHP, Ruby, but differs to the semantic of PostgreSQL. Implementation of many functions has been simplified and optimized. Documentation for several functions was wrong and has now been fixed. Keep in mind that the output of `byteSize` for String columns and complex types, which consisted of String columns, has changed (from 9 bytes per empty string to 8 bytes per empty string), and this is normal. [#85063](https://github.com/ClickHouse/ClickHouse/pull/85063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimize the materialization of constants in cases when we do this materialization only to return a single row. [#85071](https://github.com/ClickHouse/ClickHouse/pull/85071) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve parallel files processing with delta-kernel-rs backend. [#85642](https://github.com/ClickHouse/ClickHouse/pull/85642) ([Azat Khuzhin](https://github.com/azat)). +* A new setting, enable_add_distinct_to_in_subqueries, has been introduced. When enabled, ClickHouse will automatically add DISTINCT to subqueries in IN clauses for distributed queries. This can significantly reduce the size of temporary tables transferred between shards and improve network efficiency. Note: This is a trade-off—while network transfer is reduced, additional merging (deduplication) work is required on each node. Enable this setting when network transfer is a bottleneck and the merging cost is acceptable. [#81908](https://github.com/ClickHouse/ClickHouse/pull/81908) ([fhw12345](https://github.com/fhw12345)). +* Reduce query memory tracking overhead for executable user-defined functions. [#83929](https://github.com/ClickHouse/ClickHouse/pull/83929) ([Eduard Karacharov](https://github.com/korowa)). +* Implement internal `delta-kernel-rs` filtering (statistics and partition pruning) in storage `DeltaLake`. [#84006](https://github.com/ClickHouse/ClickHouse/pull/84006) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable skipping indexes that depend on columns updated on the fly or by patch parts more granularly. Now, skipping indexes are not used only in parts affected by on-the-fly mutations or patch parts; previously, those indexes were disabled for all parts. [#84241](https://github.com/ClickHouse/ClickHouse/pull/84241) ([Anton Popov](https://github.com/CurtizJ)). +* Allocate the minimum amount of memory needed for encrypted_buffer for encrypted named collections. [#84432](https://github.com/ClickHouse/ClickHouse/pull/84432) ([Pablo Marcos](https://github.com/pamarcos)). +* Improved support for bloom filter indexes (regular, ngram, and token) to be utilized when the first argument is a constant array (the set) and the second is the indexed column (the subset), enabling more efficient query execution. [#84700](https://github.com/ClickHouse/ClickHouse/pull/84700) ([Doron David](https://github.com/dorki)). +* Reduce contention on storage lock in Keeper. [#84732](https://github.com/ClickHouse/ClickHouse/pull/84732) ([Antonio Andelic](https://github.com/antonio2368)). +* Add missing support of `read_in_order_use_virtual_row` for `WHERE`. It allows to skip reading more parts for queries with filters that were not fully pushed to `PREWHERE`. [#84835](https://github.com/ClickHouse/ClickHouse/pull/84835) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Allows asynchronously iterating objects from Iceberg table without storing objects for each data file explicitly. [#85369](https://github.com/ClickHouse/ClickHouse/pull/85369) ([Daniil Ivanik](https://github.com/divanik)). +* Execute non-correlated `EXISTS` as a scalar subquery. This allows using a scalar subquery cache and constant-folding the result, which is helpful for indexes. For compatibility, the new setting `execute_exists_as_scalar_subquery=1` is added. [#85481](https://github.com/ClickHouse/ClickHouse/pull/85481) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Improvement +* Add `database_replicated` settings defining the default values of DatabaseReplicatedSettings. If the setting is not present in the Replicated DB create query, the value from this setting is used. [#85127](https://github.com/ClickHouse/ClickHouse/pull/85127) ([Tuan Pham Anh](https://github.com/tuanpach)). +* Made the table columns in the web UI (play) resizable. [#84012](https://github.com/ClickHouse/ClickHouse/pull/84012) ([Doron David](https://github.com/dorki)). +* Support compressed `.metadata.json` file via `iceberg_metadata_compression_method` setting. It supports all clickhouse compression methods. This closes [#84895](https://github.com/ClickHouse/ClickHouse/issues/84895). [#85196](https://github.com/ClickHouse/ClickHouse/pull/85196) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Show the number of ranges to be read in the output of `EXPLAIN indexes = 1`. [#79938](https://github.com/ClickHouse/ClickHouse/pull/79938) ([Christoph Wurm](https://github.com/cwurm)). +* Introduce settings to set ORC compression block size, and update its default value from 64KB to 256KB to keep consistent with spark or hive. [#80602](https://github.com/ClickHouse/ClickHouse/pull/80602) ([李扬](https://github.com/taiyang-li)). +* Add `columns_substreams.txt` file to Wide part to track all substreams stored in the part. It helps to track dynamic streams in JSON and Dynamic types and so avoid reading sample of these columns to get the list of dynamic streams (for example for columns sizes calculation). Also now all dynamic streams are reflected in `system.parts_columns`. [#81091](https://github.com/ClickHouse/ClickHouse/pull/81091) ([Pavel Kruglov](https://github.com/Avogar)). +* Add a CLI flag --show_secrets to clickhouse format to hide sensitive data by default. [#81524](https://github.com/ClickHouse/ClickHouse/pull/81524) ([Nikolai Ryzhov](https://github.com/Dolaxom)). +* S3 read and write requests are throttled on the HTTP socket level (instead of whole S3 requests) to avoid issues with `max_remote_read_network_bandwidth_for_server` and `max_remote_write_network_bandwidth_for_server` throttling. [#81837](https://github.com/ClickHouse/ClickHouse/pull/81837) ([Sergei Trifonov](https://github.com/serxa)). +* Allow to mix different collations for the same column in different windows (for window functions). [#82877](https://github.com/ClickHouse/ClickHouse/pull/82877) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Add a tool to simulate, visualize and compare merge selectors. [#71496](https://github.com/ClickHouse/ClickHouse/pull/71496) ([Sergei Trifonov](https://github.com/serxa)). +* Add support of `remote*` table functions with parallel replicas if cluster is provided in `address_expression` argument. Also, fixes [#73295](https://github.com/ClickHouse/ClickHouse/issues/73295). [#82904](https://github.com/ClickHouse/ClickHouse/pull/82904) ([Igor Nikonov](https://github.com/devcrafter)). +* Set all log messages for writing backup files to TRACE. [#82907](https://github.com/ClickHouse/ClickHouse/pull/82907) ([Hans Krutzer](https://github.com/hkrutzer)). +* User-defined functions with unusual names and codecs can be formatted inconsistently by the SQL formatter. This closes [#83092](https://github.com/ClickHouse/ClickHouse/issues/83092). [#83644](https://github.com/ClickHouse/ClickHouse/pull/83644) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Users can now use Time and Time64 types inside the JSON type. [#83784](https://github.com/ClickHouse/ClickHouse/pull/83784) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Joins with parallel replicas now use the join logical step. In case of any issues with join queries using parallel replicas, try `SET query_plan_use_new_logical_join_step=0` and report an issue. [#83801](https://github.com/ClickHouse/ClickHouse/pull/83801) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fix compatibility for cluster_function_process_archive_on_multiple_nodes. [#83968](https://github.com/ClickHouse/ClickHouse/pull/83968) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support changing mv insert settings on `S3Queue` table level. Added new `S3Queue` level settings: `min_insert_block_size_rows_for_materialized_views` and `min_insert_block_size_bytes_for_materialized_views`. By default profile level settings will be used and `S3Queue` level settings will override those. [#83971](https://github.com/ClickHouse/ClickHouse/pull/83971) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added profile event `MutationAffectedRowsUpperBound` that shows the number of affected rows in a mutation (e.g., the total number of rows that satisfy the condition in `ALTER UPDATE` or `ALTER DELETE` query. [#83978](https://github.com/ClickHouse/ClickHouse/pull/83978) ([Anton Popov](https://github.com/CurtizJ)). +* Use information from cgroup (if applicable, i.e. `memory_worker_use_cgroup` and cgroups are available) to adjust memory tracker (`memory_worker_correct_memory_tracker`). [#83981](https://github.com/ClickHouse/ClickHouse/pull/83981) ([Azat Khuzhin](https://github.com/azat)). +* MongoDB: Implicit parsing of strings to numeric types. Previously, if a string value was received from a MongoDB source for a numeric column in a ClickHouse table, an exception was thrown. Now, the engine attempts to parse the numeric value from the string automatically. Closes [#81167](https://github.com/ClickHouse/ClickHouse/issues/81167). [#84069](https://github.com/ClickHouse/ClickHouse/pull/84069) ([Kirill Nikiforov](https://github.com/allmazz)). +* Highlight digit groups in `Pretty` formats for `Nullable` numbers. [#84070](https://github.com/ClickHouse/ClickHouse/pull/84070) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dashboard: the tooltip will not overflow the container at the top. [#84072](https://github.com/ClickHouse/ClickHouse/pull/84072) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly better-looking dots on the dashboard. [#84074](https://github.com/ClickHouse/ClickHouse/pull/84074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dashboard now has a slightly better favicon. [#84076](https://github.com/ClickHouse/ClickHouse/pull/84076) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Web UI: Give browsers a chance to save the password. Also, it will remember the URL values. [#84087](https://github.com/ClickHouse/ClickHouse/pull/84087) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add support for applying extra ACL on specific Keeper nodes using `apply_to_children` config. [#84137](https://github.com/ClickHouse/ClickHouse/pull/84137) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix usage of "compact" Variant discriminators serialization in MergeTree. Perviously it wasn't used in some cases when it could be used. [#84141](https://github.com/ClickHouse/ClickHouse/pull/84141) ([Pavel Kruglov](https://github.com/Avogar)). +* Added a server setting, `logs_to_keep` to database replicated settings, that allows changing the default `logs_to_keep` parameter for replicated databases. Lower values reduce the number of ZNodes (especially if there are many databases), while higher values allow a missing replica to catch up after a longer period of time. [#84183](https://github.com/ClickHouse/ClickHouse/pull/84183) ([Alexey Khatskevich](https://github.com/Khatskevich)). +* Add a setting `json_type_escape_dots_in_keys` to escape dots in JSON keys during JSON type parsing. The setting is disabled by default. [#84207](https://github.com/ClickHouse/ClickHouse/pull/84207) ([Pavel Kruglov](https://github.com/Avogar)). +* Check if connection is cancelled before checking for EOF to prevent reading from closed connection. Fixes [#83893](https://github.com/ClickHouse/ClickHouse/issues/83893). [#84227](https://github.com/ClickHouse/ClickHouse/pull/84227) ([Raufs Dunamalijevs](https://github.com/rienath)). +* Slightly better colors of text selection in Web UI. The difference is significant only for selected table cells in the dark mode. In previous versions, there was not enough contrast between the text and the selection background. [#84258](https://github.com/ClickHouse/ClickHouse/pull/84258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improved server shutdown handling for client connections by simplifying internal checks. [#84312](https://github.com/ClickHouse/ClickHouse/pull/84312) ([Raufs Dunamalijevs](https://github.com/rienath)). +* Added a setting `delta_lake_enable_expression_visitor_logging` to turn off expression visitor logs as they can be too verbose even for test log level when debugging something. [#84315](https://github.com/ClickHouse/ClickHouse/pull/84315) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Cgroup-level and system-wide metrics are reported now altogether. Cgroup-level metrics have names `CGroup` and OS-level metrics (collected from procfs) have names `OS`. [#84317](https://github.com/ClickHouse/ClickHouse/pull/84317) ([Nikita Taranov](https://github.com/nickitat)). +* Slightly better charts in Web UI. Not much, but better. [#84326](https://github.com/ClickHouse/ClickHouse/pull/84326) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change the default of the Replicated database setting `max_retries_before_automatic_recovery` to 10, so it will recover faster in some cases. [#84369](https://github.com/ClickHouse/ClickHouse/pull/84369) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix formatting of CREATE USER with query parameters (i.e. `CREATE USER {username:Identifier} IDENTIFIED WITH no_password`). [#84376](https://github.com/ClickHouse/ClickHouse/pull/84376) ([Azat Khuzhin](https://github.com/azat)). +* Introduce `backup_restore_s3_retry_initial_backoff_ms`, `backup_restore_s3_retry_max_backoff_ms`, `backup_restore_s3_retry_jitter_factor` to configure the S3 retry backoff strategy used during backup and restore operations. [#84421](https://github.com/ClickHouse/ClickHouse/pull/84421) ([Julia Kartseva](https://github.com/jkartseva)). +* S3Queue ordered mode fix: quit earlier if shutdown was called. [#84463](https://github.com/ClickHouse/ClickHouse/pull/84463) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support iceberg writes to read from pyiceberg. [#84466](https://github.com/ClickHouse/ClickHouse/pull/84466) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Allow set values type casting when pushing down `IN` / `GLOBAL IN` filters over KeyValue storage primary keys (e.g., EmbeddedRocksDB, KeeperMap). [#84515](https://github.com/ClickHouse/ClickHouse/pull/84515) ([Eduard Karacharov](https://github.com/korowa)). +* Bump chdig to [25.7.1](https://github.com/azat/chdig/releases/tag/v25.7.1). [#84521](https://github.com/ClickHouse/ClickHouse/pull/84521) ([Azat Khuzhin](https://github.com/azat)). +* Low-level errors during UDF execution now fail with error code `UDF_EXECUTION_FAILED`, whereas previously different error codes could be returned. [#84547](https://github.com/ClickHouse/ClickHouse/pull/84547) ([Xu Jia](https://github.com/XuJia0210)). +* Add `get_acl` command to KeeperClient. [#84641](https://github.com/ClickHouse/ClickHouse/pull/84641) ([Antonio Andelic](https://github.com/antonio2368)). +* Adds snapshot version to data lake table engines. [#84659](https://github.com/ClickHouse/ClickHouse/pull/84659) ([Pete Hampton](https://github.com/pjhampton)). +* Add a dimensional metric for the size of `ConcurrentBoundedQueue`, labelled by the queue type (i.e. what the queue is there for) and queue id (i.e. randomly generated id for the current instance of the queue). [#84675](https://github.com/ClickHouse/ClickHouse/pull/84675) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* The `system.columns` table now provides `column` as an alias for the existing `name` column. [#84695](https://github.com/ClickHouse/ClickHouse/pull/84695) ([Yunchi Pang](https://github.com/yunchipang)). +* New MergeTree setting `search_orphaned_parts_drives` to limit scope to look for parts e.g. by disks with local metadata. [#84710](https://github.com/ClickHouse/ClickHouse/pull/84710) ([Ilya Golshtein](https://github.com/ilejn)). +* Add 4LW in Keeper, `lgrq`, for toggling request logging of received requests. [#84719](https://github.com/ClickHouse/ClickHouse/pull/84719) ([Antonio Andelic](https://github.com/antonio2368)). +* Match external auth forward_headers in case-insensitive way. [#84737](https://github.com/ClickHouse/ClickHouse/pull/84737) ([ingodwerust](https://github.com/ingodwerust)). +* The `encrypt_decrypt` tool now supports encrypted ZooKeeper connections. [#84764](https://github.com/ClickHouse/ClickHouse/pull/84764) ([Roman Vasin](https://github.com/rvasin)). +* Add format string column to `system.errors`. This column is needed to group by the same error type in alerting rules. [#84776](https://github.com/ClickHouse/ClickHouse/pull/84776) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* Updated `clickhouse-format` to accept `--highlight` as an alias for `--hilite`. - Updated `clickhouse-client` to accept `--hilite` as an alias for `--highlight`. - Updated `clickhouse-format` documentation to reflect the change. [#84806](https://github.com/ClickHouse/ClickHouse/pull/84806) ([Rishabh Bhardwaj](https://github.com/rishabh1815769)). +* Fix iceberg reading by field ids for complex types. [#84821](https://github.com/ClickHouse/ClickHouse/pull/84821) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Introduce a new `backup_slow_all_threads_after_retryable_s3_error` setting to reduce pressure on S3 during retry storms caused by errors such as `SlowDown`, by slowing down all threads once a single retryable error is observed. [#84854](https://github.com/ClickHouse/ClickHouse/pull/84854) ([Julia Kartseva](https://github.com/jkartseva)). +* Skip creating and renaming the old temp table of non-append RMV DDLs in Replicated DBs. [#84858](https://github.com/ClickHouse/ClickHouse/pull/84858) ([Tuan Pham Anh](https://github.com/tuanpach)). +* Limit Keeper log entry cache size by number of entries using `keeper_server.coordination_settings.latest_logs_cache_entry_count_threshold` and `keeper_server.coordination_settings.commit_logs_cache_entry_count_threshold`. [#84877](https://github.com/ClickHouse/ClickHouse/pull/84877) ([Antonio Andelic](https://github.com/antonio2368)). +* Allow using `simdjson` on unsupported architectures (previously leads to `CANNOT_ALLOCATE_MEMORY` errors). [#84966](https://github.com/ClickHouse/ClickHouse/pull/84966) ([Azat Khuzhin](https://github.com/azat)). +* Async logging: Make limits tuneable and add introspection. [#85105](https://github.com/ClickHouse/ClickHouse/pull/85105) ([Raúl Marín](https://github.com/Algunenano)). +* Collect all removed objects to execute single object storage remove operation. [#85316](https://github.com/ClickHouse/ClickHouse/pull/85316) ([Mikhail Artemenko](https://github.com/Michicosun)). +* Iceberg's current implementation of positional delete files keeps all data in RAM. This can be quite expensive if the positional delete files are large, which is often the case. My implementation keeps only the last row-group of Parquet delete files in RAM, which is significantly cheaper. [#85329](https://github.com/ClickHouse/ClickHouse/pull/85329) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* chdig: fix leftovers on the screen, fix crash after edit query in editor, search in `path` for `editor`, update to [25.8.1](https://github.com/azat/chdig/releases/tag/v25.8.1). [#85341](https://github.com/ClickHouse/ClickHouse/pull/85341) ([Azat Khuzhin](https://github.com/azat)). +* Add missing `partition_columns_in_data_file` to azure configuration. [#85373](https://github.com/ClickHouse/ClickHouse/pull/85373) ([Arthur Passos](https://github.com/arthurpassos)). +* Allow zero step in functions `timeSeries*ToGrid` This is part of [#75036](https://github.com/ClickHouse/ClickHouse/pull/75036). [#85390](https://github.com/ClickHouse/ClickHouse/pull/85390) ([Vitaly Baranov](https://github.com/vitlibar)). +* Added show_data_lake_catalogs_in_system_tables flag to manage adding data lake tables in system.tables. Resolves [#85384](https://github.com/ClickHouse/ClickHouse/issues/85384). [#85411](https://github.com/ClickHouse/ClickHouse/pull/85411) ([Smita Kulkarni](https://github.com/SmitaRKulkarni)). +* Added support for macro expansion in `remote_fs_zero_copy_zookeeper_path`. [#85437](https://github.com/ClickHouse/ClickHouse/pull/85437) ([Mikhail Koviazin](https://github.com/mkmkme)). +* AI in clickhouse-client will look slightly better. [#85447](https://github.com/ClickHouse/ClickHouse/pull/85447) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable trace_log.symbolize for old deployments by default. [#85456](https://github.com/ClickHouse/ClickHouse/pull/85456) ([Azat Khuzhin](https://github.com/azat)). +* Support resolution of more cases for compound identifiers. Particularly, it improves the compatibility of `ARRAY JOIN` with the old analyzer. Introduce a new setting `analyzer_compatibility_allow_compound_identifiers_in_unflatten_nested` to keep the old behaviour. [#85492](https://github.com/ClickHouse/ClickHouse/pull/85492) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Ignore UNKNOWN_DATABASE while obtaining table columns sizes for system.columns. [#85632](https://github.com/ClickHouse/ClickHouse/pull/85632) ([Azat Khuzhin](https://github.com/azat)). +* Added a limit (table setting `max_uncompressed_bytes_in_patches`) for total uncompressed bytes in patch parts. It prevents significant slowdowns of SELECT queries after lightweight updates and prevents possible misuse of lightweight updates. [#85641](https://github.com/ClickHouse/ClickHouse/pull/85641) ([Anton Popov](https://github.com/CurtizJ)). +* Add a `parameter` column to `system.grants` to determine source type for `GRANT READ/WRITE` and the table engine for `GRANT TABLE ENGINE`. [#85643](https://github.com/ClickHouse/ClickHouse/pull/85643) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix parsing of a trailing comma in columns of the CREATE DICTIONARY query after a column with parameters, for example, Decimal(8). Closes [#85586](https://github.com/ClickHouse/ClickHouse/issues/85586). [#85653](https://github.com/ClickHouse/ClickHouse/pull/85653) ([Nikolay Degterinsky](https://github.com/evillique)). +* Support inner arrays for the function `nested`. [#85719](https://github.com/ClickHouse/ClickHouse/pull/85719) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* This pr fixes the metadata resolution when querying iceberg tables through rest catalog. ... [#80562](https://github.com/ClickHouse/ClickHouse/pull/80562) ([Saurabh Kumar Ojha](https://github.com/saurabhojha)). +* Fix markReplicasActive in DDLWorker and DatabaseReplicatedDDLWorker. [#81395](https://github.com/ClickHouse/ClickHouse/pull/81395) ([Tuan Pham Anh](https://github.com/tuanpach)). +* Fix rollback of Dynamic column on parsing failure. [#82169](https://github.com/ClickHouse/ClickHouse/pull/82169) ([Pavel Kruglov](https://github.com/Avogar)). +* If function `trim` called with all-constant inputs now produces a constant output string. (Bug [#78796](https://github.com/ClickHouse/ClickHouse/issues/78796)). [#82900](https://github.com/ClickHouse/ClickHouse/pull/82900) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix logical error with duplicate subqueries when `optimize_syntax_fuse_functions` is enabled, close [#75511](https://github.com/ClickHouse/ClickHouse/issues/75511). [#83300](https://github.com/ClickHouse/ClickHouse/pull/83300) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Fixed incorrect result of queries with `WHERE ... IN ()` clause and enabled query condition cache (setting `use_query_condition_cache`). [#83445](https://github.com/ClickHouse/ClickHouse/pull/83445) ([LB7666](https://github.com/acking-you)). +* Historically, `gcs` function did not require any access to use. Now it will check `GRANT READ ON S3` permission for usage. Closes [#70567](https://github.com/ClickHouse/ClickHouse/issues/70567). [#83503](https://github.com/ClickHouse/ClickHouse/pull/83503) ([pufit](https://github.com/pufit)). +* Skip unavailable nodes during INSERT SELECT from s3Cluster() into replicated MergeTree. [#83676](https://github.com/ClickHouse/ClickHouse/pull/83676) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix write with append (in MergeTree used for experimental transactions) with `plain_rewritable`/`plain` metadata types, previously they were simply ignored. [#83695](https://github.com/ClickHouse/ClickHouse/pull/83695) ([Tuan Pham Anh](https://github.com/tuanpach)). +* Mask Avro schema registry authentication details to be not visible to user or in logs. [#83713](https://github.com/ClickHouse/ClickHouse/pull/83713) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix the issue where, if a MergeTree table is created with `add_minmax_index_for_numeric_columns=1` or `add_minmax_index_for_string_columns=1`, the index is later materialized during an ALTER operation, and it prevents the Replicated database from initializing correctly on a new replica. [#83751](https://github.com/ClickHouse/ClickHouse/pull/83751) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fixed parquet writer outputting incorrect statistics (min/max) for Decimal types. [#83754](https://github.com/ClickHouse/ClickHouse/pull/83754) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix sort of NaN values in `LowCardinality(Float32|Float64|BFloat16)` type. [#83786](https://github.com/ClickHouse/ClickHouse/pull/83786) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* When restoring from backup, the definer user may not be backed up, which will cause the whole backup to be broken. To fix this, we postpone the permissions check on the target table's creation during restore and only check it during runtime. [#83818](https://github.com/ClickHouse/ClickHouse/pull/83818) ([pufit](https://github.com/pufit)). +* Fix crash in client due to connection left in disconnected state after bad INSERT. [#83842](https://github.com/ClickHouse/ClickHouse/pull/83842) ([Azat Khuzhin](https://github.com/azat)). +* Allow referencing any table in `view(...)` argument of `remote` table function with enabled analyzer. Fixes [#78717](https://github.com/ClickHouse/ClickHouse/issues/78717). Fixes [#79377](https://github.com/ClickHouse/ClickHouse/issues/79377). [#83844](https://github.com/ClickHouse/ClickHouse/pull/83844) ([Dmitry Novik](https://github.com/novikd)). +* Onprogress call in jsoneachrowwithprogress is synchronized with finalization. [#83879](https://github.com/ClickHouse/ClickHouse/pull/83879) ([Sema Checherinda](https://github.com/CheSema)). +* This closes [#81303](https://github.com/ClickHouse/ClickHouse/issues/81303). [#83892](https://github.com/ClickHouse/ClickHouse/pull/83892) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Fix colorSRGBToOKLCH/colorOKLCHToSRGB for mix of const and non-const args. [#83906](https://github.com/ClickHouse/ClickHouse/pull/83906) ([Azat Khuzhin](https://github.com/azat)). +* Fix writing JSON paths with NULL values in RowBinary format. [#83923](https://github.com/ClickHouse/ClickHouse/pull/83923) ([Pavel Kruglov](https://github.com/Avogar)). +* Overflow large values (>2106-02-07) when casting from Date to DateTime64 is fixed. [#83982](https://github.com/ClickHouse/ClickHouse/pull/83982) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Always apply `filesystem_prefetches_limit` (not only from `MergeTreePrefetchedReadPool`). [#83999](https://github.com/ClickHouse/ClickHouse/pull/83999) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare bug when `MATERIALIZE COLUMN` query could lead to unexpected files in `checksums.txt` and eventually detached data parts. [#84007](https://github.com/ClickHouse/ClickHouse/pull/84007) ([alesapin](https://github.com/alesapin)). +* Fix the logical error `Expected single dictionary argument for function` while doing JOIN on an inequality condition when one of the columns is `LowCardinality` and the other is a constant. Closes [#81779](https://github.com/ClickHouse/ClickHouse/issues/81779). [#84019](https://github.com/ClickHouse/ClickHouse/pull/84019) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix crash with clickhouse client when used in interactive mode with syntax highlighting. [#84025](https://github.com/ClickHouse/ClickHouse/pull/84025) ([Bharat Nallan](https://github.com/bharatnc)). +* Fixed wrong results when the query condition cache is used in conjunction with recursive CTEs (issue [#81506](https://github.com/ClickHouse/ClickHouse/issues/81506)). [#84026](https://github.com/ClickHouse/ClickHouse/pull/84026) ([zhongyuankai](https://github.com/zhongyuankai)). +* Handle exceptions properly in periodic parts refresh. [#84083](https://github.com/ClickHouse/ClickHouse/pull/84083) ([Azat Khuzhin](https://github.com/azat)). +* Fix filter merging into JOIN condition in cases when equality operands have different types or they reference constants. Fixes [#83432](https://github.com/ClickHouse/ClickHouse/issues/83432). [#84145](https://github.com/ClickHouse/ClickHouse/pull/84145) ([Dmitry Novik](https://github.com/novikd)). +* Fix rare clickhouse crash when table has projection, `lightweight_mutation_projection_mode = 'rebuild'` and user execute lighweight delete which deletes ALL rows from any block in table. [#84158](https://github.com/ClickHouse/ClickHouse/pull/84158) ([alesapin](https://github.com/alesapin)). +* Fix deadlock caused by background cancellation checker thread. [#84203](https://github.com/ClickHouse/ClickHouse/pull/84203) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix infinite recursive analysis of invalid `WINDOW` definitions. Fixes [#83131](https://github.com/ClickHouse/ClickHouse/issues/83131). [#84242](https://github.com/ClickHouse/ClickHouse/pull/84242) ([Dmitry Novik](https://github.com/novikd)). +* Fixed a bug that was causing incorrect Bech32 Encoding and Decoding. The bug wasn't caught originally due to an online implementation of the algorithm used for testing having the same issue. [#84257](https://github.com/ClickHouse/ClickHouse/pull/84257) ([George Larionov](https://github.com/george-larionov)). +* Fixed incorrect construction of empty tuples in the `array()` function. This fixes [#84202](https://github.com/ClickHouse/ClickHouse/issues/84202). [#84297](https://github.com/ClickHouse/ClickHouse/pull/84297) ([Amos Bird](https://github.com/amosbird)). +* Fix `LOGICAL_ERROR` for queries with parallel replicas and multiple INNER joins followed by RIGHT join. Do not use parallel replicas for such queries. [#84299](https://github.com/ClickHouse/ClickHouse/pull/84299) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Previously, `set` indexes didn't consider `Nullable` columns while checking if granules passed the filter (issue [#75485](https://github.com/ClickHouse/ClickHouse/issues/75485)). [#84305](https://github.com/ClickHouse/ClickHouse/pull/84305) ([Elmi Ahmadov](https://github.com/ahmadov)). +* Now ClickHouse read tables from Glue Catalog where table type specified in lower case. [#84316](https://github.com/ClickHouse/ClickHouse/pull/84316) ([alesapin](https://github.com/alesapin)). +* Do not try to substitute table functions to its cluster alternative in presence of a JOIN or subquery. [#84335](https://github.com/ClickHouse/ClickHouse/pull/84335) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fix logger usage in `IAccessStorage`. [#84365](https://github.com/ClickHouse/ClickHouse/pull/84365) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fixed a logical error in lightweight updates that update all columns in the table. [#84380](https://github.com/ClickHouse/ClickHouse/pull/84380) ([Anton Popov](https://github.com/CurtizJ)). +* Codec `DoubleDelta` codec can now only be applied to columns of numeric type. In particular `FixedString` columns can no longer be compressed using `DoubleDelta`. (fixes [#80220](https://github.com/ClickHouse/ClickHouse/issues/80220)). [#84383](https://github.com/ClickHouse/ClickHouse/pull/84383) ([Jimmy Aguilar Mena](https://github.com/Ergus)). +* The comparison against nan value was not using the correct ranges during `MinMax` index evaluation. [#84386](https://github.com/ClickHouse/ClickHouse/pull/84386) ([Elmi Ahmadov](https://github.com/ahmadov)). +* Fix reading Variant column with lazy materialization. [#84400](https://github.com/ClickHouse/ClickHouse/pull/84400) ([Pavel Kruglov](https://github.com/Avogar)). +* Make `zoutofmemory` hardware error, otherwise it will throw logical error. see https://github.com/clickhouse/clickhouse-core-incidents/issues/877. [#84420](https://github.com/ClickHouse/ClickHouse/pull/84420) ([Han Fei](https://github.com/hanfei1991)). +* Fixed server crash when a user created with `no_password` attempts to login after the server setting `allow_no_password` was changed to 0. [#84426](https://github.com/ClickHouse/ClickHouse/pull/84426) ([Shankar Iyer](https://github.com/shankar-iyer)). +* Fix out-of-order writes to Keeper changelog. Previously, we could have in-flight writes to changelog, but rollback could cause concurrent change of the destination file. This would lead to inconsistent logs, and possible data loss. [#84434](https://github.com/ClickHouse/ClickHouse/pull/84434) ([Antonio Andelic](https://github.com/antonio2368)). +* Now if all TTL are removed from table MergeTree will do nothing related to TTL. [#84441](https://github.com/ClickHouse/ClickHouse/pull/84441) ([alesapin](https://github.com/alesapin)). +* Parallel distributed INSERT SELECT with LIMIT was allowed which is not correct, it leads to data duplication in target table. [#84477](https://github.com/ClickHouse/ClickHouse/pull/84477) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix pruning files by virtual column in data lakes. [#84520](https://github.com/ClickHouse/ClickHouse/pull/84520) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix leaks for keeper with rocksdb storage (iterators was not destroyed). [#84523](https://github.com/ClickHouse/ClickHouse/pull/84523) ([Azat Khuzhin](https://github.com/azat)). +* Fix ALTER MODIFY ORDER BY not validating TTL columns in sorting keys. TTL columns are now properly rejected when used in ORDER BY clauses during ALTER operations, preventing potential table corruption. [#84536](https://github.com/ClickHouse/ClickHouse/pull/84536) ([xiaohuanlin](https://github.com/xiaohuanlin)). +* Change pre-25.5 value of `allow_experimental_delta_kernel_rs` to `false` for compatibility. [#84587](https://github.com/ClickHouse/ClickHouse/pull/84587) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Stops taking schema from manifest files but stores relevant schemas for each snapshot independently. Infer relevant schema for each data file from its corresponding snapshot. Previous behaviour violated Iceberg specification for manifest files entries with existing status. [#84588](https://github.com/ClickHouse/ClickHouse/pull/84588) ([Daniil Ivanik](https://github.com/divanik)). +* Fixed issue where Keeper setting `rotate_log_storage_interval = 0` would cause ClickHouse to crash. (issue [#83975](https://github.com/ClickHouse/ClickHouse/issues/83975)). [#84637](https://github.com/ClickHouse/ClickHouse/pull/84637) ([George Larionov](https://github.com/george-larionov)). +* Fix logical error from S3Queue "Table is already registered". Closes [#84433](https://github.com/ClickHouse/ClickHouse/issues/84433). Broken after https://github.com/ClickHouse/ClickHouse/pull/83530. [#84677](https://github.com/ClickHouse/ClickHouse/pull/84677) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Lock 'mutex' when getting zookeeper from 'view' in RefreshTask. [#84699](https://github.com/ClickHouse/ClickHouse/pull/84699) ([Tuan Pham Anh](https://github.com/tuanpach)). +* Fix `CORRUPTED_DATA` error when lazy columns are used with external sort. [#84738](https://github.com/ClickHouse/ClickHouse/pull/84738) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix column pruning with delta-kernel in storage `DeltaLake`. Closes [#84543](https://github.com/ClickHouse/ClickHouse/issues/84543). [#84745](https://github.com/ClickHouse/ClickHouse/pull/84745) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Refresh credentials in delta-kernel in storage DeltaLake. [#84751](https://github.com/ClickHouse/ClickHouse/pull/84751) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix starting superfluous internal backups after connection problems. [#84755](https://github.com/ClickHouse/ClickHouse/pull/84755) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed issue where querying a delayed remote source could result in vector out of bounds. [#84820](https://github.com/ClickHouse/ClickHouse/pull/84820) ([George Larionov](https://github.com/george-larionov)). +* The `ngram` and `no_op` tokenizers no longer crash the (experimental) text index for empty input tokens. [#84849](https://github.com/ClickHouse/ClickHouse/pull/84849) ([Robert Schulze](https://github.com/rschu1ze)). +* Fixed lightweight updates for tables with `ReplacingMergeTree` and `CollapsingMergeTree` engines. [#84851](https://github.com/ClickHouse/ClickHouse/pull/84851) ([Anton Popov](https://github.com/CurtizJ)). +* Correctly store all settings in table metadata for tables using object queue engine. [#84860](https://github.com/ClickHouse/ClickHouse/pull/84860) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix total watches count returned by Keeper. [#84890](https://github.com/ClickHouse/ClickHouse/pull/84890) ([Antonio Andelic](https://github.com/antonio2368)). +* Fixed lightweight updates for tables with `ReplicatedMergeTree` engine created on servers with a version lower than 25.7. [#84933](https://github.com/ClickHouse/ClickHouse/pull/84933) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed lightweight updates for tables with non-replicated `MergeTree` engine after running a `ALTER TABLE ... REPLACE PARTITION` query. [#84941](https://github.com/ClickHouse/ClickHouse/pull/84941) ([Anton Popov](https://github.com/CurtizJ)). +* Fixes column name generation for boolean literals to use "true"/"false" instead of "1"/"0", preventing column name conflicts between boolean and integer literals in queries. [#84945](https://github.com/ClickHouse/ClickHouse/pull/84945) ([xiaohuanlin](https://github.com/xiaohuanlin)). +* Fix memory tracking drift from background schedule pool and executor. [#84946](https://github.com/ClickHouse/ClickHouse/pull/84946) ([Azat Khuzhin](https://github.com/azat)). +* Fix potential inaccurate sorting issues in the Merge table engine. [#85025](https://github.com/ClickHouse/ClickHouse/pull/85025) ([Xiaozhe Yu](https://github.com/wudidapaopao)). +* Implement missing APIs for DiskEncrypted. [#85028](https://github.com/ClickHouse/ClickHouse/pull/85028) ([Azat Khuzhin](https://github.com/azat)). +* Add a check if a correlated subquery is used in a distributed context to avoid a crash. Fixes [#82205](https://github.com/ClickHouse/ClickHouse/issues/82205). [#85030](https://github.com/ClickHouse/ClickHouse/pull/85030) ([Dmitry Novik](https://github.com/novikd)). +* Now Iceberg doesn't try to cache relevant snapshot version between select queries and always try to resolve snapshot honestly. Earlier attempt to cache iceberg snapshot led to problems with usage of Iceberg table with time travel. [#85038](https://github.com/ClickHouse/ClickHouse/pull/85038) ([Daniil Ivanik](https://github.com/divanik)). +* Fixed double-free in `AzureIteratorAsync`. [#85064](https://github.com/ClickHouse/ClickHouse/pull/85064) ([Nikita Taranov](https://github.com/nickitat)). +* Improve error message on attempt to create user identified with JWT. [#85072](https://github.com/ClickHouse/ClickHouse/pull/85072) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fixed cleanup of patch parts in `ReplicatedMergeTree`. Previously, the result of a lightweight update may temporarily not be visible on the replica until the merged or mutated part that materializes the patch parts is downloaded from another replica. [#85121](https://github.com/ClickHouse/ClickHouse/pull/85121) ([Anton Popov](https://github.com/CurtizJ)). +* Fixing illegal_type_of_argument in mv when types are different. [#85135](https://github.com/ClickHouse/ClickHouse/pull/85135) ([Sema Checherinda](https://github.com/CheSema)). +* Fix segfault in delta-kernel implementation. [#85160](https://github.com/ClickHouse/ClickHouse/pull/85160) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix recovering replicated databases when moving the metadata file takes a long time. [#85177](https://github.com/ClickHouse/ClickHouse/pull/85177) ([Tuan Pham Anh](https://github.com/tuanpach)). +* Fix `Not-ready Set` for `IN (subquery)` inside `additional_table_filters expression` setting. [#85210](https://github.com/ClickHouse/ClickHouse/pull/85210) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Get rid of unnecessary `getStatus()` calls during SYSTEM DROP REPLICA queries. Fixes the case when a table is dropped in the background, and the `Shutdown for storage is called` exception is thrown. [#85220](https://github.com/ClickHouse/ClickHouse/pull/85220) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix race in `DeltaLake` engine delta-kernel implementation. [#85221](https://github.com/ClickHouse/ClickHouse/pull/85221) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix reading partitioned data with disabled delta-kernel in `DeltaLake` engine. It was broken in 25.7 (https://github.com/ClickHouse/ClickHouse/pull/81136). [#85223](https://github.com/ClickHouse/ClickHouse/pull/85223) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added missing table name length checks in CREATE OR REPLACE and RENAME queries. [#85326](https://github.com/ClickHouse/ClickHouse/pull/85326) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix the creation of RMV on a new replica of the Replicated database if DEFINER is dropped. [#85327](https://github.com/ClickHouse/ClickHouse/pull/85327) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix iceberg writes for complex types. [#85330](https://github.com/ClickHouse/ClickHouse/pull/85330) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Writing lower and upper bounds are not supported for complex types. [#85332](https://github.com/ClickHouse/ClickHouse/pull/85332) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Fix logical error while reading from object storage functions through Distributed table or remote table function. Fixes: [#84658](https://github.com/ClickHouse/ClickHouse/issues/84658), Fixes [#85173](https://github.com/ClickHouse/ClickHouse/issues/85173), Fixes [#52022](https://github.com/ClickHouse/ClickHouse/issues/52022). [#85359](https://github.com/ClickHouse/ClickHouse/pull/85359) ([alesapin](https://github.com/alesapin)). +* Fix backup of parts with broken projections. [#85362](https://github.com/ClickHouse/ClickHouse/pull/85362) ([Antonio Andelic](https://github.com/antonio2368)). +* Forbid using `_part_offset` column in projection in releases until it is stabilized. [#85372](https://github.com/ClickHouse/ClickHouse/pull/85372) ([Sema Checherinda](https://github.com/CheSema)). +* Fix crash and data corruption during ALTER UPDATE for JSON. [#85383](https://github.com/ClickHouse/ClickHouse/pull/85383) ([Pavel Kruglov](https://github.com/Avogar)). +* Queries with parallel replicas which uses reading reverse in order optimization can produce incorrect result. [#85406](https://github.com/ClickHouse/ClickHouse/pull/85406) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix possible UB (crashes) in case of MEMORY_LIMIT_EXCEEDED during String deserialization. [#85440](https://github.com/ClickHouse/ClickHouse/pull/85440) ([Azat Khuzhin](https://github.com/azat)). +* Fix incorrect metrics KafkaAssignedPartitions and KafkaConsumersWithAssignment. [#85494](https://github.com/ClickHouse/ClickHouse/pull/85494) ([Ilya Golshtein](https://github.com/ilejn)). +* Fixed processed bytes stat being underestimated when PREWHERE (explicit or automatic) is used. [#85495](https://github.com/ClickHouse/ClickHouse/pull/85495) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix early return condition for S3 request rate slowdown: require either s3_slow_all_threads_after_network_error or backup_slow_all_threads_after_retryable_s3_error to be true to enable slowdown behavior when all threads are paused due to a retryable error, instead of requiring both. [#85505](https://github.com/ClickHouse/ClickHouse/pull/85505) ([Julia Kartseva](https://github.com/jkartseva)). +* This pr fixes the metadata resolution when querying iceberg tables through rest catalog. ... [#85531](https://github.com/ClickHouse/ClickHouse/pull/85531) ([Saurabh Kumar Ojha](https://github.com/saurabhojha)). +* Fixed rare crash in asynchronous inserts that change settings `log_comment` or `insert_deduplication_token`. [#85540](https://github.com/ClickHouse/ClickHouse/pull/85540) ([Anton Popov](https://github.com/CurtizJ)). +* Arameters like date_time_input_format have been just ignored when http with multipart. [#85570](https://github.com/ClickHouse/ClickHouse/pull/85570) ([Sema Checherinda](https://github.com/CheSema)). +* Fix secrets masking in icebergS3Cluster and icebergAzureCluster table functions. [#85658](https://github.com/ClickHouse/ClickHouse/pull/85658) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix precision loss in `JSONExtract` when converting JSON numbers to Decimal types. Now numeric JSON values preserve their exact decimal representation, avoiding floating-point rounding errors. [#85665](https://github.com/ClickHouse/ClickHouse/pull/85665) ([ssive7b](https://github.com/ssive7b)). +* Fixed `LOGICAL_ERROR` when using `COMMENT COLUMN IF EXISTS` in the same `ALTER` statement after `DROP COLUMN`. The `IF EXISTS` clause now correctly skips the comment operation when the column has been dropped within the same statement. [#85688](https://github.com/ClickHouse/ClickHouse/pull/85688) ([xiaohuanlin](https://github.com/xiaohuanlin)). +* Fix reading count from cache for delta lake. [#85704](https://github.com/ClickHouse/ClickHouse/pull/85704) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix coalescing merge tree segfault for large strings. This closes [#84582](https://github.com/ClickHouse/ClickHouse/issues/84582). [#85709](https://github.com/ClickHouse/ClickHouse/pull/85709) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Update metadata timestamp in iceberg writes. [#85711](https://github.com/ClickHouse/ClickHouse/pull/85711) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Using `distributed_depth` as an indicator of *Cluster function was incorrect and may lead to data duplication; use `client_info.collaborate_with_initiator` instead. [#85734](https://github.com/ClickHouse/ClickHouse/pull/85734) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Spark can't read position delete files. [#85762](https://github.com/ClickHouse/ClickHouse/pull/85762) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Fix send_logs_source_regexp (after async logging refactoring in [#85105](https://github.com/ClickHouse/ClickHouse/issues/85105)). [#85797](https://github.com/ClickHouse/ClickHouse/pull/85797) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible inconsistency for dictionaries with update_field on MEMORY_LIMIT_EXCEEDED errors. [#85807](https://github.com/ClickHouse/ClickHouse/pull/85807) ([Azat Khuzhin](https://github.com/azat)). +* Support global constants from `WITH` statement for the parallel distributed `INSERT SELECT` with the `Distributed` destination table. Before, the query could throw an `Unknown expression identifier` error. [#85811](https://github.com/ClickHouse/ClickHouse/pull/85811) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Mask credentials for `deltaLakeAzure`, `deltaLakeCluster`, `icebergS3Cluster` and `icebergAzureCluster`. [#85889](https://github.com/ClickHouse/ClickHouse/pull/85889) ([Julian Maicher](https://github.com/jmaicher)). +* Fix logical error on attempt to `CREATE ... AS (SELECT * FROM s3Cluster(...))` with `DatabaseReplicated`. [#85904](https://github.com/ClickHouse/ClickHouse/pull/85904) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fixes HTTP requests made by the `url()` table function to properly include port numbers in the Host header when accessing non-standard ports. This resolves authentication failures when using presigned URLs with S3-compatible services like MinIO running on custom ports, which is common in development environments. (Fixes [#85898](https://github.com/ClickHouse/ClickHouse/issues/85898)). [#85921](https://github.com/ClickHouse/ClickHouse/pull/85921) ([Tom Quist](https://github.com/tomquist)). +* Now unity catalog will ignore schemas with weird data types in case of non-delta tables. Fixes [#85699](https://github.com/ClickHouse/ClickHouse/issues/85699). [#85950](https://github.com/ClickHouse/ClickHouse/pull/85950) ([alesapin](https://github.com/alesapin)). +* Fix nullability of fields in iceberg. [#85977](https://github.com/ClickHouse/ClickHouse/pull/85977) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Fixed a bug in `Replicated` database recovery: if a table name contains the `%` symbol, it could re-create the table with a different name during recovery. [#85987](https://github.com/ClickHouse/ClickHouse/pull/85987) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix backup restores failing due to `BACKUP_ENTRY_NOT_FOUND` error when restoring an empty `Memory` table. [#86012](https://github.com/ClickHouse/ClickHouse/pull/86012) ([Julia Kartseva](https://github.com/jkartseva)). +* Add checks for sharding_key during ALTER of the Distributed table. Previously incorrect ALTER would break the table definition and server restart. [#86015](https://github.com/ClickHouse/ClickHouse/pull/86015) ([Nikolay Degterinsky](https://github.com/evillique)). +* Don't create empty iceberg delete file. [#86061](https://github.com/ClickHouse/ClickHouse/pull/86061) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Fix large setting values breaking S3Queue tables and replica restart. [#86074](https://github.com/ClickHouse/ClickHouse/pull/86074) ([Nikolay Degterinsky](https://github.com/evillique)). + +#### Build/Testing/Packaging Improvement +* Use encrypted disks for tests with S3 by default. [#59898](https://github.com/ClickHouse/ClickHouse/pull/59898) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use `clickhouse` binary in integrations tests to get unstripped debug symbols. [#83779](https://github.com/ClickHouse/ClickHouse/pull/83779) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Bumped the internal libxml2 from 2.14.4 to 2.14.5. [#84230](https://github.com/ClickHouse/ClickHouse/pull/84230) ([Robert Schulze](https://github.com/rschu1ze)). +* Bumped internal curl from 8.14.0 to 8.15.0. [#84231](https://github.com/ClickHouse/ClickHouse/pull/84231) ([Robert Schulze](https://github.com/rschu1ze)). +* Now we use less memory for caches in CI and have better tests for eviction. [#84676](https://github.com/ClickHouse/ClickHouse/pull/84676) ([alesapin](https://github.com/alesapin)). + + ### ClickHouse release 25.7, 2025-07-24 {#257} #### Backward Incompatible Change -* Changes to `extractKeyValuePairs` function: Introduce a new argument `unexpected_quoting_character_strategy` that controls what happens when a `quoting_character` is unexpectedly found when reading a non quoted key or value. The value can be one of: `invalid`, `accept` or `promote`. Invalid will discard the key and go back to waiting key state. Accept will treat it as part of the key. Promote will discard previous character and start parsing as a quoted key. In addition, after parsing a quoted value, only parse the next key if a pair delimiter is found. [#80657](https://github.com/ClickHouse/ClickHouse/pull/80657) ([Arthur Passos](https://github.com/arthurpassos)). +* Changes to `extractKeyValuePairs` function: introduce a new argument `unexpected_quoting_character_strategy` that controls what happens when a `quoting_character` is unexpectedly found when reading a non quoted key or value. The value can be one of: `invalid`, `accept` or `promote`. Invalid will discard the key and go back to waiting key state. Accept will treat it as part of the key. Promote will discard previous character and start parsing as a quoted key. In addition, after parsing a quoted value, only parse the next key if a pair delimiter is found. [#80657](https://github.com/ClickHouse/ClickHouse/pull/80657) ([Arthur Passos](https://github.com/arthurpassos)). * Support zero-byte match in `countMatches` function. Users who like to retain the old behavior can enable setting `count_matches_stop_at_empty_match`. [#81676](https://github.com/ClickHouse/ClickHouse/pull/81676) ([Elmi Ahmadov](https://github.com/ahmadov)). * Use server-wide throttlers for local (`max_local_read_bandwidth_for_server` and `max_local_write_bandwidth_for_server`) and remote (`max_remote_read_network_bandwidth_for_server` and `max_remote_write_network_bandwidth_for_server`) when generating BACKUPs in addition to their dedicated server settings (`max_backup_bandwidth_for_server`, `max_mutations_bandwidth_for_server` and `max_merges_bandwidth_for_server`). [#81753](https://github.com/ClickHouse/ClickHouse/pull/81753) ([Sergei Trifonov](https://github.com/serxa)). * Forbid the creation of a table without insertable columns. [#81835](https://github.com/ClickHouse/ClickHouse/pull/81835) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). -* Cluster functions with archives used to send over the whole archives to replicas, making reading within archives not-parallelizable with clusters (e.g. with a single archive we would just send it to one of the replicas as a whole to process and all other replicas will just be idle, which is inefficient). Added a new setting `cluster_function_process_archive_on_multiple_nodes`, by default equal to `true`. If set to `true`, increases performance of processing archives in cluster functions. Should be set to `false` for compatibility and to avoid errors during upgrade to 25.7+ if using cluster functions with archives on earlier versions. [#82355](https://github.com/ClickHouse/ClickHouse/pull/82355) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Parallelize cluster functions by files within archives. In previous versions, the whole archive (such as zip, tar, or 7z) was a unit of work. Added a new setting `cluster_function_process_archive_on_multiple_nodes`, by default equal to `true`. If set to `true`, increases performance of processing archives in cluster functions. Should be set to `false` for compatibility and to avoid errors during upgrade to 25.7+ if using cluster functions with archives on earlier versions. [#82355](https://github.com/ClickHouse/ClickHouse/pull/82355) ([Kseniia Sumarokova](https://github.com/kssenii)). * `SYSTEM RESTART REPLICAS` query led to the wakeup of tables in the Lazy database, even without access to that database, and it happened while these tables were being concurrently dropped. Note: Now `SYSTEM RESTART REPLICAS` will only restart replicas in the databases where you have permission to `SHOW TABLES`, which is natural. [#83321](https://github.com/ClickHouse/ClickHouse/pull/83321) ([Alexey Milovidov](https://github.com/alexey-milovidov)). #### New Feature * Added support for lightweight updates for `MergeTree`-family tables. Lightweight updates can be used by a new syntax: `UPDATE SET col1 = val1, col2 = val2, ... WHERE `. Added implementation of lightweight deletes via lightweight updates. It can be enabled by setting `lightweight_delete_mode = 'lightweight_update'`. [#82004](https://github.com/ClickHouse/ClickHouse/pull/82004) ([Anton Popov](https://github.com/CurtizJ)). -* Support complex types in iceberg schema evolution. [#73714](https://github.com/ClickHouse/ClickHouse/pull/73714) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Support complex types in Iceberg schema evolution. [#73714](https://github.com/ClickHouse/ClickHouse/pull/73714) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Introduce support for INSERTs into Iceberg tables. [#82692](https://github.com/ClickHouse/ClickHouse/pull/82692) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Read Iceberg data files by field ids. This improves compatibility with Iceberg: the fields can be renamed in the metadata while being mapped to the different names in the underlying Parquet files. This closes [#83065](https://github.com/ClickHouse/ClickHouse/issues/83065). [#83653](https://github.com/ClickHouse/ClickHouse/pull/83653) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Now clickhouse supports compressed `metadata.json` files for Iceberg. Fixes [#70874](https://github.com/ClickHouse/ClickHouse/issues/70874). [#81451](https://github.com/ClickHouse/ClickHouse/pull/81451) ([alesapin](https://github.com/alesapin)). +* Support `TimestampTZ` in Glue catalog. This closes [#81654](https://github.com/ClickHouse/ClickHouse/issues/81654). [#83132](https://github.com/ClickHouse/ClickHouse/pull/83132) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Add AI-powered SQL generation to ClickHouse client. Users can now generate SQL queries from natural language descriptions by prefixing their query with `??`. Supports OpenAI and Anthropic providers with automatic schema discovery. [#83314](https://github.com/ClickHouse/ClickHouse/pull/83314) ([Kaushik Iska](https://github.com/iskakaushik)). +* Add a function to write Geo types into WKB format. [#82935](https://github.com/ClickHouse/ClickHouse/pull/82935) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Introduced two new access types: `READ` and `WRITE` for sources and deprecates all previous access types related to sources. Before `GRANT S3 ON *.* TO user`, now: `GRANT READ, WRITE ON S3 TO user`. This also allows to separate `READ` and `WRITE` permissions for sources, e.g.: `GRANT READ ON * TO user`, `GRANT WRITE ON S3 TO user`. The feature is controlled by a setting `access_control_improvements.enable_read_write_grants` and disabled by default. [#73659](https://github.com/ClickHouse/ClickHouse/pull/73659) ([pufit](https://github.com/pufit)). * NumericIndexedVector: new vector data-structure backed by bit-sliced, Roaring-bitmap compression, together with more than 20 functions for building, analysing and point-wise arithmetic. Can cut storage and speed up joins, filters and aggregations on sparse data. Implements [#70582](https://github.com/ClickHouse/ClickHouse/issues/70582) and [“Large-Scale Metric Computation in Online Controlled Experiment Platform” paper](https://arxiv.org/abs/2405.08411) by T. Xiong and Y. Wang from VLDB 2024. [#74193](https://github.com/ClickHouse/ClickHouse/pull/74193) ([FriendLey](https://github.com/FriendLey)). * The workload setting `max_waiting_queries` is now supported. It can be used to limit the size of the query queue. If the limit is reached, all subsequent queries will be terminated with the `SERVER_OVERLOADED` error. [#81250](https://github.com/ClickHouse/ClickHouse/pull/81250) ([Oleg Doronin](https://github.com/dorooleg)). * Add financial functions: `financialInternalRateOfReturnExtended` (`XIRR`), `financialInternalRateOfReturn` (`IRR`), `financialNetPresentValueExtended` (`XNPV`), `financialNetPresentValue` (`NPV`). [#81599](https://github.com/ClickHouse/ClickHouse/pull/81599) ([Joanna Hulboj](https://github.com/jh0x)). -* Add the geospatial functions `polygonIntersectsCartesian` and `polygonIntersectsSpherical` to check if two polygons intersect. [#81882](https://github.com/ClickHouse/ClickHouse/pull/81882) ([Paul Lamb](https://github.com/plamb)). -* Support `_part_granule_offset` virtual column in MergeTree-family tables. This column indicates the zero-based index of the granule/mark each row belongs to within its data part. This addresses [#79572](https://github.com/ClickHouse/ClickHouse/issues/79572). [#82341](https://github.com/ClickHouse/ClickHouse/pull/82341) ([Amos Bird](https://github.com/amosbird)). -* Introduce Iceberg writes for `insert` queries. [#82692](https://github.com/ClickHouse/ClickHouse/pull/82692) ([Konstantin Vedernikov](https://github.com/scanhex12)). -* Add SZ3 as a lossy yet error-bounded compression codec for columns of type `Float32` and `Float64`. [#83088](https://github.com/ClickHouse/ClickHouse/pull/83088) ([Konstantin Vedernikov](https://github.com/scanhex12)). -* Add AI-powered SQL generation to ClickHouse client. Users can now generate SQL queries from natural language descriptions by prefixing their query with "??". Supports OpenAI and Anthropic providers with automatic schema discovery. [#83314](https://github.com/ClickHouse/ClickHouse/pull/83314) ([Kaushik Iska](https://github.com/iskakaushik)). -* Read iceberg data files by field ids. This closes [#83065](https://github.com/ClickHouse/ClickHouse/issues/83065). [#83653](https://github.com/ClickHouse/ClickHouse/pull/83653) ([Konstantin Vedernikov](https://github.com/scanhex12)). +* Add the geospatial functions `polygonsIntersectCartesian` and `polygonsIntersectSpherical` to check if two polygons intersect. [#81882](https://github.com/ClickHouse/ClickHouse/pull/81882) ([Paul Lamb](https://github.com/plamb)). +* Support `_part_granule_offset` virtual column in MergeTree-family tables. This column indicates the zero-based index of the granule/mark each row belongs to within its data part. This addresses [#79572](https://github.com/ClickHouse/ClickHouse/issues/79572). [#82341](https://github.com/ClickHouse/ClickHouse/pull/82341) ([Amos Bird](https://github.com/amosbird)). [#82341](https://github.com/ClickHouse/ClickHouse/pull/82341) ([Amos Bird](https://github.com/amosbird)) * Added SQL functions `colorSRGBToOkLCH` and `colorOkLCHToSRGB` for converting colours between the sRGB and OkLCH colour spaces. [#83679](https://github.com/ClickHouse/ClickHouse/pull/83679) ([Fgrtue](https://github.com/Fgrtue)). -* Add _part_granule_offset column". [#82341](https://github.com/ClickHouse/ClickHouse/pull/82341) ([Amos Bird](https://github.com/amosbird)). +* Allow parameters in `CREATE USER` queries for usernames. [#81387](https://github.com/ClickHouse/ClickHouse/pull/81387) ([Diskein](https://github.com/Diskein)). +* The `system.formats` table now contains extended information about formats, such as HTTP content type, the capabilities of schema inference, etc. [#81505](https://github.com/ClickHouse/ClickHouse/pull/81505) ([Alexey Milovidov](https://github.com/alexey-milovidov)). #### Experimental Feature * Added functions `searchAny` and `searchAll` which are general purpose tools to search text indexes. [#80641](https://github.com/ClickHouse/ClickHouse/pull/80641) ([Elmi Ahmadov](https://github.com/ahmadov)). -* The text index now supports `string` tokenizer. [#81752](https://github.com/ClickHouse/ClickHouse/pull/81752) ([Elmi Ahmadov](https://github.com/ahmadov)). +* The text index now supports the new `split` tokenizer. [#81752](https://github.com/ClickHouse/ClickHouse/pull/81752) ([Elmi Ahmadov](https://github.com/ahmadov)). * Changed the default index granularity value for `text` indexes to 64. This improves the expected performance for the average test query in internal benchmarks. [#82162](https://github.com/ClickHouse/ClickHouse/pull/82162) ([Jimmy Aguilar Mena](https://github.com/Ergus)). * The 256-bit bitmap stores the outgoing labels of a state ordered, but outgoing states are saved into disk in the order they appear in the hash table. Therefore, a label would point to a wrong next state while reading from disk. [#82783](https://github.com/ClickHouse/ClickHouse/pull/82783) ([Elmi Ahmadov](https://github.com/ahmadov)). -* Enable zstd compression for FST tree blob. [#83093](https://github.com/ClickHouse/ClickHouse/pull/83093) ([Elmi Ahmadov](https://github.com/ahmadov)). -* Promote vector similarity index to BETA. Introduced alias setting `enable_vector_similarity_index` which must be enabled to use the vector similarity index. [#83459](https://github.com/ClickHouse/ClickHouse/pull/83459) ([Robert Schulze](https://github.com/rschu1ze)). +* Enable zstd compression for FST tree blob in text indices. [#83093](https://github.com/ClickHouse/ClickHouse/pull/83093) ([Elmi Ahmadov](https://github.com/ahmadov)). +* Promote vector similarity index to beta. Introduced alias setting `enable_vector_similarity_index` which must be enabled to use the vector similarity index. [#83459](https://github.com/ClickHouse/ClickHouse/pull/83459) ([Robert Schulze](https://github.com/rschu1ze)). +* Removed experimental `send_metadata` logic related to experimental zero-copy replication. It wasn't ever used and nobody supports this code. Since there were even no tests related to it, there is a high chance that it's broken long time ago. [#82508](https://github.com/ClickHouse/ClickHouse/pull/82508) ([alesapin](https://github.com/alesapin)). +* Integrate `StorageKafka2` to `system.kafka_consumers`. [#82652](https://github.com/ClickHouse/ClickHouse/pull/82652) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Estimate complex CNF/DNF, for example, `(a < 1 and a > 0) or b = 3`, by statistics. [#82663](https://github.com/ClickHouse/ClickHouse/pull/82663) ([Han Fei](https://github.com/hanfei1991)). #### Performance Improvement -* Trivial optimization for -If combinator. [#78454](https://github.com/ClickHouse/ClickHouse/pull/78454) ([李扬](https://github.com/taiyang-li)). +* Introduce async logging. When logs are output to a slow device, it no longer delays queries. [#82516](https://github.com/ClickHouse/ClickHouse/pull/82516) ([Raúl Marín](https://github.com/Algunenano)). Limit the max number of entries that are hold in the queue. [#83214](https://github.com/ClickHouse/ClickHouse/pull/83214) ([Raúl Marín](https://github.com/Algunenano)). +* Parallel distributed INSERT SELECT is enabled by default in mode where INSERT SELECT executed on each shard independently, see `parallel_distributed_insert_select` setting. [#83040](https://github.com/ClickHouse/ClickHouse/pull/83040) ([Igor Nikonov](https://github.com/devcrafter)). +* When the aggregation query contains only a single `count()` function on a not-`Nullable` column, the aggregation logic is fully inlined during hash table probing. This avoids allocating and maintaining any aggregation state, significantly reducing memory usage and CPU overhead. This partially addresses [#81982](https://github.com/ClickHouse/ClickHouse/issues/81982). [#82104](https://github.com/ClickHouse/ClickHouse/pull/82104) ([Amos Bird](https://github.com/amosbird)). +* Performance of `HashJoin` optimised by removing the additional loop over hash maps in the typical case of only one key column, also `null_map` and `join_mask` checks are eliminated when they're always `true`/`false`. [#82308](https://github.com/ClickHouse/ClickHouse/pull/82308) ([Nikita Taranov](https://github.com/nickitat)). +* Trivial optimization for `-If` combinator. [#78454](https://github.com/ClickHouse/ClickHouse/pull/78454) ([李扬](https://github.com/taiyang-li)). * Vector search queries using a vector similarity index complete with lower latency due to reduced storage reads and reduced CPU usage. [#79103](https://github.com/ClickHouse/ClickHouse/pull/79103) ([Shankar Iyer](https://github.com/shankar-iyer)). * Respect `merge_tree_min_{rows,bytes}_for_seek` in `filterPartsByQueryConditionCache` to align it with other methods filtering by indexes. [#80312](https://github.com/ClickHouse/ClickHouse/pull/80312) ([李扬](https://github.com/taiyang-li)). -* Make the pipeline after the TOTALS step multithreaded. [#80331](https://github.com/ClickHouse/ClickHouse/pull/80331) ([UnamedRus](https://github.com/UnamedRus)). -* Fix filter by key for Redis and KeeperMap storages. [#81833](https://github.com/ClickHouse/ClickHouse/pull/81833) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). +* Make the pipeline after the `TOTALS` step multithreaded. [#80331](https://github.com/ClickHouse/ClickHouse/pull/80331) ([UnamedRus](https://github.com/UnamedRus)). +* Fix filter by key for `Redis` and `KeeperMap` storages. [#81833](https://github.com/ClickHouse/ClickHouse/pull/81833) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)). * Add new setting `min_joined_block_size_rows` (analogous to `min_joined_block_size_bytes`; defaults to 65409) to control the minimum block size (in rows) for JOIN input and output blocks (if the join algorithm supports it). Small blocks will be squashed. [#81886](https://github.com/ClickHouse/ClickHouse/pull/81886) ([Nikita Taranov](https://github.com/nickitat)). -* When the aggregation query contains only a single `COUNT()` function on a NOT NULL column, the aggregation logic is fully inlined during hash table probing. This avoids allocating and maintaining any aggregation state, significantly reducing memory usage and CPU overhead. This partially addresses [#81982](https://github.com/ClickHouse/ClickHouse/issues/81982). [#82104](https://github.com/ClickHouse/ClickHouse/pull/82104) ([Amos Bird](https://github.com/amosbird)). -* Performance of `HashJoin` optimised by removing the additional loop over hash maps in the typical case of only one key column, also `null_map` and `join_mask` checks are eliminated when they're always `true`/`false`. [#82308](https://github.com/ClickHouse/ClickHouse/pull/82308) ([Nikita Taranov](https://github.com/nickitat)). * `ATTACH PARTITION` no longer leads to the dropping of all caches. [#82377](https://github.com/ClickHouse/ClickHouse/pull/82377) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Optimize the generated plan for correlated subqueries by removing redundant JOIN operations using equivalence classes. If there are equivalent expressions for all correlated columns, `CROSS JOIN` is not produced if `query_plan_correlated_subqueries_use_substitution` setting is enabled. [#82435](https://github.com/ClickHouse/ClickHouse/pull/82435) ([Dmitry Novik](https://github.com/novikd)). * Read only required columns in correlated subquery when it appears to be an argument of function `EXISTS`. [#82443](https://github.com/ClickHouse/ClickHouse/pull/82443) ([Dmitry Novik](https://github.com/novikd)). -* Introduce async logging. [#82516](https://github.com/ClickHouse/ClickHouse/pull/82516) ([Raúl Marín](https://github.com/Algunenano)). -* Speedup QueryTreeHash a bit. [#82617](https://github.com/ClickHouse/ClickHouse/pull/82617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Speedup comparisons of query trees during the query analysis a bit. [#82617](https://github.com/ClickHouse/ClickHouse/pull/82617) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Add alignment in the Counter of ProfileEvents to reduce false sharing. [#82697](https://github.com/ClickHouse/ClickHouse/pull/82697) ([Jiebin Sun](https://github.com/jiebinn)). -* Parallel distributed INSERT SELECT is enabled by default in mode where INSERT SELECT executed on each shard independently, see `parallel_distributed_insert_select` setting. [#83040](https://github.com/ClickHouse/ClickHouse/pull/83040) ([Igor Nikonov](https://github.com/devcrafter)). * The optimizations for `null_map` and `JoinMask` from [#82308](https://github.com/ClickHouse/ClickHouse/issues/82308) were applied to the case of JOIN with multiple disjuncts. Also, the `KnownRowsHolder` data structure was optimized. [#83041](https://github.com/ClickHouse/ClickHouse/pull/83041) ([Nikita Taranov](https://github.com/nickitat)). * Plain `std::vector` is used for join flags to avoid calculating a hash on each access to flags. [#83043](https://github.com/ClickHouse/ClickHouse/pull/83043) ([Nikita Taranov](https://github.com/nickitat)). * Don't pre-allocate memory for result columns beforehand when `HashJoin` uses `lazy` output mode. It is suboptimal, especially when the number of matches is low. Moreover, we know the exact amount of matches after joining is done, so we can preallocate more precisely. [#83304](https://github.com/ClickHouse/ClickHouse/pull/83304) ([Nikita Taranov](https://github.com/nickitat)). * Minimize memory copy in port headers during pipeline construction. Original [PR](https://github.com/ClickHouse/ClickHouse/pull/70105) by [heymind](https://github.com/heymind). [#83381](https://github.com/ClickHouse/ClickHouse/pull/83381) ([Raúl Marín](https://github.com/Algunenano)). -* Improve Keeper with rocksdb initial loading. [#83390](https://github.com/ClickHouse/ClickHouse/pull/83390) ([Antonio Andelic](https://github.com/antonio2368)). +* Improve the startup of clickhouse-keeper when it uses rocksdb storage. [#83390](https://github.com/ClickHouse/ClickHouse/pull/83390) ([Antonio Andelic](https://github.com/antonio2368)). * Avoid holding the lock while creating storage snapshot data to reduce lock contention with high concurrent load. [#83510](https://github.com/ClickHouse/ClickHouse/pull/83510) ([Duc Canh Le](https://github.com/canhld94)). -* Improved performance of the ProtobufSingle input format by reusing the serializer when no parsing errors occur. [#83613](https://github.com/ClickHouse/ClickHouse/pull/83613) ([Eduard Karacharov](https://github.com/korowa)). -* Improve the performance of pipeline building. [#83631](https://github.com/ClickHouse/ClickHouse/pull/83631) ([Raúl Marín](https://github.com/Algunenano)). -* Optimize MergeTreeReadersChain::getSampleBlock. [#83875](https://github.com/ClickHouse/ClickHouse/pull/83875) ([Raúl Marín](https://github.com/Algunenano)). +* Improved performance of the `ProtobufSingle` input format by reusing the serializer when no parsing errors occur. [#83613](https://github.com/ClickHouse/ClickHouse/pull/83613) ([Eduard Karacharov](https://github.com/korowa)). +* Improve the performance of pipeline building that speeds up short queries. [#83631](https://github.com/ClickHouse/ClickHouse/pull/83631) ([Raúl Marín](https://github.com/Algunenano)). +* Optimize `MergeTreeReadersChain::getSampleBlock` that speeds up short queries. [#83875](https://github.com/ClickHouse/ClickHouse/pull/83875) ([Raúl Marín](https://github.com/Algunenano)). +* Speedup tables listing in data catalogs by asynchronous requests. [#81084](https://github.com/ClickHouse/ClickHouse/pull/81084) ([alesapin](https://github.com/alesapin)). +* Introduce jitter to the S3 retry mechanism when the `s3_slow_all_threads_after_network_error` configuration is enabled. [#81849](https://github.com/ClickHouse/ClickHouse/pull/81849) ([zoomxi](https://github.com/zoomxi)). #### Improvement -* Introduced two new access types: `READ` and `WRITE` for sources and deprecates all previous access types related to sources. Before `GRANT S3 ON *.* TO user`, now: `GRANT READ, WRITE ON S3 TO user`. This also allows to separate `READ` and `WRITE` permissions for sources, e.g.: `GRANT READ ON * TO user`, `GRANT WRITE ON S3 TO user`. The feature is controlled by a setting `access_control_improvements.enable_read_write_grants` and disabled by default. [#73659](https://github.com/ClickHouse/ClickHouse/pull/73659) ([pufit](https://github.com/pufit)). -* Verify the part has consistent checksum.txt file right before committing it. [#76625](https://github.com/ClickHouse/ClickHouse/pull/76625) ([Sema Checherinda](https://github.com/CheSema)). -* Implement methods `moveFile` and `replaceFile` in s3_plain_rewritable to support it as a database disk. [#79424](https://github.com/ClickHouse/ClickHouse/pull/79424) ([Tuan Pham Anh](https://github.com/tuanpach)). -* Allow backups for PostgreSQL, MySQL & DataLake databases. A backup of such a database would only save the definition and not the data inside of it. [#79982](https://github.com/ClickHouse/ClickHouse/pull/79982) ([Nikolay Degterinsky](https://github.com/evillique)). -* Support position deletes for Iceberg TableEngine. [#80237](https://github.com/ClickHouse/ClickHouse/pull/80237) ([YanghongZhong](https://github.com/yahoNanJing)). -* Setting `allow_experimental_join_condition` marked as obsolete. [#80566](https://github.com/ClickHouse/ClickHouse/pull/80566) ([Vladimir Cherkasov](https://github.com/vdimir)). +* Color parenthesis in multiple colors for better readability. [#82538](https://github.com/ClickHouse/ClickHouse/pull/82538) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Highlight metacharacters in LIKE/REGEXP patterns as you type. We already have it in `clickhouse-format` and in the echo in `clickhouse-client`, but now it is done in the command prompt as well. [#82871](https://github.com/ClickHouse/ClickHouse/pull/82871) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Highlighting in `clickhouse-format` and in the client's echo will work in the same way as the highlighting in the command line prompt. [#82874](https://github.com/ClickHouse/ClickHouse/pull/82874) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Now `plain_rewritable` disks are allowed as disks for database metadata. Implement methods `moveFile` and `replaceFile` in `plain_rewritable` to support it as a database disk. [#79424](https://github.com/ClickHouse/ClickHouse/pull/79424) ([Tuan Pham Anh](https://github.com/tuanpach)). +* Allow backups for `PostgreSQL`, `MySQL` and `DataLake` databases. A backup of such a database would only save the definition and not the data inside of it. [#79982](https://github.com/ClickHouse/ClickHouse/pull/79982) ([Nikolay Degterinsky](https://github.com/evillique)). +* Setting `allow_experimental_join_condition` marked as obsolete, because it is now always allowed. [#80566](https://github.com/ClickHouse/ClickHouse/pull/80566) ([Vladimir Cherkasov](https://github.com/vdimir)). * Add pressure metrics to ClickHouse async metrics. [#80779](https://github.com/ClickHouse/ClickHouse/pull/80779) ([Xander Garbett](https://github.com/Garbett1)). * Added metrics `MarkCacheEvictedBytes`, `MarkCacheEvictedMarks`, `MarkCacheEvictedFiles` for tracking evictions from the mark cache. (issue [#60989](https://github.com/ClickHouse/ClickHouse/issues/60989)). [#80799](https://github.com/ClickHouse/ClickHouse/pull/80799) ([Shivji Kumar Jha](https://github.com/shiv4289)). -* Speedup tables listing in data catalogs by asynchronous requests. [#81084](https://github.com/ClickHouse/ClickHouse/pull/81084) ([alesapin](https://github.com/alesapin)). -* Support writing parquet enum as byte array as the [spec](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum) dictates. I'll write more info later. [#81090](https://github.com/ClickHouse/ClickHouse/pull/81090) ([Arthur Passos](https://github.com/arthurpassos)). +* Support writing Parquet enum as byte array as the [spec](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum) dictates. [#81090](https://github.com/ClickHouse/ClickHouse/pull/81090) ([Arthur Passos](https://github.com/arthurpassos)). * An improvement for `DeltaLake` table engine: delta-kernel-rs has `ExpressionVisitor` API which is implemented in this PR and is applied to partition column expressions transform (it will replace an old deprecated within the delta-kernel-rs way, which was used before in our code). In the future this `ExpressionVisitor` will also allow to implement statistics based pruning and some delta-lake proprietary features. Additionally the purpose of this change is to support partition pruning in `DeltaLakeCluster` table engine (the result of a parsed expression - ActionsDAG - will be serialized and sent from the initiator along with the data path, because this kind of information, which is needed for pruning, is only available as meta information on data files listing, which is done by initiator only, but it has to be applied to data on each reading server). [#81136](https://github.com/ClickHouse/ClickHouse/pull/81136) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Try to preserve element names when deriving supertypes for named tuples. [#81345](https://github.com/ClickHouse/ClickHouse/pull/81345) ([lgbo](https://github.com/lgbo-ustc)). -* Allow parameters in `CREATE USER` queries for usernames. [#81387](https://github.com/ClickHouse/ClickHouse/pull/81387) ([Diskein](https://github.com/Diskein)). -* Now clickhouse supports compressed `metadata.json` files for Iceberg. Fixes [#70874](https://github.com/ClickHouse/ClickHouse/issues/70874). [#81451](https://github.com/ClickHouse/ClickHouse/pull/81451) ([alesapin](https://github.com/alesapin)). -* The `system.formats` table now contains extended information about formats, such as HTTP content type, the capabilities of schema inference, etc. [#81505](https://github.com/ClickHouse/ClickHouse/pull/81505) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Preserve element names when deriving supertypes for named tuples. [#81345](https://github.com/ClickHouse/ClickHouse/pull/81345) ([lgbo](https://github.com/lgbo-ustc)). * Count consumed messages manually to avoid depending on previous committed offset in StorageKafka2. [#81662](https://github.com/ClickHouse/ClickHouse/pull/81662) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). * Added `clickhouse-keeper-utils`, a new command-line tool for managing and analyzing ClickHouse Keeper data. The tool supports dumping state from snapshots and changelogs, analyzing changelog files, and extracting specific log ranges. [#81677](https://github.com/ClickHouse/ClickHouse/pull/81677) ([Antonio Andelic](https://github.com/antonio2368)). * The total and per-user network throttlers are never reset, which ensures that `max_network_bandwidth_for_all_users` and `max_network_bandwidth_for_all_users` limits are never exceeded. [#81729](https://github.com/ClickHouse/ClickHouse/pull/81729) ([Sergei Trifonov](https://github.com/serxa)). * Support writing geoparquets as output format. [#81784](https://github.com/ClickHouse/ClickHouse/pull/81784) ([Konstantin Vedernikov](https://github.com/scanhex12)). * Forbid to start `RENAME COLUMN` alter mutation if it will rename some column that right now affected by incomplete data mutation. [#81823](https://github.com/ClickHouse/ClickHouse/pull/81823) ([Mikhail Artemenko](https://github.com/Michicosun)). -* Introduce jitter to the S3 retry mechanism when the `s3_slow_all_threads_after_network_error` configuration is enabled. [#81849](https://github.com/ClickHouse/ClickHouse/pull/81849) ([zoomxi](https://github.com/zoomxi)). * Header Connection is send at the end of headers. When we know is the connection should be preserved. [#81951](https://github.com/ClickHouse/ClickHouse/pull/81951) ([Sema Checherinda](https://github.com/CheSema)). * Tune TCP servers queue (64 by default) based on listen_backlog (4096 by default). [#82045](https://github.com/ClickHouse/ClickHouse/pull/82045) ([Azat Khuzhin](https://github.com/azat)). * Add ability to reload `max_local_read_bandwidth_for_server` and `max_local_write_bandwidth_for_server` on fly without restart server. [#82083](https://github.com/ClickHouse/ClickHouse/pull/82083) ([Kai Zhu](https://github.com/nauu)). * Add support for clearing all warnings from the `system.warnings` table using `TRUNCATE TABLE system.warnings`. [#82087](https://github.com/ClickHouse/ClickHouse/pull/82087) ([Vladimir Cherkasov](https://github.com/vdimir)). * Fix partition pruning with data lake cluster functions. [#82131](https://github.com/ClickHouse/ClickHouse/pull/82131) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fix reading partitioned data in DeltaLakeCluster table function. In this PR cluster functions protocol version is increased, allowing to send extra info from initiator to replicas. This extra info contains delta-kernel transform expression, which is needed to parse partition columns (and some other staff in the future, like generated columns, etc). [#82132](https://github.com/ClickHouse/ClickHouse/pull/82132) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix a list of problems that can occur when trying to run integration tests on a local host. [#82135](https://github.com/ClickHouse/ClickHouse/pull/82135) ([Oleg Doronin](https://github.com/dorooleg)). +* Function `reinterpret` function now supports conversion to `Array(T)` where `T` is a fixed-size data type (issue [#82621](https://github.com/ClickHouse/ClickHouse/issues/82621)). [#83399](https://github.com/ClickHouse/ClickHouse/pull/83399) ([Shankar Iyer](https://github.com/shankar-iyer)). * Now database Datalake throw more convenient exception. Fixes [#81211](https://github.com/ClickHouse/ClickHouse/issues/81211). [#82304](https://github.com/ClickHouse/ClickHouse/pull/82304) ([alesapin](https://github.com/alesapin)). -* Improve HashJoin::needUsedFlagsForPerRightTableRow, returns false for cross join. [#82379](https://github.com/ClickHouse/ClickHouse/pull/82379) ([lgbo](https://github.com/lgbo-ustc)). -* Allow write/read map columns as array of tuples. [#82408](https://github.com/ClickHouse/ClickHouse/pull/82408) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). -* List the licenses of rust crates in system.licenses. [#82440](https://github.com/ClickHouse/ClickHouse/pull/82440) ([Raúl Marín](https://github.com/Algunenano)). +* Improve CROSS JOIN by returning false from `HashJoin::needUsedFlagsForPerRightTableRow`. [#82379](https://github.com/ClickHouse/ClickHouse/pull/82379) ([lgbo](https://github.com/lgbo-ustc)). +* Allow write/read map columns as Array of Tuples. [#82408](https://github.com/ClickHouse/ClickHouse/pull/82408) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* List the licenses of [Rust](https://clickhouse.com/blog/rust) crates in `system.licenses`. [#82440](https://github.com/ClickHouse/ClickHouse/pull/82440) ([Raúl Marín](https://github.com/Algunenano)). * Macros like `{uuid}` can now be used in the `keeper_path` setting of the S3Queue table engine. [#82463](https://github.com/ClickHouse/ClickHouse/pull/82463) ([Nikolay Degterinsky](https://github.com/evillique)). * Keeper improvement: move changelog files between disk in a background thread. Previously, moving changelog to a different disk would block Keeper globally until the move is finished. This lead to performance degradation if moving is a long operation (e.g. to S3 disk). [#82485](https://github.com/ClickHouse/ClickHouse/pull/82485) ([Antonio Andelic](https://github.com/antonio2368)). * Keeper improvement: add new config `keeper_server.cleanup_old_and_ignore_new_acl`. If enabled, all nodes will have their ACLs cleared while ACL for new requests will be ignored. If the goal is to completely remove ACL from nodes, it's important to leave the config enabled until a new snapshot is created. [#82496](https://github.com/ClickHouse/ClickHouse/pull/82496) ([Antonio Andelic](https://github.com/antonio2368)). -* Removed experimental `send_metadata` logic related to experimental zero-copy replication. It wasn't ever used and nobody supports this code. Since there were even no tests related to it, there is a high chance that it's broken long time ago. [#82508](https://github.com/ClickHouse/ClickHouse/pull/82508) ([alesapin](https://github.com/alesapin)). * Added a new server setting `s3queue_disable_streaming` which disables streaming in tables with S3Queue table engine. This setting is changeable without server restart. [#82515](https://github.com/ClickHouse/ClickHouse/pull/82515) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Color parenthesis in multiple colors for better readability. [#82538](https://github.com/ClickHouse/ClickHouse/pull/82538) ([Konstantin Bogdanov](https://github.com/thevar1able)). * Refactor dynamic resize feature of filesystem cache. Added more logs for introspection. [#82556](https://github.com/ClickHouse/ClickHouse/pull/82556) ([Kseniia Sumarokova](https://github.com/kssenii)). * `clickhouse-server` without a configuration file will also listen to the PostgreSQL port 9005, like with the default config. [#82633](https://github.com/ClickHouse/ClickHouse/pull/82633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Integrate `StorageKafka2` to `system.kafka_consumers`. [#82652](https://github.com/ClickHouse/ClickHouse/pull/82652) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). -* Estimate complex cnf/dnf, for example, `(a < 1 and a > 0) or b = 3`, by statistics. [#82663](https://github.com/ClickHouse/ClickHouse/pull/82663) ([Han Fei](https://github.com/hanfei1991)). * In `ReplicatedMergeTree::executeMetadataAlter`, we get the StorageID, and without taking DDLGuard, we try to call `IDatabase::alterTable`. In between this time we could have technically exchanged the table in question with another table, so when we get the definiton we would get the wrong one. To avoid this we add a separate check for UUIDs to match when we try to call `IDatabase::alterTable`. [#82666](https://github.com/ClickHouse/ClickHouse/pull/82666) ([Nikolay Degterinsky](https://github.com/evillique)). * When attaching a database with a read-only remote disk, manually add table UUIDs into DatabaseCatalog. [#82670](https://github.com/ClickHouse/ClickHouse/pull/82670) ([Tuan Pham Anh](https://github.com/tuanpach)). * Prevent user from using `nan` and `inf` with `NumericIndexedVector`. Fixes [#82239](https://github.com/ClickHouse/ClickHouse/issues/82239) and a little more. [#82681](https://github.com/ClickHouse/ClickHouse/pull/82681) ([Raufs Dunamalijevs](https://github.com/rienath)). * Do not omit zero values in the `X-ClickHouse-Progress` and `X-ClickHouse-Summary` header formats. [#82727](https://github.com/ClickHouse/ClickHouse/pull/82727) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Keeper improvement: support specific permissions for world:anyone ACL. [#82755](https://github.com/ClickHouse/ClickHouse/pull/82755) ([Antonio Andelic](https://github.com/antonio2368)). -* Do not allow RENAME COLUMN or DROP COLUMN involving explicitly listed columns to sum in SummingMergeTree. Closes [#81836](https://github.com/ClickHouse/ClickHouse/issues/81836). [#82821](https://github.com/ClickHouse/ClickHouse/pull/82821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not allow `RENAME COLUMN` or `DROP COLUMN` involving explicitly listed columns to sum in SummingMergeTree. Closes [#81836](https://github.com/ClickHouse/ClickHouse/issues/81836). [#82821](https://github.com/ClickHouse/ClickHouse/pull/82821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Improve the precision of conversion from `Decimal` to `Float32`. Implement conversion from `Decimal` to `BFloat16`. Closes [#82660](https://github.com/ClickHouse/ClickHouse/issues/82660). [#82823](https://github.com/ClickHouse/ClickHouse/pull/82823) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Scrollbars in the Web UI will look slightly better. [#82869](https://github.com/ClickHouse/ClickHouse/pull/82869) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * `clickhouse-server` with embedded configuration will allow using the Web UI by providing an HTTP OPTIONS response. [#82870](https://github.com/ClickHouse/ClickHouse/pull/82870) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Highlight metacharacters in LIKE/REGEXP patterns as you type. We already have it in `clickhouse-format` and in the echo in `clickhouse-client`, but now it is done in the command prompt as well. [#82871](https://github.com/ClickHouse/ClickHouse/pull/82871) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Highlighting in `clickhouse-format` and in the client's echo will work in the same way as the highlighting in the command line prompt. [#82874](https://github.com/ClickHouse/ClickHouse/pull/82874) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Add support for specifying extra Keeper ACL for paths in config. If you want to add extra ACL for a specific path you define it in the config under `zookeeper.path_acls`. [#82898](https://github.com/ClickHouse/ClickHouse/pull/82898) ([Antonio Andelic](https://github.com/antonio2368)). -* Add function to write types into wkb format. [#82935](https://github.com/ClickHouse/ClickHouse/pull/82935) ([Konstantin Vedernikov](https://github.com/scanhex12)). * Now mutations snapshot will be built from the visible parts snapshot. Also mutation counters used in snapshot will be recalculated from the included mutations. [#82945](https://github.com/ClickHouse/ClickHouse/pull/82945) ([Mikhail Artemenko](https://github.com/Michicosun)). * Adds ProfileEvent when Keeper rejects a write due to soft memory limit. [#82963](https://github.com/ClickHouse/ClickHouse/pull/82963) ([Xander Garbett](https://github.com/Garbett1)). * Add columns `commit_time`, `commit_id` to `system.s3queue_log`. [#83016](https://github.com/ClickHouse/ClickHouse/pull/83016) ([Kseniia Sumarokova](https://github.com/kssenii)). -* In some cases, we need to have multiple dimensions to our metrics. For example, counting failed merges or mutations by error codes rather than having a single counter. [#83030](https://github.com/ClickHouse/ClickHouse/pull/83030) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). +* In some cases, we need to have multiple dimensions to our metrics. For example, counting failed merges or mutations by error codes rather than having a single counter. Introduce `system.dimensional_metrics`, which does precisely that and adds the first dimensional metric called `failed_merges`. [#83030](https://github.com/ClickHouse/ClickHouse/pull/83030) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). * Consolidate unknown settings warnings in clickhouse client and log them as a summary. [#83042](https://github.com/ClickHouse/ClickHouse/pull/83042) ([Bharat Nallan](https://github.com/bharatnc)). * Clickhouse client now reports the local port when connection error happens. [#83050](https://github.com/ClickHouse/ClickHouse/pull/83050) ([Jianfei Hu](https://github.com/incfly)). * Slightly better error handling in `AsynchronousMetrics`. If the `/sys/block` directory exists but is not accessible, the server will start without monitoring the block devices. Closes [#79229](https://github.com/ClickHouse/ClickHouse/issues/79229). [#83115](https://github.com/ClickHouse/ClickHouse/pull/83115) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Support `TimestampTZ` in Glue catalog. This closes [#81654](https://github.com/ClickHouse/ClickHouse/issues/81654). [#83132](https://github.com/ClickHouse/ClickHouse/pull/83132) ([Konstantin Vedernikov](https://github.com/scanhex12)). * Shutdown SystemLogs after ordinary tables (and before system tables, instead of before ordinary). [#83134](https://github.com/ClickHouse/ClickHouse/pull/83134) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add logs for s3queue shutdown process. [#83163](https://github.com/ClickHouse/ClickHouse/pull/83163) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Async logs: Limit the max number of entries that are hold in the queue. [#83214](https://github.com/ClickHouse/ClickHouse/pull/83214) ([Raúl Marín](https://github.com/Algunenano)). -* Possibility to parse Time and Time64 as MM:SS, M:SS, SS, or S. [#83299](https://github.com/ClickHouse/ClickHouse/pull/83299) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Add logs for `S3Queue` shutdown process. [#83163](https://github.com/ClickHouse/ClickHouse/pull/83163) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Possibility to parse `Time` and `Time64` as `MM:SS`, `M:SS`, `SS`, or `S`. [#83299](https://github.com/ClickHouse/ClickHouse/pull/83299) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). * When `distributed_ddl_output_mode='*_only_active'`, don't wait for new or recovered replicas that have replication lag bigger than `max_replication_lag_to_enqueue`. This should help to avoid `DDL task is not finished on some hosts` when a new replica becomes active after finishing initialization or recovery, but it accumulated huge replication log while initializing. Also, implement `SYSTEM SYNC DATABASE REPLICA STRICT` query that waits for replication log to become below `max_replication_lag_to_enqueue`. [#83302](https://github.com/ClickHouse/ClickHouse/pull/83302) ([Alexander Tokmakov](https://github.com/tavplubix)). * Do not output too long descriptions of expression actions in exception messages. Closes [#83164](https://github.com/ClickHouse/ClickHouse/issues/83164). [#83350](https://github.com/ClickHouse/ClickHouse/pull/83350) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Add ability to parse part's prefix and suffix and also check coverage for non constant columns. [#83377](https://github.com/ClickHouse/ClickHouse/pull/83377) ([Mikhail Artemenko](https://github.com/Michicosun)). -* Function `reinterpret()` function now supports conversion to `Array(T)` where `T` is a fixed-size data type (issue [#82621](https://github.com/ClickHouse/ClickHouse/issues/82621)). [#83399](https://github.com/ClickHouse/ClickHouse/pull/83399) ([Shankar Iyer](https://github.com/shankar-iyer)). * Unify parameter names in ODBC and JDBC when using named collections. [#83410](https://github.com/ClickHouse/ClickHouse/pull/83410) ([Andrey Zvonov](https://github.com/zvonand)). * When the storage is shutting down, `getStatus` throws an `ErrorCodes::ABORTED` exception. Previously, this would fail the select query. Now we catch the `ErrorCodes::ABORTED` exceptions and intentionally ignore them instead. [#83435](https://github.com/ClickHouse/ClickHouse/pull/83435) ([Miсhael Stetsyuk](https://github.com/mstetsyuk)). * Add process resource metrics (such as `UserTimeMicroseconds`, `SystemTimeMicroseconds`, `RealTimeMicroseconds`) to part_log profile events for `MergeParts` entries. [#83460](https://github.com/ClickHouse/ClickHouse/pull/83460) ([Vladimir Cherkasov](https://github.com/vdimir)). * Enable `create_if_not_exists`, `check_not_exists`, `remove_recursive` feature flags in Keeper by default which enable new types of requests. [#83488](https://github.com/ClickHouse/ClickHouse/pull/83488) ([Antonio Andelic](https://github.com/antonio2368)). * Shutdown S3(Azure/etc)Queue streaming before shutting down any tables on server shutdown. [#83530](https://github.com/ClickHouse/ClickHouse/pull/83530) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Enable Date/Date32 as integers in JSON input formats. [#83597](https://github.com/ClickHouse/ClickHouse/pull/83597) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Enable `Date`/`Date32` as integers in `JSON` input formats. [#83597](https://github.com/ClickHouse/ClickHouse/pull/83597) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). * Made exception messages for certain situations for loading and adding projections easier to read. [#83728](https://github.com/ClickHouse/ClickHouse/pull/83728) ([Robert Schulze](https://github.com/rschu1ze)). * Introduce a configuration option to skip binary checksum integrity checks for `clickhouse-server`. Resolves [#83637](https://github.com/ClickHouse/ClickHouse/issues/83637). [#83749](https://github.com/ClickHouse/ClickHouse/pull/83749) ([Rafael Roquetto](https://github.com/rafaelroquetto)). @@ -266,16 +576,16 @@ title: '2025 Changelog' * Fix deadlock on shutdown due to recursive context locking during library bridge cleanup. [#83824](https://github.com/ClickHouse/ClickHouse/pull/83824) ([Azat Khuzhin](https://github.com/azat)). #### Build/Testing/Packaging Improvement -* Build a minimal C library (10 KB) for the ClickHouse lexer. This is needed for [#80977](https://github.com/ClickHouse/ClickHouse/issues/80977). [#81347](https://github.com/ClickHouse/ClickHouse/pull/81347) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Build a minimal C library (10 KB) for the ClickHouse lexer. This is needed for [#80977](https://github.com/ClickHouse/ClickHouse/issues/80977). [#81347](https://github.com/ClickHouse/ClickHouse/pull/81347) ([Alexey Milovidov](https://github.com/alexey-milovidov)). Add test for standalone lexer, add test tag `fasttest-only`. [#82472](https://github.com/ClickHouse/ClickHouse/pull/82472) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Add a check for Nix submodule inputs. [#81691](https://github.com/ClickHouse/ClickHouse/pull/81691) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fix a list of problems that can occur when trying to run integration tests on a localhost. [#82135](https://github.com/ClickHouse/ClickHouse/pull/82135) ([Oleg Doronin](https://github.com/dorooleg)). * Compile SymbolIndex on Mac and FreeBSD. (But it will work only on ELF systems, Linux and FreeBSD). [#82347](https://github.com/ClickHouse/ClickHouse/pull/82347) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add test for standalone lexer, add test tag `fasttest-only`. [#82472](https://github.com/ClickHouse/ClickHouse/pull/82472) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Bumped Azure SDK to v1.15.0. [#82747](https://github.com/ClickHouse/ClickHouse/pull/82747) ([Smita Kulkarni](https://github.com/SmitaRKulkarni)). * Add storage module from google-cloud-cpp to build system. [#82881](https://github.com/ClickHouse/ClickHouse/pull/82881) ([Pablo Marcos](https://github.com/pamarcos)). * Change `Dockerfile.ubuntu` for clickhouse-server to fit requirements in Docker Official Library. [#83039](https://github.com/ClickHouse/ClickHouse/pull/83039) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * A follow-up for [#83158](https://github.com/ClickHouse/ClickHouse/issues/83158) to fix uploading builds to `curl clickhouse.com`. [#83463](https://github.com/ClickHouse/ClickHouse/pull/83463) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Adding `busybox` binary and install tools in `clickhouse/clickhouse-server` and official `clickhouse` images. [#83735](https://github.com/ClickHouse/ClickHouse/pull/83735) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Added support for the **`CLICKHOUSE_HOST` environment variable** to specify the ClickHouse server host, aligning with existing `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` environment variables. This allows for easier configuration without modifying client or configuration files directly. [#83659](https://github.com/ClickHouse/ClickHouse/pull/83659) ([Doron David](https://github.com/dorki)). +* Added support for the `CLICKHOUSE_HOST` environment variable to specify the ClickHouse server host, aligning with existing `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` environment variables. This allows for easier configuration without modifying client or configuration files directly. [#83659](https://github.com/ClickHouse/ClickHouse/pull/83659) ([Doron David](https://github.com/dorki)). ### ClickHouse release 25.6, 2025-06-26 {#256} diff --git a/docusaurus.config.en.js b/docusaurus.config.en.js index ad0d063e997..94cffb9f0f0 100644 --- a/docusaurus.config.en.js +++ b/docusaurus.config.en.js @@ -186,6 +186,9 @@ const config = { blogPath ); }, + remarkPlugins: [math, remarkCustomBlocks, glossaryTransformer], + beforeDefaultRemarkPlugins: [fixLinks], + rehypePlugins: [katex], }, pages: { diff --git a/docusaurus.config.jp.js b/docusaurus.config.jp.js index 742e74a0c91..4efd9ee3c43 100644 --- a/docusaurus.config.jp.js +++ b/docusaurus.config.jp.js @@ -5,6 +5,7 @@ import chHeader from "./plugins/header.js"; import fixLinks from "./src/hooks/fixLinks.js"; import prismLight from "./src/utils/prismLight"; import prismDark from "./src/utils/prismDark"; +import glossaryTransformer from "./plugins/glossary-transformer.js"; const remarkCustomBlocks = require('./plugins/remark-custom-blocks'); // Helper function to skip over index.md files. @@ -154,6 +155,9 @@ const config = { blogPath ); }, + remarkPlugins: [math, remarkCustomBlocks, glossaryTransformer], + beforeDefaultRemarkPlugins: [fixLinks], + rehypePlugins: [katex], }, theme: { customCss: [require.resolve("./src/css/custom.scss")], diff --git a/docusaurus.config.ru.js b/docusaurus.config.ru.js index 2369b02c088..ec5e39c1b72 100644 --- a/docusaurus.config.ru.js +++ b/docusaurus.config.ru.js @@ -5,6 +5,7 @@ import chHeader from "./plugins/header.js"; import fixLinks from "./src/hooks/fixLinks.js"; import prismLight from "./src/utils/prismLight"; import prismDark from "./src/utils/prismDark"; +import glossaryTransformer from "./plugins/glossary-transformer.js"; const remarkCustomBlocks = require('./plugins/remark-custom-blocks'); // Helper function to skip over index.md files. @@ -155,6 +156,9 @@ const config = { blogPath ); }, + remarkPlugins: [math, remarkCustomBlocks, glossaryTransformer], + beforeDefaultRemarkPlugins: [fixLinks], + rehypePlugins: [katex], }, theme: { customCss: [require.resolve("./src/css/custom.scss")], diff --git a/docusaurus.config.zh.js b/docusaurus.config.zh.js index ecff04ba03a..2f9e7739cff 100644 --- a/docusaurus.config.zh.js +++ b/docusaurus.config.zh.js @@ -5,6 +5,7 @@ import chHeader from "./plugins/header.js"; import fixLinks from "./src/hooks/fixLinks.js"; import prismLight from "./src/utils/prismLight"; import prismDark from "./src/utils/prismDark"; +import glossaryTransformer from "./plugins/glossary-transformer.js"; const remarkCustomBlocks = require('./plugins/remark-custom-blocks'); // Helper function to skip over index.md files. @@ -154,6 +155,9 @@ const config = { blogPath ); }, + remarkPlugins: [math, remarkCustomBlocks, glossaryTransformer], + beforeDefaultRemarkPlugins: [fixLinks], + rehypePlugins: [katex], }, theme: { customCss: [require.resolve("./src/css/custom.scss")], diff --git a/knowledgebase/marketplace.mdx b/knowledgebase/marketplace.mdx deleted file mode 100644 index 169063817af..00000000000 --- a/knowledgebase/marketplace.mdx +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Marketplace -description: "Learn how to use CSP marketplaces to subscribe to ClickHouse Cloud." -date: 2022-12-06 -tags: ['Managing Cloud'] -keywords: ['CSP Marketplaces'] ---- - -import Content from '@site/docs/cloud/manage/billing/marketplace/index.md'; - -{frontMatter.description} -{/* truncate */} - -## Marketplace {#marketplace} - -Details on using CSP marketplaces to subscribe to ClickHouse Cloud. - - diff --git a/docs/faq/troubleshooting.md b/knowledgebase/unable-to-access-cloud-service.mdx similarity index 62% rename from docs/faq/troubleshooting.md rename to knowledgebase/unable-to-access-cloud-service.mdx index 4b1221d7dea..46fc174e860 100644 --- a/docs/faq/troubleshooting.md +++ b/knowledgebase/unable-to-access-cloud-service.mdx @@ -1,12 +1,15 @@ --- -title: 'Troubleshooting' -slug: /faq/troubleshooting -description: 'How to troubleshoot common ClickHouse Cloud error messages.' +title: Tips and tricks on optimizing basic data types in ClickHouse +description: "Tips and tricks on optimizing basic data types in ClickHouse" +date: 2024-07-02 +tags: ['Errors and Exceptions'] +keywords: ['accessing cloud service'] --- -## ClickHouse Cloud troubleshooting {#clickhouse-cloud-troubleshooting} +{frontMatter.description} +{/* truncate */} -### Unable to access a ClickHouse Cloud service {#unable-to-access-a-clickhouse-cloud-service} +## I am unable to access a ClickHouse Cloud service {#i-am-unable-to-access-a-clickhouse-cloud-service} If you are seeing an error message like one of these, your IP Access List may be denying access: diff --git a/plugins/remark-custom-blocks.js b/plugins/remark-custom-blocks.js index 5e857e05d20..6c6895fd2df 100644 --- a/plugins/remark-custom-blocks.js +++ b/plugins/remark-custom-blocks.js @@ -76,7 +76,7 @@ const plugin = (options) => { if (child.type === 'heading' && child.depth === headerLevel) { finalizeStep(); // Finalize the previous step first currentStepLabel = extractText(child.children); - currentAnchorId = child.data.hProperties.id; + currentAnchorId = child.data?.hProperties?.id || null; currentStepId = `step-${total_steps}`; // Generate step-X ID currentStepContent.push(child); // We need the header otherwise onBrokenAnchors fails } else if (currentStepLabel) { diff --git a/scripts/aspell-dict-file.txt b/scripts/aspell-dict-file.txt index 72d87ddd127..ae9a5a3c439 100644 --- a/scripts/aspell-dict-file.txt +++ b/scripts/aspell-dict-file.txt @@ -300,7 +300,7 @@ dataview --docs/integrations/data-ingestion/clickpipes/postgres/source/supabase.md-- pooler supabase ---docs/cloud/security/saml-sso-setup.md-- +--docs/cloud/guides/security/cloud_access_management/saml-sso-setup.md-- IdP IdPs Entra @@ -327,7 +327,7 @@ typings Rockset's Workspaces workspaces ---docs/cloud/security/azure-privatelink.md-- +--docs/cloud/features/04_security/connectivity/private_networking/azure-privatelink.md-- privatelink VNets guid @@ -354,15 +354,12 @@ UserIDs --docs/cloud/security/shared-responsibility-model.md-- Entra RelayState ---docs/cloud/manage/_snippets/_network_transfer_rates.md-- +--docs/cloud/reference/_snippets/_network_transfer_rates.md-- asia eastus europe germanywestcentral westus ---docs/cloud/manage/jan2025_faq/dimensions.md-- -intra -clickpipesPricingFaq --docs/cloud/manage/backups.md-- slideout --docs/integrations/data-ingestion/clickpipes/postgres/table_resync.md-- @@ -403,14 +400,14 @@ pageId rdhnsx sessionId ybtm ---docs/cloud/reference/supported-regions.md-- +--docs/cloud/reference/05_supported-regions.md-- asia australia europe Montréal northamerica JapanEast ---docs/cloud/reference/byoc.md-- +--docs/cloud/features/02_infrastructure_and_deploy/byoc.md-- Acceptor Autoscaler byoc @@ -957,7 +954,7 @@ payg bursty --docs/cloud/security/gcp-private-service-connect.md-- privatelink ---docs/cloud/security/aws-privatelink.md-- +--docs/cloud/features/04_security/connectivity/private_networking/aws-privatelink.md-- dnsname nsname pecreate @@ -1001,7 +998,7 @@ Probabilistically tunable --docs/best-practices/use_materialized_views.md-- DAGs ---docs/migrations/postgres/appendix.md-- +--docs/cloud/onboard/02_migrate/01_migration_guides/02_postgres/appendix.md-- Citus --docs/integrations/data-ingestion/azure-synapse/index.md-- microsoft @@ -1094,7 +1091,15 @@ MCP's daemonset --docs/use-cases/observability/clickstack/ingesting-data/kubernetes.md-- daemonset +--docs/cloud/onboard/01_discover/02_use_cases/04_machine_learning_and_genAI/02_agent_facing_analytics.md-- +AgentForce +DeepSeek +OpenAI's +PDFs +ReAct +ServiceNow +explorative --docs/integrations/data-ingestion/clickpipes/postgres/faq.md-- ReceiveMessage --docs/integrations/data-ingestion/clickpipes/postgres/postgres_generated_columns.md-- -RelationMessage +RelationMessage \ No newline at end of file diff --git a/scripts/aspell-ignore/en/aspell-dict.txt b/scripts/aspell-ignore/en/aspell-dict.txt index 0f545b0fcb6..ceb9dac151c 100644 --- a/scripts/aspell-ignore/en/aspell-dict.txt +++ b/scripts/aspell-ignore/en/aspell-dict.txt @@ -1,9 +1,11 @@ personal_ws-1.1 en 3638 AArch ACLs +Accepter AICPA ALTERs AMPLab +AmazonKinesis AMQP ANNIndex ANNIndexes @@ -58,6 +60,7 @@ Authenticators Authy AutoFDO AutoML +Autoscaler Autocompletion AvroConfluent AzureQueue @@ -324,6 +327,7 @@ DestroyAggregatesThreads DestroyAggregatesThreadsActive DictCacheRequests DigiCert +DigitalOcean DiskAvailable DiskObjectStorage DiskObjectStorageAsyncThreads @@ -1076,6 +1080,7 @@ Refactorings ReferenceKeyed Refreshable RegexpTree +RelationMessage RemoteRead ReplacingMergeTree ReplacingReplicatedMergeTree @@ -1374,6 +1379,7 @@ VARCHAR VIEWs VLDB VNet +VNets VPCs VPNs Vadim @@ -2266,8 +2272,10 @@ hiveHash hnsw holistics homebrew +homebrew hopEnd hopStart +Hopsworks horgh hostName hostname @@ -2409,6 +2417,7 @@ kusto lagInFrame laion lakehouse +Lakehouses lang laravel largestTriangleThreeBuckets @@ -3013,6 +3022,7 @@ resending resharding reshards resolvers +resourceGUID restartable resultset resync @@ -3645,3 +3655,22 @@ znode znodes zookeeperSessionUptime zstd +Coinhall +Instacart +SingleStore +Fastly +Fong +MTTD +MTTR +O'Reilly +transformative +MLOps +chatbots +SageMaker +GWLBs +NLBs +explorative +pointwise +summarization +reusability +lakehouses \ No newline at end of file diff --git a/scripts/autogenerate-table-of-contents.sh b/scripts/autogenerate-table-of-contents.sh index 1d614c97c56..09c96c3dfdd 100644 --- a/scripts/autogenerate-table-of-contents.sh +++ b/scripts/autogenerate-table-of-contents.sh @@ -40,8 +40,7 @@ COMMANDS=( '--single-toc --dir="docs/sql-reference/aggregate-functions/reference" --md="docs/sql-reference/aggregate-functions/reference/index.md"' '--single-toc --dir="docs/sql-reference/table-functions" --md="docs/sql-reference/table-functions/index.md"' '--single-toc --dir="docs/chdb/guides" --md="docs/chdb/guides/index.md" --ignore images' - '--single-toc --dir="docs/cloud/manage/jan2025_faq" --md="docs/cloud/manage/jan2025_faq/index.md" --ignore images' - '--single-toc --dir="docs/cloud/changelogs" --md="docs/cloud/reference/release-notes-index.md"' + '--single-toc --dir="docs/cloud/reference/01_changelog" --md="docs/cloud/reference/01_changelog/02_release_notes/index.md"' '--single-toc --dir="docs/development" --md="docs/development/index.md" --ignore images' '--single-toc --dir="docs/getting-started/example-datasets" --md="docs/getting-started/index.md" --ignore images' '--single-toc --dir="docs/integrations/data-ingestion/clickpipes/kafka" --md="docs/integrations/data-ingestion/clickpipes/kafka/index.md" --ignore images' diff --git a/scripts/check-doc-aspell b/scripts/check-doc-aspell index cd7f412a9d2..bc79d14dc68 100755 --- a/scripts/check-doc-aspell +++ b/scripts/check-doc-aspell @@ -131,6 +131,7 @@ IGNORE_LIST=( "${ROOT_PATH}/docs/whats-new/security-changelog.md" "${ROOT_PATH}/docs/cloud/changelogs/*" "${ROOT_PATH}/docs/whats-new/changelog/*" # ignore all changelogs + "${ROOT_PATH}/docs/cloud/reference/01_changelog/*" "${ROOT_PATH}/docs/sql-reference/*" # we ignore all files from ClickHouse/ClickHouse "${ROOT_PATH}/docs/engines/*" "${ROOT_PATH}/docs/getting-started/*" @@ -141,6 +142,7 @@ IGNORE_LIST=( "${ROOT_PATH}/docs/zh/*" "${ROOT_PATH}/docs/cloud/manage/api/*" # api files "${ROOT_PATH}/docs/index.md" + "${ROOT_PATH}/docs/about-us/beta-and-experimental-features.md" # Add more files to ignore here, e.g.: # "${ROOT_PATH}/docs/some-other-file.md" ) diff --git a/scripts/sed_links.sh b/scripts/sed_links.sh index 6a21ae6cc43..707c36df5ed 100755 --- a/scripts/sed_links.sh +++ b/scripts/sed_links.sh @@ -14,6 +14,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then sed -i '' 's|(https://clickhouse.com/docs/sql-reference/statements/select#apply)|(/sql-reference/statements/select)|g' docs/guides/developer/dynamic-column-selection.md sed -i '' 's|(/sql-reference/statements/select#replace)|(/sql-reference/statements/select)|g' docs/guides/developer/dynamic-column-selection.md sed -i '' 's|(/sql-reference/statements/select#except)|(/sql-reference/statements/select)|g' docs/guides/developer/dynamic-column-selection.md + sed -i '' 's|(/cloud/reference/cloud-compatibility.md)|(/whats-new/cloud-compatibility)|g' docs/sql-reference/dictionaries/_snippet_dictionary_in_cloud.md else # Linux sed -i 's|(../../quick-start\.mdx)|(/get-started/quick-start)|g' docs/operations/utilities/clickhouse-local.md @@ -21,4 +22,5 @@ else sed -i 's|(https://clickhouse.com/docs/sql-reference/statements/select#apply)|(/sql-reference/statements/select)|g' docs/guides/developer/dynamic-column-selection.md sed -i 's|(/sql-reference/statements/select#replace)|(/sql-reference/statements/select)|g' docs/guides/developer/dynamic-column-selection.md sed -i 's|(/sql-reference/statements/select#except)|(/sql-reference/statements/select)|g' docs/guides/developer/dynamic-column-selection.md + sed -i 's|(/cloud/reference/cloud-compatibility.md)|(/whats-new/cloud-compatibility)|g' docs/sql-reference/dictionaries/_snippet_dictionary_in_cloud.md fi diff --git a/sidebars.js b/sidebars.js index 13248a353ae..b5cf31adbb1 100644 --- a/sidebars.js +++ b/sidebars.js @@ -196,61 +196,6 @@ const sidebars = { }, ] }, - { - type: "category", - label: "Migration Guides", - collapsed: false, - collapsible: false, - link: { type: "doc", id: "migrations/index" }, - items: [ - { - type: "category", - label: "BigQuery", - link: { type: "doc", id: "migrations/bigquery/index" }, - items: [ - { - type: "doc", - id: "migrations/bigquery/equivalent-concepts", - }, - { - type: "doc", - id: "migrations/bigquery/migrating-to-clickhouse-cloud", - }, - { - type: "doc", - id: "migrations/bigquery/loading-data", - }, - ] - }, - "migrations/snowflake", - { - type: "category", - label: "PostgreSQL", - collapsed: true, - collapsible: true, - link: { type: "doc", id: "migrations/postgres/index" }, - items: [ - { - type: "doc", - id: "migrations/postgres/overview", - label: "Overview", - }, - "migrations/postgres/dataset", - "migrations/postgres/rewriting-queries", - "migrations/postgres/data-modeling-techniques", - "migrations/postgres/appendix" - ], - }, - "integrations/data-ingestion/dbms/mysql/index", - "integrations/data-ingestion/redshift/index", - "integrations/data-ingestion/dbms/dynamodb/index", - { - label: "Elasticsearch", - type: "doc", - id: "use-cases/observability/clickstack/migration/elastic/index", - }, - ], - }, { type: "category", label: "Tips and Community Wisdom", @@ -284,241 +229,88 @@ const sidebars = { cloud: [ { type: "category", - label: "Get Started", - collapsed: false, - collapsible: false, - className: "top-nav-item", - link: { type: "doc", id: "cloud/get-started/index" }, - items: [ - "cloud-index", - "cloud/get-started/sql-console", - "cloud/get-started/query-insights", - "cloud/get-started/query-endpoints", - "cloud/manage/dashboards", - "cloud/manage/hyperdx", - "cloud/support", - ], - }, - { - type: "category", - label: "Best Practices", - collapsed: false, - collapsible: false, - className: "top-nav-item", - link: { type: "doc", id: "cloud/bestpractices/index" }, - items: [ - "cloud/bestpractices/usagelimits", - "cloud/bestpractices/multitenancy", - ], - }, - { - type: "category", - label: "Managing Cloud", - collapsed: false, - collapsible: false, - className: "top-nav-item", - link: { type: "doc", id: "cloud/manage/index" }, + label: "Get started", + collapsed: true, + collapsible: true, + link: { type: "doc", id: "cloud/onboard/index" }, items: [ - "cloud/manage/cloud-tiers", - "cloud/manage/integrations", { type: "category", - label: "Backups", + label: "Discover", collapsed: true, collapsible: true, - link: { type: "doc", id: "cloud/manage/backups/index" }, items: [ { type: "autogenerated", - dirName: "cloud/manage/backups", + dirName: "cloud/onboard/01_discover" } ] }, { type: "category", - label: "Monitoring", + label: "Setup", collapsed: true, collapsible: true, items: [ { type: "autogenerated", - dirName: "cloud/manage/monitoring" - } - ], - }, - { - type: "category", - label: "Billing", - link: { type: "doc", id: "cloud/manage/billing/index" }, - items: [ - "cloud/manage/billing", - "cloud/manage/billing/payment-thresholds", - "cloud/manage/troubleshooting-billing-issues", - { - type: "category", - label: "Marketplace", - link: { type: "doc", id: "cloud/manage/billing/marketplace/index" }, - items: [ - { - type: "autogenerated", - dirName: "cloud/manage/billing/marketplace", - } - ], + dirName: "cloud/onboard/02_migrate", } - ], + ] }, - "cloud/manage/settings", - "cloud/manage/replica-aware-routing", - "cloud/manage/scaling", - "cloud/manage/service-uptime", - "cloud/manage/notifications", - "cloud/manage/upgrades", - "cloud/manage/account-close", - "cloud/manage/postman", - "faq/troubleshooting", - "cloud/manage/network-data-transfer", { type: "category", - label: "Jan 2025 Changes FAQ", + label: "Tune", collapsed: true, collapsible: true, - link: { type: "doc", id: "cloud/manage/jan2025_faq/index" }, items: [ - "cloud/manage/jan2025_faq/summary", - "cloud/manage/jan2025_faq/new_tiers", - "cloud/manage/jan2025_faq/plan_migrations", - "cloud/manage/jan2025_faq/dimensions", - "cloud/manage/jan2025_faq/billing", - "cloud/manage/jan2025_faq/scaling", - "cloud/manage/jan2025_faq/backup", - ], + { + type: "autogenerated", + dirName: "cloud/onboard/03_tune", + } + ] } - ], + ] }, { type: "category", - label: "Cloud API", - collapsed: false, - collapsible: false, + label: "Features", + collapsed: true, + collapsible: true, className: "top-nav-item", - link: { type: "doc", id: "cloud/manage/api/index" }, items: [ - "cloud/manage/api/api-overview", - "cloud/manage/openapi", { - type: 'link', - label: "API Reference", - href: "https://clickhouse.com/docs/cloud/manage/api/swagger", + type: "autogenerated", + dirName: "cloud/features", } ], }, { type: "category", - label: "Cloud Reference ", - collapsed: false, - collapsible: false, + label: "Guides", + collapsed: true, + collapsible: true, className: "top-nav-item", - link: { type: "doc", id: "cloud/reference/index" }, + link: { type: "doc", id: "cloud/guides/index" }, items: [ - "cloud/reference/architecture", - "cloud/reference/shared-merge-tree", - "cloud/reference/shared-catalog", - "cloud/reference/warehouses", - "cloud/reference/byoc", { - type: "category", - link: { type: "doc", id: "cloud/reference/changelogs-index" }, - label: "Changelogs", - collapsed: true, - items: [ - "cloud/reference/changelog", - { - type: "category", - label: "Release Notes", - collapsed: true, - link: { type: "doc", id: "cloud/reference/release-notes-index" }, - items: [ - { - type: "autogenerated", - dirName: "cloud/changelogs" - } - ] - } - ], - }, - "cloud/reference/cloud-compatibility", - "cloud/reference/supported-regions" + type: "autogenerated", + dirName: "cloud/guides", + } ], }, { type: "category", - label: "Security", - collapsed: false, - collapsible: false, + label: "Reference", + collapsed: true, + collapsible: true, className: "top-nav-item", - link: { type: "doc", id: "cloud/security/index" }, + link: { type: "doc", id: "cloud/reference/index" }, items: [ - "cloud/security/shared-responsibility-model", - { - type: "category", - label: "Cloud Access Management", - link: { type: "doc", id: "cloud/security/cloud-access-management/index" }, - items: [ - "cloud/security/cloud-access-management/cloud-access-management", - "cloud/security/cloud-access-management/cloud-authentication", - "cloud/security/saml-sso-setup", - "cloud/security/common-access-management-queries", - "cloud/security/inviting-new-users", - ], - }, { - type: "category", - label: "Connectivity", - link: { type: "doc", id: "cloud/security/connectivity-overview" }, - items: [ - "cloud/security/setting-ip-filters", - { - type: "category", - label: "Private Networking", - link: { type: "doc", id: "cloud/security/private-link-overview" }, - items: [ - "cloud/security/aws-privatelink", - "cloud/security/gcp-private-service-connect", - "cloud/security/azure-privatelink", - ], - }, - "cloud/security/accessing-s3-data-securely", - "cloud/security/cloud-endpoints-api", - ], - }, - "cloud/security/cmek", - "cloud/security/audit-logging", - { - type: "category", - label: "Privacy and Compliance", - collapsed: true, - collapsible: true, - link: { type: "doc", id: "cloud/security/privacy-compliance-overview" }, - items: [ - "cloud/security/compliance-overview", - "cloud/security/personal-data-access", - ], - }, - ], - }, - { - type: "category", - label: "Migrating to Cloud", - collapsed: false, - collapsible: false, - link: { type: "doc", id: "integrations/migration/index" }, - items: [ - "integrations/migration/overview", - "integrations/migration/clickhouse-to-cloud", - "integrations/migration/clickhouse-local-etl", - "integrations/migration/etl-tool-to-clickhouse", - "integrations/migration/object-storage-to-clickhouse", - "cloud/migrate/upload-a-csv-file", + type: "autogenerated", + dirName: "cloud/reference", + } ], }, ], @@ -857,7 +649,8 @@ const sidebars = { ], }, ], - } + }, + "integrations/data-ingestion/dbms/dynamodb/index" ], }, { @@ -1808,12 +1601,6 @@ const sidebars = { description: "Start here when learning ClickHouse", href: "/starter-guides" }, - { - type: "link", - label: "Troubleshooting", - description: "Troubleshooting ClickHouse", - href: "/troubleshooting" - }, { type: "link", label: "Best Practices", @@ -1834,8 +1621,8 @@ const sidebars = { }, { type: "link", - label: "Tips and Community Wisdom", - description: "Community Lessons", + label: "Tips and community wisdom", + description: "Tips and tricks from the community", href: "/tips-and-tricks/community-wisdom" }, { @@ -1863,40 +1650,22 @@ const sidebars = { }, { type: "link", - label: "Best Practices", - description: "How to get the most out of ClickHouse Cloud", - href: "/cloud/bestpractices/" + label: "Features", + description: "Features offered by ClickHouse Cloud", + href: "/cloud/features/" }, { type: "link", - label: "Managing Cloud", - description: "Manage your ClickHouse Cloud services", - href: "/cloud/bestpractices" - }, - { - type: "link", - label: "Cloud API", - description: "Automate your ClickHouse Cloud services", - href: "/cloud/manage/cloud-api/" + label: "Guides", + description: "ClickHouse Cloud guides", + href: "/cloud/guides" }, { type: "link", - label: "Cloud Reference", - description: "Understanding how ClickHouse Cloud works", + label: "Reference", + description: "Reference docs for ClickHouse Cloud", href: "/cloud/reference/" }, - { - type: "link", - label: "Security", - description: "Secure your ClickHouse Cloud services", - href: "/cloud/security/" - }, - { - type: "link", - label: "Migrating to Cloud", - description: "Migrate your database to ClickHouse Cloud", - href: "/integrations/migration" - }, ] }, { diff --git a/static/images/cloud/onboard/discover/use_cases/0_rta.png b/static/images/cloud/onboard/discover/use_cases/0_rta.png new file mode 100644 index 00000000000..d294c84d49d Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/0_rta.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/1_rta.png b/static/images/cloud/onboard/discover/use_cases/1_rta.png new file mode 100644 index 00000000000..524a35b5ae3 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/1_rta.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/2_rta.png b/static/images/cloud/onboard/discover/use_cases/2_rta.png new file mode 100644 index 00000000000..6e5c7e2e1f9 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/2_rta.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/3_rta.png b/static/images/cloud/onboard/discover/use_cases/3_rta.png new file mode 100644 index 00000000000..6c31e82a3cc Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/3_rta.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/4_rta.png b/static/images/cloud/onboard/discover/use_cases/4_rta.png new file mode 100644 index 00000000000..78baae8cfb1 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/4_rta.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/cloud_architecture.png b/static/images/cloud/onboard/discover/use_cases/cloud_architecture.png new file mode 100644 index 00000000000..394a3cc4c4b Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/cloud_architecture.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/datalakehouse_01.png b/static/images/cloud/onboard/discover/use_cases/datalakehouse_01.png new file mode 100644 index 00000000000..120f30c54d2 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/datalakehouse_01.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/ml_ai_05.png b/static/images/cloud/onboard/discover/use_cases/ml_ai_05.png new file mode 100644 index 00000000000..035aebd1b5c Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/ml_ai_05.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/ml_ai_06.png b/static/images/cloud/onboard/discover/use_cases/ml_ai_06.png new file mode 100644 index 00000000000..48a17c9e0b9 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/ml_ai_06.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/ml_ai_07.png b/static/images/cloud/onboard/discover/use_cases/ml_ai_07.png new file mode 100644 index 00000000000..ff365134141 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/ml_ai_07.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/ml_ai_08.png b/static/images/cloud/onboard/discover/use_cases/ml_ai_08.png new file mode 100644 index 00000000000..7c6488df3bc Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/ml_ai_08.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/ml_ai_09.png b/static/images/cloud/onboard/discover/use_cases/ml_ai_09.png new file mode 100644 index 00000000000..ae54653aa67 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/ml_ai_09.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/ml_data_layer.png b/static/images/cloud/onboard/discover/use_cases/ml_data_layer.png new file mode 100644 index 00000000000..383b32f5292 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/ml_data_layer.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/online_feature_store.png b/static/images/cloud/onboard/discover/use_cases/online_feature_store.png new file mode 100644 index 00000000000..033802c0c30 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/online_feature_store.png differ diff --git a/static/images/cloud/onboard/discover/use_cases/snowflake_architecture.png b/static/images/cloud/onboard/discover/use_cases/snowflake_architecture.png new file mode 100644 index 00000000000..05ae8e7c2a7 Binary files /dev/null and b/static/images/cloud/onboard/discover/use_cases/snowflake_architecture.png differ diff --git a/translation-architecture-diagram.md b/translation-architecture-diagram.md new file mode 100644 index 00000000000..835fd34f0d8 --- /dev/null +++ b/translation-architecture-diagram.md @@ -0,0 +1,135 @@ +# ClickHouse Documentation Translation System Architecture + +```mermaid +flowchart TD + %% Input Sources + A[📁 docs/] --> B[🔄 Translation Script] + A1[📄 Config Files
languages/jp.json] --> B + A2[🔑 OpenAI API] --> B + + %% Main Translation Process + B --> C{📋 File Processing} + + %% File Type Routing + C -->|.md/.mdx files| D[📝 Markdown Translation Pipeline] + C -->|.json files| E[🔧 Plugin Data Translation] + C -->|Other files| F[📋 Direct Copy] + + %% Markdown Translation Pipeline + D --> D1[🔍 Change Detection
xxhash comparison] + D1 -->|Changed| D2[📤 Pre-processing] + D1 -->|Unchanged| D9[⏭️ Skip Translation] + + D2 --> D3[🧩 Extract Components
Code blocks, JSX, Imports] + D3 --> D4[✂️ Text Chunking
30KB max chunks] + D4 --> D5[🤖 OpenAI Translation
gpt-4o-mini] + D5 --> D6[🔧 Post-processing
Restore components] + D6 --> D7[💾 Write Output] + D7 --> D8[#️⃣ Generate Hash] + + %% Plugin Data Translation + E --> E1[🔍 Check .done files] + E1 -->|Not done| E2[🤖 Translate JSON] + E1 -->|Done| E3[⏭️ Skip] + E2 --> E4[✅ Mark .done] + + %% Output Structure + D8 --> G[📁 i18n/{lang}/docusaurus-plugin-content-docs/current/] + E4 --> H[📁 i18n/{lang}/code.json] + F --> I[📁 i18n/{lang}/other files] + + %% Deployment Pipeline + G --> J[🏗️ Docusaurus Build] + H --> J + I --> J + J --> K[🚀 Vercel Deployment
Per-language projects] + K --> L[🌐 clickhouse-docs-proxy
Language routing] + + %% File State Management + subgraph "File States" + S1[📄 source.md] + S2[📄 source.md.hash] + S3[📄 source.translate] + S4[📄 source.translated] + S5[📄 source.md final] + S6[📄 config.json.done] + + S1 -->|hash created| S2 + S3 -->|translation done| S4 + S4 -->|renamed| S5 + end + + %% Content Protection System + subgraph "Content Protection" + P1[🔒 Code Block Protection
```code``` → <CODEBLOCK_N>] + P2[🔒 Component Protection
<VersionHistory/> → <COMPONENT_N>] + P3[🔒 Import Path Transform
@site/docs → @site/i18n/{lang}/...] + P4[🔒 ALL CAPS Preservation
SELECT, CREATE, etc.] + end + + %% Translation Rules + subgraph "Translation Intelligence" + T1[📚 Glossary Terms
Technical consistency] + T2[🎯 Context-aware Prompts
Database-specific] + T3[📝 Override Files
Manual corrections] + T4[🚫 Exclusion Lists
Skip certain files] + end + + %% Parallel Processing + subgraph "Processing" + PR1[⚡ ThreadPoolExecutor
5 workers max] + PR2[🔄 Concurrent translation] + PR3[⚠️ Error handling
Graceful failures] + end + + %% Configuration Files + subgraph "Config System" + CF1[📋 docusaurus.config.en.js
Main config] + CF2[📋 docusaurus.config.jp.js
Japanese config] + CF3[📋 docusaurus.config.ru.js
Russian config] + CF4[📋 docusaurus.config.zh.js
Chinese config] + end + + %% Styling + classDef inputNode fill:#e1f5fe + classDef processNode fill:#f3e5f5 + classDef outputNode fill:#e8f5e8 + classDef configNode fill:#fff3e0 + classDef protectionNode fill:#ffebee + + class A,A1,A2 inputNode + class B,D,E processNode + class G,H,I,J,K,L outputNode + class CF1,CF2,CF3,CF4 configNode + class P1,P2,P3,P4 protectionNode +``` + +## Key Architecture Components + +### 1. **Input Layer** +- **Source Documentation** (`docs/`) - English markdown/MDX files +- **Language Configurations** - JSON files with glossaries and prompts +- **OpenAI API** - GPT models for translation + +### 2. **Processing Engine** +- **Multi-threaded Translation** - 5 concurrent workers +- **Intelligent Change Detection** - xxhash-based incremental updates +- **Content-aware Processing** - Different pipelines for different file types + +### 3. **Content Protection System** +- **Code Block Preservation** - Prevents translation of code samples +- **Component Protection** - Preserves React/JSX components +- **Technical Term Protection** - Maintains ALL CAPS SQL keywords +- **Import Path Transformation** - Updates paths for i18n structure + +### 4. **Output & Deployment** +- **Structured Output** - Mirrors source structure in i18n folders +- **Per-language Builds** - Independent Docusaurus builds +- **Proxy Routing** - Central proxy routes to language-specific deployments + +### 5. **State Management** +- **File State Tracking** - `.hash`, `.done`, `.translate` files +- **Incremental Processing** - Only processes changed files +- **Error Recovery** - Handles interrupted translations gracefully + +This architecture enables **scalable, automated, and reliable** translation of complex technical documentation while maintaining quality and consistency across multiple languages. \ No newline at end of file diff --git a/vercel.json b/vercel.json index cc2b1eb79f1..177e084699b 100644 --- a/vercel.json +++ b/vercel.json @@ -3372,8 +3372,53 @@ "permanent": true }, { - "source": "/docs/manage/troubleshooting-billing-issues", - "destination": "/docs/manage/clickhouse-cloud-billing-compliance", + "source": "/docs/cloud/migrate/upload-a-csv-file", + "destination": "/docs/knowledge-base/upload-a-file", + "permanent": true + }, + { + "source": "/docs/faq/troubleshooting", + "destination": "/docs/knowledge-base/unable-to-access-cloud-service", + "permanent": true + }, + { + "source": "/docs/cloud/manage/jan-2025-faq/summary", + "destination": "/cloud/manage/cloud-tiers", + "permanent": true + }, + { + "source" : "/docs/cloud/manage/jan-2025-faq/new-tiers", + "destination" : "/cloud/manage/cloud-tiers", + "permanent" : true + }, + { + "source": "/docs/cloud/manage/jan-2025-faq/scaling", + "destination": "/manage/scaling", + "permanent": true + }, + { + "source": "/docs/cloud/manage/jan-2025-faq/backup", + "destination": "/docs/cloud/manage/backups/overview", + "permanent": true + }, + { + "source": "/docs/cloud/manage/jan-2025-faq/pricing-dimensions", + "destination": "/docs/cloud/manage/cloud-tiers", + "permanent": true + }, + { + "source": "/docs/cloud/manage/jan-2025-faq/billing", + "destination": "/docs/cloud/manage/billing", + "permanent": true + }, + { + "source": "/docs/cloud/manage/jan-2025-faq", + "destination": "/cloud/manage/cloud-tiers", + "permanent": true + }, + { + "source": "/cloud/manage/jan-2025-faq/plan-migrations", + "destination": "/cloud/manage/cloud-tiers", "permanent": true }, { @@ -3400,6 +3445,21 @@ "source": "/docs/integrations/data-ingestion/clickpipes/postgres/parallel_initial_load", "destination": "/docs/integrations/clickpipes/postgres/parallel_initial_load", "permanent": true + }, + { + "source": "/docs/integrations/connecting-to-mysql", + "destination": "/docs/integrations/mysql", + "permanent": true + }, + { + "source": "/docs/integrations/redshift", + "destination": "/docs/migrations/redshift/migration-guide", + "permanent": true + }, + { + "source": "/docs/integrations/migration", + "destination": "/integrations/migration/overview", + "permanent": true } ] }