Skip to content

fix: blob data corruption when adding columnar attribute to RT table #4179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sanikolaev merged 2 commits into from
Jan 25, 2026
Merged

fix: blob data corruption when adding columnar attribute to RT table #4179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d6f62b0
fix: blob data corruption when adding columnar attribute to RT table
sanikolaev Jan 23, 2026
68f5f69
reverted previous fix; implemented proper fix
glookka Jan 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions src/sphinx.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2798,6 +2798,14 @@ bool CSphIndex_VLN::AddRemoveAttribute ( bool bAddAttr, const AttrAddRemoveCtx_t
if ( !IndexBuildDone ( tBuildHeader, tWriteHeader, GetTmpFilename(SPH_EXT_SPH), sError ) )
return false;

if ( !tNewSchema.GetAttrsCount() )
{
sError = "table must have at least one attribute";
return false;
}

bool bColumnar = bAddAttr ? tNewSchema.GetAttr ( tCtx.m_sName.cstr() )->IsColumnar() : m_tSchema.GetAttr ( tCtx.m_sName.cstr() )->IsColumnar();

// generate new .SPA, .SPB files
CSphWriter tSPAWriter;
CSphWriter tSPBWriter;
Expand All @@ -2811,9 +2819,16 @@ bool CSphIndex_VLN::AddRemoveAttribute ( bool bAddAttr, const AttrAddRemoveCtx_t
CSphString sSPBfile = GetTmpFilename ( SPH_EXT_SPB );
CSphString sSPHIfile = GetTmpFilename ( SPH_EXT_SPHI );

// don't open SPA file for columnar operations
bool bHaveNonColumnar = tNewSchema.HasNonColumnarAttrs();
if ( bHaveNonColumnar && !tSPAWriter.OpenFile ( sSPAfile, sError ) )
return false;
bool bNeedToCloseSPA = false;
if ( !bColumnar && bHaveNonColumnar )
{
if ( !tSPAWriter.OpenFile ( sSPAfile, sError ) )
return false;

bNeedToCloseSPA = true;
}

bool bHadBlobs = false;
for ( int i = 0; i < m_tSchema.GetAttrsCount(); i++ )
Expand All @@ -2823,8 +2838,9 @@ bool CSphIndex_VLN::AddRemoveAttribute ( bool bAddAttr, const AttrAddRemoveCtx_t
for ( int i = 0; i < tNewSchema.GetAttrsCount(); i++ )
bHaveBlobs |= sphIsBlobAttr ( tNewSchema.GetAttr(i) );

// Columnar attributes don't use the blob pool, so don't open SPB file for columnar operations
bool bBlob = sphIsBlobAttr ( tCtx.m_eType );
bool bBlobsModified = bBlob && ( bAddAttr || bHaveBlobs==bHadBlobs );
bool bBlobsModified = bBlob && !bColumnar && ( bAddAttr || bHaveBlobs==bHadBlobs );
if ( bBlobsModified )
{
if ( !tSPBWriter.OpenFile ( sSPBfile, sError ) )
Expand All @@ -2833,13 +2849,6 @@ bool CSphIndex_VLN::AddRemoveAttribute ( bool bAddAttr, const AttrAddRemoveCtx_t
tSPBWriter.PutOffset(0);
}

if ( !tNewSchema.GetAttrsCount() )
{
sError = "table must have at least one attribute";
return false;
}

bool bColumnar = bAddAttr ? tNewSchema.GetAttr ( tCtx.m_sName.cstr() )->IsColumnar() : m_tSchema.GetAttr ( tCtx.m_sName.cstr() )->IsColumnar();
if ( bColumnar )
AddRemoveColumnarAttr ( bAddAttr, tCtx.m_sName, tCtx.m_eType, m_tSchema, tNewSchema, sError );
else
Expand Down Expand Up @@ -2874,7 +2883,7 @@ bool CSphIndex_VLN::AddRemoveAttribute ( bool bAddAttr, const AttrAddRemoveCtx_t
if ( !AddRemoveFromKNN ( m_tSchema, tNewSchema, sError ) )
return false;

if ( bHaveNonColumnar )
if ( bNeedToCloseSPA )
{
if ( tSPAWriter.IsError() )
{
Expand All @@ -2887,7 +2896,6 @@ bool CSphIndex_VLN::AddRemoveAttribute ( bool bAddAttr, const AttrAddRemoveCtx_t

bool bHadColumnar = m_tSchema.HasColumnarAttrs();
bool bHaveColumnar = tNewSchema.HasColumnarAttrs();

bool bHadNonColumnar = m_tSchema.HasNonColumnarAttrs();

m_tAttr.Reset();
Expand Down Expand Up @@ -2921,7 +2929,7 @@ bool CSphIndex_VLN::AddRemoveAttribute ( bool bAddAttr, const AttrAddRemoveCtx_t
if ( bHaveNonColumnar && !m_tAttr.Setup ( GetFilename ( SPH_EXT_SPA ), sError, true ) )
return false;

if ( bBlob )
if ( bBlob && !bColumnar )
{
m_tBlobAttrs.Reset();

Expand Down
5 changes: 4 additions & 1 deletion test/test_479/model.bin
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
a:1:{i:0;a:10:{i:0;a:2:{s:8:"sphinxql";s:26:"drop table if exists t3661";s:14:"total_affected";i:0;}i:1;a:2:{s:8:"sphinxql";s:50:"create table t3661 (tag integer) engine='columnar'";s:14:"total_affected";i:0;}i:2;a:2:{s:8:"sphinxql";s:46:"insert into t3661 values(1, 0), (2, 1), (3, 2)";s:14:"total_affected";i:3;}i:3;a:2:{s:8:"sphinxql";s:20:"flush ramchunk t3661";s:14:"total_affected";i:0;}i:4;a:2:{s:8:"sphinxql";s:59:"alter table t3661 add column value string attribute indexed";s:14:"total_affected";i:0;}i:5;a:3:{s:8:"sphinxql";s:10:"desc t3661";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:3:{s:5:"Field";s:2:"id";s:4:"Type";s:6:"bigint";s:10:"Properties";s:19:"columnar fast_fetch";}i:1;a:3:{s:5:"Field";s:5:"value";s:4:"Type";s:6:"string";s:10:"Properties";s:37:"indexed attribute columnar fast_fetch";}i:2;a:3:{s:5:"Field";s:3:"tag";s:4:"Type";s:4:"uint";s:10:"Properties";s:19:"columnar fast_fetch";}}}i:6;a:2:{s:8:"sphinxql";s:37:"select * from t3661 where value='abc'";s:10:"total_rows";i:0;}i:7;a:3:{s:8:"sphinxql";s:19:"select * from t3661";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:3:{s:2:"id";s:1:"1";s:5:"value";s:0:"";s:3:"tag";s:1:"0";}i:1;a:3:{s:2:"id";s:1:"2";s:5:"value";s:0:"";s:3:"tag";s:1:"1";}i:2;a:3:{s:2:"id";s:1:"3";s:5:"value";s:0:"";s:3:"tag";s:1:"2";}}}i:8;a:3:{s:8:"sphinxql";s:26:"select count(*) from t3661";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:1:{s:8:"count(*)";s:1:"3";}}}i:9;a:2:{s:8:"sphinxql";s:16:"drop table t3661";s:14:"total_affected";i:0;}}}
a:1:{i:0;a:20:{i:0;a:2:{s:8:"sphinxql";s:26:"drop table if exists t3661";s:14:"total_affected";i:0;}i:1;a:2:{s:8:"sphinxql";s:50:"create table t3661 (tag integer) engine='columnar'";s:14:"total_affected";i:0;}i:2;a:2:{s:8:"sphinxql";s:46:"insert into t3661 values(1, 0), (2, 1), (3, 2)";s:14:"total_affected";i:3;}i:3;a:2:{s:8:"sphinxql";s:20:"flush ramchunk t3661";s:14:"total_affected";i:0;}i:4;a:2:{s:8:"sphinxql";s:59:"alter table t3661 add column value string attribute indexed";s:14:"total_affected";i:0;}i:5;a:3:{s:8:"sphinxql";s:10:"desc t3661";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:3:{s:5:"Field";s:2:"id";s:4:"Type";s:6:"bigint";s:10:"Properties";s:19:"columnar fast_fetch";}i:1;a:3:{s:5:"Field";s:5:"value";s:4:"Type";s:6:"string";s:10:"Properties";s:37:"indexed attribute columnar fast_fetch";}i:2;a:3:{s:5:"Field";s:3:"tag";s:4:"Type";s:4:"uint";s:10:"Properties";s:19:"columnar fast_fetch";}}}i:6;a:2:{s:8:"sphinxql";s:37:"select * from t3661 where value='abc'";s:10:"total_rows";i:0;}i:7;a:3:{s:8:"sphinxql";s:19:"select * from t3661";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:3:{s:2:"id";s:1:"1";s:5:"value";s:0:"";s:3:"tag";s:1:"0";}i:1;a:3:{s:2:"id";s:1:"2";s:5:"value";s:0:"";s:3:"tag";s:1:"1";}i:2;a:3:{s:2:"id";s:1:"3";s:5:"value";s:0:"";s:3:"tag";s:1:"2";}}}i:8;a:3:{s:8:"sphinxql";s:26:"select count(*) from t3661";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:1:{s:8:"count(*)";s:1:"3";}}}i:9;a:2:{s:8:"sphinxql";s:16:"drop table t3661";s:14:"total_affected";i:0;}i:10;a:2:{s:8:"sphinxql";s:31:"drop table if exists t3661_blob";s:14:"total_affected";i:0;}i:11;a:2:{s:8:"sphinxql";s:92:"create table t3661_blob (name string attribute indexed, description text) rt_mem_limit='16m'";s:14:"total_affected";i:0;}i:12;a:2:{s:8:"sphinxql";s:182:"insert into t3661_blob (id, name, description) values
(1, 'Product Name 1', 'Description 1'),
(2, 'Product Name 2', 'Description 2'),
(3, 'Product Name 3', 'Description 3')";s:14:"total_affected";i:3;}i:13;a:2:{s:8:"sphinxql";s:25:"flush ramchunk t3661_blob";s:14:"total_affected";i:0;}i:14;a:2:{s:8:"sphinxql";s:73:"alter table t3661_blob add column attribute_ids multi64 engine='columnar'";s:14:"total_affected";i:0;}i:15;a:3:{s:8:"sphinxql";s:15:"desc t3661_blob";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:3:{s:5:"Field";s:2:"id";s:4:"Type";s:6:"bigint";s:10:"Properties";s:0:"";}i:1;a:3:{s:5:"Field";s:4:"name";s:4:"Type";s:6:"string";s:10:"Properties";s:17:"indexed attribute";}i:2;a:3:{s:5:"Field";s:11:"description";s:4:"Type";s:4:"text";s:10:"Properties";s:14:"indexed stored";}i:3;a:3:{s:5:"Field";s:13:"attribute_ids";s:4:"Type";s:5:"mva64";s:10:"Properties";s:19:"columnar fast_fetch";}}}i:16;a:3:{s:8:"sphinxql";s:56:"select id, name, description from t3661_blob order by id";s:5:"errno";i:1064;s:5:"error";s:39:"table t3661_blob: no sort order defined";}i:17;a:3:{s:8:"sphinxql";s:52:"select * from t3661_blob where name='Product Name 2'";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:4:"name";s:14:"Product Name 2";s:11:"description";s:13:"Description 2";s:13:"attribute_ids";s:0:"";}}}i:18;a:3:{s:8:"sphinxql";s:31:"select count(*) from t3661_blob";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:1:{s:8:"count(*)";s:1:"3";}}}i:19;a:2:{s:8:"sphinxql";s:21:"drop table t3661_blob";s:14:"total_affected";i:0;}}}
18 changes: 18 additions & 0 deletions test/test_479/test.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,23 @@ searchd
select count(*) from t3661;
drop table t3661;
</sphinxql>

<!-- Regression: blob data corruption when adding columnar attribute to RT table with existing STRING attributes -->
<sphinxql>
drop table if exists t3661_blob;
create table t3661_blob (name string attribute indexed, description text) rt_mem_limit='16m';
insert into t3661_blob (id, name, description) values
(1, 'Product Name 1', 'Description 1'),
(2, 'Product Name 2', 'Description 2'),
(3, 'Product Name 3', 'Description 3');
flush ramchunk t3661_blob;
alter table t3661_blob add column attribute_ids multi64 engine='columnar';
desc t3661_blob;
<!-- Verify blob data is preserved after ALTER -->
select id, name, description from t3661_blob order by id;
select * from t3661_blob where name='Product Name 2';
select count(*) from t3661_blob;
drop table t3661_blob;
</sphinxql>
</queries>
</test>