diff --git a/dali/dfuXRefLib/XRefFilesNode.cpp b/dali/dfuXRefLib/XRefFilesNode.cpp index 24124729d2c..97db311453f 100644 --- a/dali/dfuXRefLib/XRefFilesNode.cpp +++ b/dali/dfuXRefLib/XRefFilesNode.cpp @@ -37,6 +37,12 @@ CXRefFilesNode::CXRefFilesNode(IPropertyTree& baseNode,const char* cluster,const prefixName.append(cluster); } +void CXRefFilesNode::setXRefPath(const char *_xrefPath, const char *_branchName) +{ + xrefPath.set(_xrefPath); + branchName.set(_branchName); +} + bool CXRefFilesNode::IsChanged() { return m_bChanged; @@ -54,7 +60,44 @@ MemoryBuffer &CXRefFilesNode::queryData() if (m_bChanged || (0 == _data.length())) { _data.clear(); - m_baseTree.getPropBin("data", _data); + + // If using file-based storage, load from file + if (xrefPath.length() > 0 && branchName.length() > 0) + { + try + { + StringBuffer filepath(xrefPath); + addPathSepChar(filepath).append(branchName).append(".xml"); + + Owned file = createIFile(filepath.str()); + if (file->exists()) + { + Owned fileIO = file->open(IFOread); + if (fileIO) + { + offset_t fileSize = file->size(); + if (fileSize > 0 && fileSize < 0x10000000) // Sanity check: < 256MB + { + _data.ensureCapacity((size32_t)fileSize); + size32_t bytesRead = fileIO->read(0, (size32_t)fileSize, _data.reserve((size32_t)fileSize)); + _data.setLength(bytesRead); + } + } + } + } + catch (IException *e) + { + StringBuffer errMsg; + OWARNLOG("XRefFilesNode: Failed to load from file '%s/%s.xml': %s", + xrefPath.str(), branchName.str(), e->errorMessage(errMsg).str()); + e->Release(); + } + } + else + { + // Old method: load from "data" attribute in Dali + m_baseTree.getPropBin("data", _data); + } } return _data; } @@ -71,7 +114,40 @@ void CXRefFilesNode::Deserialize(IPropertyTree& inTree) CleanTree(inTree); StringBuffer datastr; toXML(&inTree,datastr); - m_baseTree.setPropBin("data",datastr.length(),(void*)datastr.str()); + + // If using file-based storage, save to file + if (xrefPath.length() > 0 && branchName.length() > 0) + { + try + { + StringBuffer filepath(xrefPath); + addPathSepChar(filepath).append(branchName).append(".xml"); + + Owned file = createIFile(filepath.str()); + Owned fileIO = file->open(IFOcreate); + if (fileIO) + { + fileIO->write(0, datastr.length(), datastr.str()); + fileIO->close(); + } + else + { + OWARNLOG("XRefFilesNode: Failed to save to file '%s'", filepath.str()); + } + } + catch (IException *e) + { + StringBuffer errMsg; + OWARNLOG("XRefFilesNode: Failed to save to file '%s/%s.xml': %s", + xrefPath.str(), branchName.str(), e->errorMessage(errMsg).str()); + e->Release(); + } + } + else + { + // Old method: save to "data" attribute in Dali + m_baseTree.setPropBin("data",datastr.length(),(void*)datastr.str()); + } } IPropertyTree* CXRefFilesNode::FindNode(const char* NodeName) diff --git a/dali/dfuXRefLib/XRefFilesNode.hpp b/dali/dfuXRefLib/XRefFilesNode.hpp index e164ee5673d..8ca98087317 100644 --- a/dali/dfuXRefLib/XRefFilesNode.hpp +++ b/dali/dfuXRefLib/XRefFilesNode.hpp @@ -52,6 +52,8 @@ class CXRefFilesNode : implements IXRefFilesNode, public CSimpleInterface MemoryBuffer _data; StringBuffer prefixName; StringAttr rootdir; + StringAttr xrefPath; // Path for file-based storage + StringAttr branchName; // Name of this branch (e.g., "Lost", "Found", "Orphans") private: IPropertyTree* FindNode(const char* NodeName); IPropertyTree& queryDataTree(); @@ -64,6 +66,7 @@ class CXRefFilesNode : implements IXRefFilesNode, public CSimpleInterface public: IMPLEMENT_IINTERFACE_USING(CSimpleInterface); CXRefFilesNode(IPropertyTree& baseNode,const char* cluster, const char *rootdir); + void setXRefPath(const char *_xrefPath, const char *_branchName); virtual ~CXRefFilesNode(){}; virtual bool IsChanged() override; void Commit() override; diff --git a/dali/dfuXRefLib/XRefNodeManager.cpp b/dali/dfuXRefLib/XRefNodeManager.cpp index 2f662505b86..ec557b94a14 100644 --- a/dali/dfuXRefLib/XRefNodeManager.cpp +++ b/dali/dfuXRefLib/XRefNodeManager.cpp @@ -23,6 +23,7 @@ #include "jstring.hpp" #include "jptree.hpp" #include "jmisc.hpp" +#include "jfile.hpp" #include "mpcomm.hpp" #include "platform.h" @@ -187,14 +188,33 @@ IXRefFilesNode* CXRefNode::getLostFiles() { if(!m_lost.get()) { - IPropertyTree* lostBranch = m_XRefTree->queryPropTree("Lost"); - if(lostBranch == 0) + const char *xrefPath = m_XRefTree->queryProp("@xrefPath"); + IPropertyTree* lostBranch = nullptr; + + if (xrefPath && *xrefPath) { - lostBranch = m_XRefTree->addPropTree("Lost",createPTree()); - commit(); + // File-based storage: create empty branch, don't add to m_XRefTree + lostBranch = m_XRefTree->queryPropTree("Lost"); + if (!lostBranch) + lostBranch = m_XRefTree->addPropTree("Lost",createPTree()); } + else + { + // Dali-based storage: use branch from m_XRefTree + lostBranch = m_XRefTree->queryPropTree("Lost"); + if(lostBranch == 0) + { + lostBranch = m_XRefTree->addPropTree("Lost",createPTree()); + commit(); + } + } + StringBuffer tmpbuf; m_lost.setown(new CXRefFilesNode(*lostBranch,getName(tmpbuf).str(),rootDir)); + + // Set xrefPath if using file-based storage + if (xrefPath && *xrefPath) + m_lost->setXRefPath(xrefPath, "Lost"); } return m_lost.getLink(); } @@ -203,14 +223,33 @@ IXRefFilesNode* CXRefNode::getFoundFiles() { if(!m_found.get()) { - IPropertyTree* foundBranch = m_XRefTree->queryPropTree("Found"); - if(foundBranch == 0) + const char *xrefPath = m_XRefTree->queryProp("@xrefPath"); + IPropertyTree* foundBranch = nullptr; + + if (xrefPath && *xrefPath) { - foundBranch = m_XRefTree->addPropTree("Found",createPTree()); - commit(); + // File-based storage: create empty branch, don't add to m_XRefTree + foundBranch = m_XRefTree->queryPropTree("Found"); + if (!foundBranch) + foundBranch = m_XRefTree->addPropTree("Found",createPTree()); } + else + { + // Dali-based storage: use branch from m_XRefTree + foundBranch = m_XRefTree->queryPropTree("Found"); + if(foundBranch == 0) + { + foundBranch = m_XRefTree->addPropTree("Found",createPTree()); + commit(); + } + } + StringBuffer tmpbuf; m_found.setown(new CXRefFilesNode(*foundBranch,getName(tmpbuf).str(),rootDir)); + + // Set xrefPath if using file-based storage + if (xrefPath && *xrefPath) + m_found->setXRefPath(xrefPath, "Found"); } return m_found.getLink(); } @@ -219,98 +258,259 @@ IXRefFilesNode* CXRefNode::getOrphanFiles() { if(!m_orphans.get()) { - IPropertyTree* orphanBranch = m_XRefTree->queryPropTree("Orphans"); - if(orphanBranch == 0) + const char *xrefPath = m_XRefTree->queryProp("@xrefPath"); + IPropertyTree* orphanBranch = nullptr; + + if (xrefPath && *xrefPath) + { + // File-based storage: create empty branch, don't add to m_XRefTree + orphanBranch = m_XRefTree->queryPropTree("Orphans"); + if (!orphanBranch) + orphanBranch = m_XRefTree->addPropTree("Orphans",createPTree()); + } + else { - orphanBranch = m_XRefTree->addPropTree("Orphans",createPTree()); - commit(); + // Dali-based storage: use branch from m_XRefTree + orphanBranch = m_XRefTree->queryPropTree("Orphans"); + if(orphanBranch == 0) + { + orphanBranch = m_XRefTree->addPropTree("Orphans",createPTree()); + commit(); + } } + StringBuffer tmpbuf; m_orphans.setown(new CXRefOrphanFilesNode(*orphanBranch,getName(tmpbuf).str(),rootDir)); + + // Set xrefPath if using file-based storage + if (xrefPath && *xrefPath) + m_orphans->setXRefPath(xrefPath, "Orphans"); } return m_orphans.getLink(); } StringBuffer &CXRefNode::serializeMessages(StringBuffer &buf) { - if(!m_messages.get()) + buf.clear(); + const char *xrefPath = m_XRefTree->queryProp("@xrefPath"); + + if (xrefPath && *xrefPath) { - IPropertyTree* messagesBranch = m_XRefTree->queryPropTree("Messages"); - if(messagesBranch == 0) + // File-based storage: load from file + try + { + StringBuffer filepath(xrefPath); + addPathSepChar(filepath).append("Messages").append(".xml"); + + Owned file = createIFile(filepath.str()); + if (file->exists()) + { + Owned fileIO = file->open(IFOread); + if (fileIO) + { + offset_t fileSize = file->size(); + if (fileSize > 0 && fileSize < 0x10000000) + { + MemoryBuffer xmlContent; + xmlContent.ensureCapacity((size32_t)fileSize); + size32_t bytesRead = fileIO->read(0, (size32_t)fileSize, xmlContent.reserve((size32_t)fileSize)); + if (bytesRead > 0) + { + buf.append(bytesRead, xmlContent.toByteArray()); + } + } + } + } + } + catch (IException *e) { - messagesBranch = m_XRefTree->addPropTree("Messages",createPTree()); - commit(); + StringBuffer errMsg; + OWARNLOG("XRefNode: Failed to load Messages from file: %s", e->errorMessage(errMsg).str()); + e->Release(); } - StringBuffer tmpbuf; - m_messages.set(messagesBranch); } - buf.clear(); - MemoryBuffer data; - m_messages->getPropBin("data",data); - if (data.length()) + else { - buf.append(data.length(),data.toByteArray()); + // Dali-based storage: use "data" attribute + if(!m_messages.get()) + { + IPropertyTree* messagesBranch = m_XRefTree->queryPropTree("Messages"); + if(messagesBranch == 0) + { + messagesBranch = m_XRefTree->addPropTree("Messages",createPTree()); + commit(); + } + m_messages.set(messagesBranch); + } + MemoryBuffer data; + m_messages->getPropBin("data",data); + if (data.length()) + { + buf.append(data.length(),data.toByteArray()); + } } return buf; } void CXRefNode::deserializeMessages(IPropertyTree& inTree) { - if(!m_messages.get()) + const char *xrefPath = m_XRefTree->queryProp("@xrefPath"); + + if (xrefPath && *xrefPath) { - IPropertyTree* messagesBranch = m_XRefTree->queryPropTree("Messages"); - if(messagesBranch == 0) + // File-based storage: save to file + try { - messagesBranch = m_XRefTree->addPropTree("Messages",createPTree()); - commit(); + StringBuffer filepath(xrefPath); + addPathSepChar(filepath).append("Messages").append(".xml"); + + StringBuffer datastr; + toXML(&inTree,datastr); + + Owned file = createIFile(filepath.str()); + Owned fileIO = file->open(IFOcreate); + if (fileIO) + { + fileIO->write(0, datastr.length(), datastr.str()); + fileIO->close(); + } + } + catch (IException *e) + { + StringBuffer errMsg; + OWARNLOG("XRefNode: Failed to save Messages to file: %s", e->errorMessage(errMsg).str()); + e->Release(); } - StringBuffer tmpbuf; - m_messages.set(messagesBranch); } - StringBuffer datastr; - toXML(&inTree,datastr); - m_messages->setPropBin("data",datastr.length(),(void*)datastr.str()); + else + { + // Dali-based storage: save to "data" attribute + if(!m_messages.get()) + { + IPropertyTree* messagesBranch = m_XRefTree->queryPropTree("Messages"); + if(messagesBranch == 0) + { + messagesBranch = m_XRefTree->addPropTree("Messages",createPTree()); + commit(); + } + m_messages.set(messagesBranch); + } + StringBuffer datastr; + toXML(&inTree,datastr); + m_messages->setPropBin("data",datastr.length(),(void*)datastr.str()); + } } StringBuffer &CXRefNode::serializeDirectories(StringBuffer &buf) { - if(!m_directories.get()) + buf.clear(); + const char *xrefPath = m_XRefTree->queryProp("@xrefPath"); + + if (xrefPath && *xrefPath) { - IPropertyTree* directoriesBranch = m_XRefTree->queryPropTree("Directories"); - if(directoriesBranch == 0) + // File-based storage: load from file + try { - directoriesBranch = m_XRefTree->addPropTree("Directories",createPTree()); - commit(); + StringBuffer filepath(xrefPath); + addPathSepChar(filepath).append("Directories").append(".xml"); + + Owned file = createIFile(filepath.str()); + if (file->exists()) + { + Owned fileIO = file->open(IFOread); + if (fileIO) + { + offset_t fileSize = file->size(); + if (fileSize > 0 && fileSize < 0x10000000) + { + MemoryBuffer xmlContent; + xmlContent.ensureCapacity((size32_t)fileSize); + size32_t bytesRead = fileIO->read(0, (size32_t)fileSize, xmlContent.reserve((size32_t)fileSize)); + if (bytesRead > 0) + { + buf.append(bytesRead, xmlContent.toByteArray()); + } + } + } + } + } + catch (IException *e) + { + StringBuffer errMsg; + OWARNLOG("XRefNode: Failed to load Directories from file: %s", e->errorMessage(errMsg).str()); + e->Release(); } - StringBuffer tmpbuf; - m_directories.set(directoriesBranch); } - buf.clear(); - MemoryBuffer data; - m_directories->getPropBin("data",data); - if (data.length()) + else { - buf.append(data.length(),data.toByteArray()); + // Dali-based storage: use "data" attribute + if(!m_directories.get()) + { + IPropertyTree* directoriesBranch = m_XRefTree->queryPropTree("Directories"); + if(directoriesBranch == 0) + { + directoriesBranch = m_XRefTree->addPropTree("Directories",createPTree()); + commit(); + } + m_directories.set(directoriesBranch); + } + MemoryBuffer data; + m_directories->getPropBin("data",data); + if (data.length()) + { + buf.append(data.length(),data.toByteArray()); + } } return buf; } void CXRefNode::deserializeDirectories(IPropertyTree& inTree) { - if(!m_directories.get()) + const char *xrefPath = m_XRefTree->queryProp("@xrefPath"); + + if (xrefPath && *xrefPath) { - IPropertyTree* directoriesBranch = m_XRefTree->queryPropTree("Directories"); - if(directoriesBranch == 0) + // File-based storage: save to file + try { - directoriesBranch = m_XRefTree->addPropTree("Directories",createPTree()); - commit(); + StringBuffer filepath(xrefPath); + addPathSepChar(filepath).append("Directories").append(".xml"); + + StringBuffer datastr; + toXML(&inTree,datastr); + + Owned file = createIFile(filepath.str()); + Owned fileIO = file->open(IFOcreate); + if (fileIO) + { + fileIO->write(0, datastr.length(), datastr.str()); + fileIO->close(); + } } - StringBuffer tmpbuf; - m_directories.set(directoriesBranch); + catch (IException *e) + { + StringBuffer errMsg; + OWARNLOG("XRefNode: Failed to save Directories to file: %s", e->errorMessage(errMsg).str()); + e->Release(); + } + } + else + { + // Dali-based storage: save to "data" attribute + if(!m_directories.get()) + { + IPropertyTree* directoriesBranch = m_XRefTree->queryPropTree("Directories"); + if(directoriesBranch == 0) + { + directoriesBranch = m_XRefTree->addPropTree("Directories",createPTree()); + commit(); + } + m_directories.set(directoriesBranch); + } + StringBuffer datastr; + toXML(&inTree,datastr); + m_directories->setPropBin("data",datastr.length(),(void*)datastr.str()); } - StringBuffer datastr; - toXML(&inTree,datastr); - m_directories->setPropBin("data",datastr.length(),(void*)datastr.str()); } diff --git a/dali/sasha/saxref.cpp b/dali/sasha/saxref.cpp index 1640e2e6896..234c52158ed 100644 --- a/dali/sasha/saxref.cpp +++ b/dali/sasha/saxref.cpp @@ -9,6 +9,10 @@ #include "jmisc.hpp" #include "jregexp.hpp" #include "jset.hpp" +#include "jfile.hpp" +#include "jplane.hpp" +#include "jutil.hpp" +#include "jsocket.hpp" #include #include @@ -1001,6 +1005,38 @@ class CNewXRefManagerBase heartbeatTimer.updatePeriod(); } + void saveBranchToSashaPlane(const char *sashaDir, const char *name, IPropertyTree *branch) + { + if (!branch) + return; + try + { + branch->setProp("Cluster",clustname); + StringBuffer filepath(sashaDir); + addPathSepChar(filepath).append(name).append(".xml"); + + StringBuffer datastr; + toXML(branch,datastr); + + Owned file = createIFile(filepath.str()); + Owned fileIO = file->open(IFOcreate); + if (!fileIO) + { + warn(filepath.str(), "Failed to create file"); + return; + } + fileIO->write(0, datastr.length(), datastr.str()); + fileIO->close(); + PROGLOG(LOGPFX "Saved branch %s to %s", name, filepath.str()); + } + catch (IException *e) + { + StringBuffer errMsg; + warn(name, "Error saving branch to Sasha plane: %s", e->errorMessage(errMsg).str()); + e->Release(); + } + } + void addBranch(IPropertyTree *root,const char *name,IPropertyTree *branch) { if (!branch) @@ -1086,11 +1122,94 @@ class CNewXRefManagerBase else if (fnum) ss.appendf(" [%d files]",fnum); croot->setProp("@status",ss.str()); - addBranch(croot,"Orphans",orphansbranch); - addBranch(croot,"Lost",lostbranch); - addBranch(croot,"Found",foundbranch); - addBranch(croot,"Directories",dirbranch); - addErrorsWarnings(croot); + + // Check if we should use Sasha plane for storage (configurable, default to containerized) + bool useSashaPlane = false; + StringBuffer sashaDir; + + // Read configuration - make this an expert option + Owned xrefConfig; + if (isContainerized()) + xrefConfig.setown(getComponentConfig()); + else + xrefConfig.setown(serverConfig->getPropTree("DfuXRef")); + + if (xrefConfig) + useSashaPlane = xrefConfig->getPropBool("@useSashaPlane", isContainerized()); + else + useSashaPlane = isContainerized(); + + if (useSashaPlane) + { + // Get Sasha plane directory - throw error if it fails + if (isContainerized()) + { + StringBuffer planeName; + if (!getDefaultPlane(planeName, nullptr, "sasha")) + throw makeStringException(0, LOGPFX "Sasha plane must be configured"); + + Owned sashaPlane = getStoragePlaneConfig(planeName, true); + if (!sashaPlane->getProp("@prefix", sashaDir)) + throw makeStringException(0, LOGPFX "Sasha plane has no prefix configured"); + } + else + { + throw makeStringException(0, LOGPFX "Sasha plane storage only supported in containerized deployments"); + } + + // Create directory structure: /xref/// + addPathSepChar(sashaDir).append("xref"); + addPathSepChar(sashaDir).append(clustname); + + // Create datestamp directory + StringBuffer datestamp; + dt.getDateString(datestamp, false); // YYYY-MM-DD format + addPathSepChar(sashaDir).append(datestamp); + + // Create the directory structure recursively + StringBuffer testFile(sashaDir); + addPathSepChar(testFile).append("test.xml"); + if (!recursiveCreateDirectoryForFile(testFile.str())) + throw makeStringException(0, LOGPFX "Failed to create directory: %s", sashaDir.str()); + + PROGLOG(LOGPFX "Using Sasha plane storage at: %s", sashaDir.str()); + + // Save branches to Sasha plane files + saveBranchToSashaPlane(sashaDir.str(), "Orphans", orphansbranch); + saveBranchToSashaPlane(sashaDir.str(), "Lost", lostbranch); + saveBranchToSashaPlane(sashaDir.str(), "Found", foundbranch); + saveBranchToSashaPlane(sashaDir.str(), "Directories", dirbranch); + + // Save Messages + Owned message = createPTree("Messages"); + ForEachItemIn(i1,errors) { + cMessage &item = errors.item(i1); + IPropertyTree *t = message->addPropTree("Error",createPTree("Error")); + t->addProp("File",item.lname.get()); + t->addProp("Text",item.msg.get()); + } + ForEachItemIn(i2,warnings) { + cMessage &item = warnings.item(i2); + IPropertyTree *t = message->addPropTree("Warning",createPTree("Warning")); + t->addProp("File",item.lname.get()); + t->addProp("Text",item.msg.get()); + } + saveBranchToSashaPlane(sashaDir.str(), "Messages", message); + + // Store path reference in Dali + croot->setProp("@xrefPath", sashaDir.str()); + PROGLOG(LOGPFX "Saved XREF data to Sasha plane with path: %s", sashaDir.str()); + } + else + { + // Use traditional Dali storage + addBranch(croot,"Orphans",orphansbranch); + addBranch(croot,"Lost",lostbranch); + addBranch(croot,"Found",foundbranch); + addBranch(croot,"Directories",dirbranch); + addErrorsWarnings(croot); + } + if (abort) return; logconn.clear();