From aad75a8504d27173859517e4d06ca8e7c28b5c4d Mon Sep 17 00:00:00 2001 From: alja Date: Thu, 8 Feb 2024 11:46:26 -0800 Subject: [PATCH 01/43] Move DirState and FPurgeState to a separate source file. --- src/XrdPfc.cmake | 2 + src/XrdPfc/XrdPfcDirState.hh | 161 +++++++++++ src/XrdPfc/XrdPfcFPurgeState.cc | 306 +++++++++++++++++++++ src/XrdPfc/XrdPfcFPurgeState.hh | 98 +++++++ src/XrdPfc/XrdPfcPurge.cc | 467 +------------------------------- 5 files changed, 573 insertions(+), 461 deletions(-) create mode 100644 src/XrdPfc/XrdPfcDirState.hh create mode 100644 src/XrdPfc/XrdPfcFPurgeState.cc create mode 100644 src/XrdPfc/XrdPfcFPurgeState.hh diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index 34d73b692c2..42c12dcdf60 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -19,6 +19,8 @@ add_library( XrdPfc/XrdPfcTypes.hh XrdPfc/XrdPfc.cc XrdPfc/XrdPfc.hh XrdPfc/XrdPfcConfiguration.cc + XrdPfc/XrdPfcDirState.hh + XrdPfc/XrdPfcFPurgeState.cc XrdPfc/XrdPfcFPurgeState.hh XrdPfc/XrdPfcPurge.cc XrdPfc/XrdPfcCommand.cc XrdPfc/XrdPfcFile.cc XrdPfc/XrdPfcFile.hh diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh new file mode 100644 index 00000000000..49d137bbb01 --- /dev/null +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -0,0 +1,161 @@ +#ifndef __XRDPFC_DIRSTATE_HH__ +#define __XRDPFC_DIRSTATE_HH__ + +// #include "XrdPfcTrace.hh" +#include "XrdPfcInfo.hh" + +using namespace XrdPfc; + +namespace XrdPfc +{ + +//============================================================================== +// DirState +//============================================================================== + +class DirState +{ + DirState *m_parent; + + Stats m_stats; // access stats from client reads in this directory (and subdirs) + + long long m_usage; // collected / measured during purge traversal + long long m_usage_extra; // collected from write events in this directory and subdirs + long long m_usage_purged; // amount of data purged from this directory (and subdirectories for leaf nodes) + + // begin purge traversal usage \_ so we can have a good estimate of what came in during the traversal + // end purge traversal usage / (should be small, presumably) + + // quota info, enabled? + + int m_depth; + int m_max_depth; // XXXX Do we need this? Should it be passed in to find functions? + bool m_stat_report; // not used yet - storing of stats requested + + typedef std::map DsMap_t; + typedef DsMap_t::iterator DsMap_i; + + DsMap_t m_subdirs; + + void init() + { + m_usage = 0; + m_usage_extra = 0; + m_usage_purged = 0; + } + + DirState* create_child(const std::string &dir) + { + std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this))); + return & ir.first->second; + } + + DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs) + { + if (pos == pt.get_n_dirs()) return this; + + DsMap_i i = m_subdirs.find(pt.m_dirs[pos]); + + DirState *ds = 0; + + if (i != m_subdirs.end()) + { + ds = & i->second; + } + if (create_subdirs && m_depth < m_max_depth) + { + ds = create_child(pt.m_dirs[pos]); + } + if (ds) return ds->find_path_tok(pt, pos + 1, create_subdirs); + + return 0; + } + +public: + + DirState(int max_depth) : m_parent(0), m_depth(0), m_max_depth(max_depth) + { + init(); + } + + DirState(DirState *parent) : m_parent(parent), m_depth(m_parent->m_depth + 1), m_max_depth(m_parent->m_max_depth) + { + init(); + } + + DirState* get_parent() { return m_parent; } + + long long get_usage() { return m_usage; } + + void set_usage(long long u) { m_usage = u; m_usage_extra = 0; } + void add_up_stats(const Stats& stats) { m_stats.AddUp(stats); } + void add_usage_purged(long long up) { m_usage_purged += up; } + + DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs) + { + PathTokenizer pt(path, max_depth, parse_as_lfn); + + return find_path_tok(pt, 0, create_subdirs); + } + + DirState* find_dir(const std::string &dir, bool create_subdirs) + { + DsMap_i i = m_subdirs.find(dir); + + if (i != m_subdirs.end()) return & i->second; + + if (create_subdirs && m_depth < m_max_depth) return create_child(dir); + + return 0; + } + + void reset_stats() + { + m_stats.Reset(); + + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.reset_stats(); + } + } + + void upward_propagate_stats() + { + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.upward_propagate_stats(); + + m_stats.AddUp(i->second.m_stats); + } + + m_usage_extra += m_stats.m_BytesWritten; + } + + long long upward_propagate_usage_purged() + { + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + m_usage_purged += i->second.upward_propagate_usage_purged(); + } + m_usage -= m_usage_purged; + + long long ret = m_usage_purged; + m_usage_purged = 0; + return ret; + } + + void dump_recursively(const char *name) + { + printf("%*d %s usage=%lld usage_extra=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", + 2 + 2*m_depth, m_depth, name, m_usage, m_usage_extra, m_usage + m_usage_extra, + m_stats.m_NumIos, m_stats.m_Duration, m_stats.m_BytesHit, m_stats.m_BytesMissed, m_stats.m_BytesBypassed, m_stats.m_BytesWritten); + + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.dump_recursively(i->first.c_str()); + } + } +}; +} + +#endif diff --git a/src/XrdPfc/XrdPfcFPurgeState.cc b/src/XrdPfc/XrdPfcFPurgeState.cc new file mode 100644 index 00000000000..91d28ab67fe --- /dev/null +++ b/src/XrdPfc/XrdPfcFPurgeState.cc @@ -0,0 +1,306 @@ +#include "XrdPfcFPurgeState.hh" + +#include "XrdPfcDirState.hh" +#include "XrdPfcFPurgeState.hh" +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucUtils.hh" +#include "XrdOss/XrdOssAt.hh" +#include "XrdSys/XrdSysTrace.hh" + +// Temporary, extensive purge tracing +// #define TRACE_PURGE(x) TRACE(Debug, x) +// #define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" +#define TRACE_PURGE(x) + +using namespace XrdPfc; + +namespace XrdPfc +{ + +XrdSysTrace* GetTrace() +{ + // needed for logging macros + return Cache::GetInstance().GetTrace(); +} + + +//---------------------------------------------------------------------------- +//! Constructor. +//---------------------------------------------------------------------------- +FPurgeState::FPurgeState(long long iNBytesReq, XrdOss &oss) : + m_nBytesReq(iNBytesReq), m_nBytesAccum(0), m_nBytesTotal(0), m_tMinTimeStamp(0), m_tMinUVKeepTimeStamp(0), + m_oss_at(oss), + m_dir_state(0), m_dir_level(0), + m_max_dir_level_for_stat_collection(Cache::Conf().m_dirStatsStoreDepth), + m_info_ext(XrdPfc::Info::s_infoExtension), + m_info_ext_len(strlen(XrdPfc::Info::s_infoExtension)), + m_trace(Cache::GetInstance().GetTrace()) +{ + m_current_path.reserve(256); + m_dir_names_stack.reserve(32); + m_dir_usage_stack.reserve(m_max_dir_level_for_stat_collection + 1); +} + +//---------------------------------------------------------------------------- +//! Initiate DirState for traversal. +//! @param directory statistics +//! @param path relative to caching proxy OSS +//---------------------------------------------------------------------------- +void FPurgeState::begin_traversal(DirState *root, const char *root_path) +{ + m_dir_state = root; + m_dir_level = 0; + m_current_path = std::string(root_path); + m_dir_usage_stack.push_back(0); + + TRACE_PURGE("FPurgeState::begin_traversal cur_path '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", level=" << m_dir_level); +} + +//---------------------------------------------------------------------------- +//! Finalize DirState at the end of traversal. +//---------------------------------------------------------------------------- +void FPurgeState::end_traversal() +{ + TRACE_PURGE("FPurgeState::end_traversal reporting for '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", nBytesTotal=" << m_nBytesTotal << ", level=" << m_dir_level); + + m_dir_state->set_usage(m_dir_usage_stack.back()); + + m_dir_state = 0; +} + +//---------------------------------------------------------------------------- +//! Move to child directory. +//! @param relative name of subdirectory +//---------------------------------------------------------------------------- +void FPurgeState::cd_down(const std::string &dir_name) +{ + ++m_dir_level; + + if (m_dir_level <= m_max_dir_level_for_stat_collection) + { + m_dir_usage_stack.push_back(0); + m_dir_state = m_dir_state->find_dir(dir_name, true); + } + + m_dir_names_stack.push_back(dir_name); + m_current_path.append(dir_name); + m_current_path.append("/"); +} + +//---------------------------------------------------------------------------- +//! Move to parent directory and set disk usage. +//---------------------------------------------------------------------------- +void FPurgeState::cd_up() +{ + if (m_dir_level <= m_max_dir_level_for_stat_collection) + { + long long tail = m_dir_usage_stack.back(); + m_dir_usage_stack.pop_back(); + + TRACE_PURGE("FPurgeState::cd_up reporting for '" << m_current_path << "', usage=" << tail << ", level=" << m_dir_level); + + m_dir_state->set_usage(tail); + m_dir_state = m_dir_state->get_parent(); + + m_dir_usage_stack.back() += tail; + } + + // remove trailing / and last dir but keep the new trailing / in place. + m_current_path.erase(m_current_path.find_last_of('/', m_current_path.size() - 2) + 1); + m_dir_names_stack.pop_back(); + + --m_dir_level; +} + +//---------------------------------------------------------------------------- +//! Move remaing entires to the member map. +//! This is used for cold files and for files collected from purge plugin. +//---------------------------------------------------------------------------- +void FPurgeState::MoveListEntriesToMap() +{ + for (list_i i = m_flist.begin(); i != m_flist.end(); ++i) + { + m_fmap.insert(std::make_pair(i->time, *i)); + } + m_flist.clear(); +} + +//---------------------------------------------------------------------------- +//! Open info file. Look at the UV stams and last access time. +//! Store the file in sorted map or in a list.s +//! @param name of the cached file +//! @param Info object +//! @param stat of the given file +//! +//---------------------------------------------------------------------------- +void FPurgeState::CheckFile(const char *fname, Info &info, struct stat &fstat /*, XrdOssDF *iOssDF*/) +{ + static const char *trc_pfx = "FPurgeState::CheckFile "; + + long long nbytes = info.GetNDownloadedBytes(); + time_t atime; + if (!info.GetLatestDetachTime(atime)) + { + // cinfo file does not contain any known accesses, use fstat.mtime instead. + TRACE(Debug, trc_pfx << "could not get access time for " << m_current_path << fname << ", using mtime from stat instead."); + atime = fstat.st_mtime; + } + // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime); + + m_nBytesTotal += nbytes; + + m_dir_usage_stack.back() += nbytes; + + // XXXX Should remove aged-out files here ... but I have trouble getting + // the DirState and purge report set up consistently. + // Need some serious code reorganization here. + // Biggest problem is maintaining overall state a traversal state consistently. + // Sigh. + + // In first two cases we lie about FS time (set to 0) to get them all removed early. + // The age-based purge atime would also be good as there should be nothing + // before that time in the map anyway. + // But we use 0 as a test in purge loop to make sure we continue even if enough + // disk-space has been freed. + + if (m_tMinTimeStamp > 0 && atime < m_tMinTimeStamp) + { + m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); + m_nBytesAccum += nbytes; + } + else if (m_tMinUVKeepTimeStamp > 0 && + Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) && + info.GetNoCkSumTimeForUVKeep() < m_tMinUVKeepTimeStamp) + { + m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); + m_nBytesAccum += nbytes; + } + else if (m_nBytesAccum < m_nBytesReq || (!m_fmap.empty() && atime < m_fmap.rbegin()->first)) + { + m_fmap.insert(std::make_pair(atime, FS(m_current_path, fname, nbytes, atime, m_dir_state))); + m_nBytesAccum += nbytes; + + // remove newest files from map if necessary + while (!m_fmap.empty() && m_nBytesAccum - m_fmap.rbegin()->second.nBytes >= m_nBytesReq) + { + m_nBytesAccum -= m_fmap.rbegin()->second.nBytes; + m_fmap.erase(--(m_fmap.rbegin().base())); + } + } +} + +//---------------------------------------------------------------------------- +//! Recursively traverse directory. Build DirState statistics and sort files. +//! @param XrdOssDF handle +//---------------------------------------------------------------------------- +void FPurgeState::TraverseNamespace(XrdOssDF *iOssDF) +{ + static const char *trc_pfx = "FPurgeState::TraverseNamespace "; + + char fname[256]; + struct stat fstat; + XrdOucEnv env; + + TRACE_PURGE("Starting to read dir [" << m_current_path << "], iOssDF->getFD()=" << iOssDF->getFD() << "."); + + iOssDF->StatRet(&fstat); + + while (true) + { + int rc = iOssDF->Readdir(fname, 256); + + if (rc == -ENOENT) + { + TRACE_PURGE(" Skipping ENOENT dir entry [" << fname << "]."); + continue; + } + if (rc != XrdOssOK) + { + TRACE(Error, trc_pfx << "Readdir error at " << m_current_path << ", err " << XrdSysE2T(-rc) << "."); + break; + } + + TRACE_PURGE(" Readdir [" << fname << "]"); + + if (fname[0] == 0) + { + TRACE_PURGE(" Finished reading dir [" << m_current_path << "]. Break loop."); + break; + } + if (fname[0] == '.' && (fname[1] == 0 || (fname[1] == '.' && fname[2] == 0))) + { + TRACE_PURGE(" Skipping here or parent dir [" << fname << "]. Continue loop."); + continue; + } + + size_t fname_len = strlen(fname); + XrdOssDF *dfh = 0; + + if (S_ISDIR(fstat.st_mode)) + { + if (m_oss_at.Opendir(*iOssDF, fname, env, dfh) == XrdOssOK) + { + cd_down(fname); + TRACE_PURGE(" cd_down -> [" << m_current_path << "]."); + TraverseNamespace(dfh); + cd_up(); + TRACE_PURGE(" cd_up -> [" << m_current_path << "]."); + } + else + TRACE(Warning, trc_pfx << "could not opendir [" << m_current_path << fname << "], " << XrdSysE2T(errno)); + } + else if (fname_len > m_info_ext_len && strncmp(&fname[fname_len - m_info_ext_len], m_info_ext, m_info_ext_len) == 0) + { + // Check if the file is currently opened / purge-protected is done before unlinking of the file. + + Info cinfo(m_trace); + + if (m_oss_at.OpenRO(*iOssDF, fname, env, dfh) == XrdOssOK && cinfo.Read(dfh, m_current_path.c_str(), fname)) + { + CheckFile(fname, cinfo, fstat); + } + else + { + TRACE(Warning, trc_pfx << "can't open or read " << m_current_path << fname << ", err " << XrdSysE2T(errno) << "; purging."); + m_oss_at.Unlink(*iOssDF, fname); + fname[fname_len - m_info_ext_len] = 0; + m_oss_at.Unlink(*iOssDF, fname); + } + } + else // XXXX devel debug only, to be removed + { + TRACE_PURGE(" Ignoring [" << fname << "], not a dir or cinfo."); + } + + delete dfh; + } +} + +/* +void FPurgeState::UnlinkInfoAndData(const char *fname, long long nbytes, XrdOssDF *iOssDF) +{ + fname[fname_len - m_info_ext_len] = 0; + if (nbytes > 0) + { + if ( ! Cache.GetInstance().IsFileActiveOrPurgeProtected(dataPath)) + { + m_n_purged++; + m_bytes_purged += nbytes; + } else + { + m_n_purge_protected++; + m_bytes_purge_protected += nbytes; + m_dir_state->add_usage_purged(nbytes); + // XXXX should also tweak other stuff? + fname[fname_len - m_info_ext_len] = '.'; + return; + } + } + m_oss_at.Unlink(*iOssDF, fname); + fname[fname_len - m_info_ext_len] = '.'; + m_oss_at.Unlink(*iOssDF, fname); +} +*/ + +} // namespace XrdPfc + diff --git a/src/XrdPfc/XrdPfcFPurgeState.hh b/src/XrdPfc/XrdPfcFPurgeState.hh new file mode 100644 index 00000000000..5cd7313e256 --- /dev/null +++ b/src/XrdPfc/XrdPfcFPurgeState.hh @@ -0,0 +1,98 @@ +#ifndef __XRDPFC_FPURGESTATE_HH__ +#define __XRDPFC_FPURGESTATE_HH__ +#include "XrdPfc.hh" +#include "XrdPfcDirState.hh" +#include "XrdPfcFPurgeState.hh" +#include "XrdPfcTrace.hh" +#include "XrdOss/XrdOssAt.hh" +#include "XrdSys/XrdSysTrace.hh" +#include "XrdOuc/XrdOucUtils.hh" +#include "XrdOuc/XrdOucEnv.hh" + + +namespace XrdPfc { + +//============================================================================== +// FPurgeState +//============================================================================== + +class FPurgeState +{ +public: + struct FS + { + std::string path; + long long nBytes; + time_t time; + DirState *dirState; + + FS(const std::string &dname, const char *fname, long long n, time_t t, DirState *ds) : + path(dname + fname), nBytes(n), time(t), dirState(ds) + {} + }; + + typedef std::list list_t; + typedef list_t::iterator list_i; + typedef std::multimap map_t; + typedef map_t::iterator map_i; + +private: + long long m_nBytesReq; + long long m_nBytesAccum; + long long m_nBytesTotal; + time_t m_tMinTimeStamp; + time_t m_tMinUVKeepTimeStamp; + std::vector m_dir_names_stack; + std::vector m_dir_usage_stack; + + + XrdOssAt m_oss_at; + + DirState *m_dir_state; + std::string m_current_path; // Includes trailing '/' + int m_dir_level; + const int m_max_dir_level_for_stat_collection; // until we honor globs from pfc.dirstats + + const char *m_info_ext; + const size_t m_info_ext_len; + XrdSysTrace *m_trace; + + static const char *m_traceID; + + + list_t m_flist; // list of files to be removed unconditionally + map_t m_fmap; // map of files that are purge candidates + +public: + FPurgeState(long long iNBytesReq, XrdOss &oss); + + map_t &refMap() { return m_fmap; } + list_t &refList() { return m_flist; } + + // ------------------------------------ + // Directory handling & stat collection + // ------------------------------------ + + void begin_traversal(DirState *root, const char *root_path = "/"); + + void end_traversal(); + + void cd_down(const std::string& dir_name); + + void cd_up(); + + void setMinTime(time_t min_time) { m_tMinTimeStamp = min_time; } + time_t getMinTime() const { return m_tMinTimeStamp; } + void setUVKeepMinTime(time_t min_time) { m_tMinUVKeepTimeStamp = min_time; } + long long getNBytesTotal() const { return m_nBytesTotal; } + + void MoveListEntriesToMap(); + + void CheckFile(const char *fname, Info &info, struct stat &fstat /*, XrdOssDF *iOssDF*/); + + void TraverseNamespace(XrdOssDF *iOssDF); +}; + +} // namespace XrdPfc + +#endif diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index 8f5fccc046a..0de01e37b53 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -1,175 +1,26 @@ #include "XrdPfc.hh" +#include "XrdPfcDirState.hh" +#include "XrdPfcFPurgeState.hh" #include "XrdPfcTrace.hh" #include #include #include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucUtils.hh" #include "XrdOss/XrdOssAt.hh" #include "XrdSys/XrdSysTrace.hh" using namespace XrdPfc; - namespace XrdPfc { +/* XrdSysTrace* GetTrace() { // needed for logging macros return Cache::GetInstance().GetTrace(); -} - -// Temporary, extensive purge tracing -// #define TRACE_PURGE(x) TRACE(Debug, x) -// #define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" -#define TRACE_PURGE(x) - -//============================================================================== -// DirState -//============================================================================== - -class DirState -{ - DirState *m_parent; - - Stats m_stats; // access stats from client reads in this directory (and subdirs) - - long long m_usage; // collected / measured during purge traversal - long long m_usage_extra; // collected from write events in this directory and subdirs - long long m_usage_purged; // amount of data purged from this directory (and subdirectories for leaf nodes) - - // begin purge traversal usage \_ so we can have a good estimate of what came in during the traversal - // end purge traversal usage / (should be small, presumably) - - // quota info, enabled? - - int m_depth; - int m_max_depth; // XXXX Do we need this? Should it be passed in to find functions? - bool m_stat_report; // not used yet - storing of stats requested - - typedef std::map DsMap_t; - typedef DsMap_t::iterator DsMap_i; - - DsMap_t m_subdirs; - - void init() - { - m_usage = 0; - m_usage_extra = 0; - m_usage_purged = 0; - } - - DirState* create_child(const std::string &dir) - { - std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this))); - return & ir.first->second; - } - - DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs) - { - if (pos == pt.get_n_dirs()) return this; - - DsMap_i i = m_subdirs.find(pt.m_dirs[pos]); - - DirState *ds = 0; - - if (i != m_subdirs.end()) - { - ds = & i->second; - } - if (create_subdirs && m_depth < m_max_depth) - { - ds = create_child(pt.m_dirs[pos]); - } - if (ds) return ds->find_path_tok(pt, pos + 1, create_subdirs); - - return 0; - } - -public: - - DirState(int max_depth) : m_parent(0), m_depth(0), m_max_depth(max_depth) - { - init(); - } - - DirState(DirState *parent) : m_parent(parent), m_depth(m_parent->m_depth + 1), m_max_depth(m_parent->m_max_depth) - { - init(); - } - - DirState* get_parent() { return m_parent; } - - void set_usage(long long u) { m_usage = u; m_usage_extra = 0; } - void add_up_stats(const Stats& stats) { m_stats.AddUp(stats); } - void add_usage_purged(long long up) { m_usage_purged += up; } - - DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs) - { - PathTokenizer pt(path, max_depth, parse_as_lfn); - - return find_path_tok(pt, 0, create_subdirs); - } - - DirState* find_dir(const std::string &dir, bool create_subdirs) - { - DsMap_i i = m_subdirs.find(dir); - - if (i != m_subdirs.end()) return & i->second; - - if (create_subdirs && m_depth < m_max_depth) return create_child(dir); - - return 0; - } - - void reset_stats() - { - m_stats.Reset(); - - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.reset_stats(); - } - } - - void upward_propagate_stats() - { - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.upward_propagate_stats(); - - m_stats.AddUp(i->second.m_stats); - } - - m_usage_extra += m_stats.m_BytesWritten; - } - - long long upward_propagate_usage_purged() - { - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - m_usage_purged += i->second.upward_propagate_usage_purged(); - } - m_usage -= m_usage_purged; - - long long ret = m_usage_purged; - m_usage_purged = 0; - return ret; - } - - void dump_recursively(const char *name) - { - printf("%*d %s usage=%lld usage_extra=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", - 2 + 2*m_depth, m_depth, name, m_usage, m_usage_extra, m_usage + m_usage_extra, - m_stats.m_NumIos, m_stats.m_Duration, m_stats.m_BytesHit, m_stats.m_BytesMissed, m_stats.m_BytesBypassed, m_stats.m_BytesWritten); - - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.dump_recursively(i->first.c_str()); - } - } -}; - +}*/ //============================================================================== // DataFsState @@ -215,312 +66,6 @@ class DataFsState }; -//============================================================================== -// FPurgeState -//============================================================================== - -class FPurgeState -{ -public: - struct FS - { - std::string path; - long long nBytes; - time_t time; - DirState *dirState; - - FS(const std::string &dname, const char *fname, long long n, time_t t, DirState *ds) : - path(dname + fname), nBytes(n), time(t), dirState(ds) - {} - }; - - typedef std::multimap map_t; - typedef map_t::iterator map_i; - - map_t m_fmap; // map of files that are purge candidates - - typedef std::list list_t; - typedef list_t::iterator list_i; - - list_t m_flist; // list of files to be removed unconditionally - - long long nBytesReq; - long long nBytesAccum; - long long nBytesTotal; - time_t tMinTimeStamp; - time_t tMinUVKeepTimeStamp; - - // XrdOss *m_oss; - XrdOssAt m_oss_at; - - // ------------------------------------ - // Directory handling & stat collection - // ------------------------------------ - - DirState *m_dir_state; - std::string m_current_path; // Includes trailing '/' - int m_dir_level; - const int m_max_dir_level_for_stat_collection; // until we honor globs from pfc.dirstats - - std::vector m_dir_names_stack; - std::vector m_dir_usage_stack; - - const char *m_info_ext; - const size_t m_info_ext_len; - XrdSysTrace *m_trace; - - static const char *m_traceID; - - - void begin_traversal(DirState *root, const char *root_path = "/") - { - m_dir_state = root; - m_dir_level = 0; - m_current_path = std::string(root_path); - m_dir_usage_stack.push_back(0); - - TRACE_PURGE("FPurgeState::begin_traversal cur_path '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", level=" << m_dir_level); - } - - void end_traversal() - { - TRACE_PURGE("FPurgeState::end_traversal reporting for '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", nBytesTotal=" << nBytesTotal << ", level=" << m_dir_level); - - m_dir_state->set_usage(m_dir_usage_stack.back()); - - m_dir_state = 0; - } - - void cd_down(const std::string& dir_name) - { - ++m_dir_level; - - if (m_dir_level <= m_max_dir_level_for_stat_collection) - { - m_dir_usage_stack.push_back(0); - m_dir_state = m_dir_state->find_dir(dir_name, true); - } - - m_dir_names_stack.push_back(dir_name); - m_current_path.append(dir_name); - m_current_path.append("/"); - } - - void cd_up() - { - if (m_dir_level <= m_max_dir_level_for_stat_collection) - { - long long tail = m_dir_usage_stack.back(); - m_dir_usage_stack.pop_back(); - - TRACE_PURGE("FPurgeState::cd_up reporting for '" << m_current_path << "', usage=" << tail << ", level=" << m_dir_level); - - m_dir_state->set_usage(tail); - m_dir_state = m_dir_state->get_parent(); - - m_dir_usage_stack.back() += tail; - } - - // remove trailing / and last dir but keep the new trailing / in place. - m_current_path.erase(m_current_path.find_last_of('/', m_current_path.size() - 2) + 1); - m_dir_names_stack.pop_back(); - - --m_dir_level; - } - - // ------------------------------------------------------------------------ - // ------------------------------------------------------------------------ - - FPurgeState(long long iNBytesReq, XrdOss &oss) : - nBytesReq(iNBytesReq), nBytesAccum(0), nBytesTotal(0), tMinTimeStamp(0), tMinUVKeepTimeStamp(0), - // m_oss(oss), - m_oss_at(oss), - m_dir_state(0), m_dir_level(0), - m_max_dir_level_for_stat_collection(Cache::Conf().m_dirStatsStoreDepth), - m_info_ext(XrdPfc::Info::s_infoExtension), - m_info_ext_len(strlen(XrdPfc::Info::s_infoExtension)), - m_trace(Cache::GetInstance().GetTrace()) - { - m_current_path.reserve(256); - m_dir_names_stack.reserve(32); - m_dir_usage_stack.reserve(m_max_dir_level_for_stat_collection + 1); - } - - // ------------------------------------------------------------------------ - - void setMinTime(time_t min_time) { tMinTimeStamp = min_time; } - time_t getMinTime() const { return tMinTimeStamp; } - void setUVKeepMinTime(time_t min_time) { tMinUVKeepTimeStamp = min_time; } - long long getNBytesTotal() const { return nBytesTotal; } - - void MoveListEntriesToMap() - { - for (list_i i = m_flist.begin(); i != m_flist.end(); ++i) - { - m_fmap.insert(std::make_pair(i->time, *i)); - } - m_flist.clear(); - } - - /* - void UnlinkInfoAndData(const char *fname, long long nbytes, XrdOssDF *iOssDF) - { - fname[fname_len - m_info_ext_len] = 0; - if (nbytes > 0) - { - if ( ! Cache.GetInstance().IsFileActiveOrPurgeProtected(dataPath)) - { - m_n_purged++; - m_bytes_purged += nbytes; - } else - { - m_n_purge_protected++; - m_bytes_purge_protected += nbytes; - m_dir_state->add_usage_purged(nbytes); - // XXXX should also tweak other stuff? - fname[fname_len - m_info_ext_len] = '.'; - return; - } - } - m_oss_at.Unlink(*iOssDF, fname); - fname[fname_len - m_info_ext_len] = '.'; - m_oss_at.Unlink(*iOssDF, fname); - } - */ - - void CheckFile(const char *fname, Info &info, struct stat &fstat /*, XrdOssDF *iOssDF*/) - { - static const char *trc_pfx = "FPurgeState::CheckFile "; - - long long nbytes = info.GetNDownloadedBytes(); - time_t atime; - if ( ! info.GetLatestDetachTime(atime)) - { - // cinfo file does not contain any known accesses, use fstat.mtime instead. - TRACE(Debug, trc_pfx << "could not get access time for " << m_current_path << fname << ", using mtime from stat instead."); - atime = fstat.st_mtime; - } - // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime); - - nBytesTotal += nbytes; - - m_dir_usage_stack.back() += nbytes; - - // XXXX Should remove aged-out files here ... but I have trouble getting - // the DirState and purge report set up consistently. - // Need some serious code reorganization here. - // Biggest problem is maintaining overall state a traversal state consistently. - // Sigh. - - // In first two cases we lie about FS time (set to 0) to get them all removed early. - // The age-based purge atime would also be good as there should be nothing - // before that time in the map anyway. - // But we use 0 as a test in purge loop to make sure we continue even if enough - // disk-space has been freed. - - if (tMinTimeStamp > 0 && atime < tMinTimeStamp) - { - m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); - nBytesAccum += nbytes; - } - else if (tMinUVKeepTimeStamp > 0 && - Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) && - info.GetNoCkSumTimeForUVKeep() < tMinUVKeepTimeStamp) - { - m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); - nBytesAccum += nbytes; - } - else if (nBytesAccum < nBytesReq || ( ! m_fmap.empty() && atime < m_fmap.rbegin()->first)) - { - m_fmap.insert(std::make_pair(atime, FS(m_current_path, fname, nbytes, atime, m_dir_state))); - nBytesAccum += nbytes; - - // remove newest files from map if necessary - while ( ! m_fmap.empty() && nBytesAccum - m_fmap.rbegin()->second.nBytes >= nBytesReq) - { - nBytesAccum -= m_fmap.rbegin()->second.nBytes; - m_fmap.erase(--(m_fmap.rbegin().base())); - } - } - } - - void TraverseNamespace(XrdOssDF *iOssDF) - { - static const char *trc_pfx = "FPurgeState::TraverseNamespace "; - - char fname[256]; - struct stat fstat; - XrdOucEnv env; - - TRACE_PURGE("Starting to read dir [" << m_current_path << "], iOssDF->getFD()=" << iOssDF->getFD() << "."); - - iOssDF->StatRet(&fstat); - - while (true) - { - int rc = iOssDF->Readdir(fname, 256); - - if (rc == -ENOENT) { - TRACE_PURGE(" Skipping ENOENT dir entry [" << fname << "]."); - continue; - } - if (rc != XrdOssOK) { - TRACE(Error, trc_pfx << "Readdir error at " << m_current_path << ", err " << XrdSysE2T(-rc) << "."); - break; - } - - TRACE_PURGE(" Readdir [" << fname << "]"); - - if (fname[0] == 0) { - TRACE_PURGE(" Finished reading dir [" << m_current_path << "]. Break loop."); - break; - } - if (fname[0] == '.' && (fname[1] == 0 || (fname[1] == '.' && fname[2] == 0))) { - TRACE_PURGE(" Skipping here or parent dir [" << fname << "]. Continue loop."); - continue; - } - - size_t fname_len = strlen(fname); - XrdOssDF *dfh = 0; - - if (S_ISDIR(fstat.st_mode)) - { - if (m_oss_at.Opendir(*iOssDF, fname, env, dfh) == XrdOssOK) - { - cd_down(fname); TRACE_PURGE(" cd_down -> [" << m_current_path << "]."); - TraverseNamespace(dfh); - cd_up(); TRACE_PURGE(" cd_up -> [" << m_current_path << "]."); - } - else - TRACE(Warning, trc_pfx << "could not opendir [" << m_current_path << fname << "], " << XrdSysE2T(errno)); - } - else if (fname_len > m_info_ext_len && strncmp(&fname[fname_len - m_info_ext_len], m_info_ext, m_info_ext_len) == 0) - { - // Check if the file is currently opened / purge-protected is done before unlinking of the file. - - Info cinfo(m_trace); - - if (m_oss_at.OpenRO(*iOssDF, fname, env, dfh) == XrdOssOK && cinfo.Read(dfh, m_current_path.c_str(), fname)) - { - CheckFile(fname, cinfo, fstat); - } - else - { - TRACE(Warning, trc_pfx << "can't open or read " << m_current_path << fname << ", err " << XrdSysE2T(errno) << "; purging."); - m_oss_at.Unlink(*iOssDF, fname); - fname[fname_len - m_info_ext_len] = 0; - m_oss_at.Unlink(*iOssDF, fname); - } - } - else // XXXX devel debug only, to be removed - { - TRACE_PURGE(" Ignoring [" << fname << "], not a dir or cinfo."); - } - - delete dfh; - } - } -}; const char *FPurgeState::m_traceID = "Purge"; @@ -892,7 +437,7 @@ void Cache::Purge() size_t info_ext_len = strlen(Info::s_infoExtension); int protected_cnt = 0; long long protected_sum = 0; - for (FPurgeState::map_i it = purgeState.m_fmap.begin(); it != purgeState.m_fmap.end(); ++it) + for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) { // Finish when enough space has been freed but not while age-based purging is in progress. // Those files are marked with time-stamp = 0. From 5ba6d908231d77f55b7c4399e68d46b57c93d716 Mon Sep 17 00:00:00 2001 From: alja Date: Thu, 8 Feb 2024 12:18:00 -0800 Subject: [PATCH 02/43] Test purge classes --- src/XrdPfc/XrdPfcDirPurge.hh | 58 +++++++++++++++++++++++++++++ src/XrdPfc/XrdPfcDirPurgeFileCfg.hh | 47 +++++++++++++++++++++++ src/XrdPfc/XrdPfcFPurgeState.cc | 2 +- src/XrdPfc/XrdPfcPurge.cc | 46 +++++++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/XrdPfc/XrdPfcDirPurge.hh create mode 100644 src/XrdPfc/XrdPfcDirPurgeFileCfg.hh diff --git a/src/XrdPfc/XrdPfcDirPurge.hh b/src/XrdPfc/XrdPfcDirPurge.hh new file mode 100644 index 00000000000..3fefbcc71d2 --- /dev/null +++ b/src/XrdPfc/XrdPfcDirPurge.hh @@ -0,0 +1,58 @@ +#ifndef __XRDPFC_PURGEPLG_HH__ +#define __XRDPFC_PURGEPLG_HH__ + +#include +#include +#include "XrdPfc.hh" + +class XrdPfcDirState; + + +namespace XrdPfc +{ +//---------------------------------------------------------------------------- +//! Base class for reguesting directory space to obtain. +//---------------------------------------------------------------------------- +class DirPurge +{ +public: + struct DirInfo + { + std::string path; + long long nBytesQuota{0}; + DirState* dirState{nullptr};// cached + long long nBytesToRecover{0}; + }; + + typedef std::vector list_t; + typedef list_t::iterator list_i; + +protected: + list_t m_list; + +public: + virtual ~DirPurgeRequest() {} + + void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) + { + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); + } + } + + //--------------------------------------------------------------------- + //! Provide erase information from directory statistics + //! + //! @param & XrdPfcDirState + //! + //! @return total number of bytes + //--------------------------------------------------------------------- + virtual long long GetBytesToRecover(XrdPfc::DirState*) = 0; + + list_t& refDirInfos() { return m_list; } +}; +} + +#endif + diff --git a/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh b/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh new file mode 100644 index 00000000000..7660645e6e9 --- /dev/null +++ b/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh @@ -0,0 +1,47 @@ +#ifndef __XRDPFC_FILEINSTRUCTPURGEPLG_HH__ +#define __XRDPFC_FILEINSTRUCTPURGEPLG_HH__ + +#include "XrdPfcDirPurge.hh" +#include "XrdPfcDirState.hh" + +class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge +{ +public: + { + DirInfo d1; + d1.path = "/test/a_LEVEL_1/"; + d1.nBytesQuota = 30 * 1024LL * 1024LL * 1024LL; + m_list.push_back(d1); + + DirInfo d2; + d2.path = "/test/b_LEVEL_1/"; + d2.nBytesQuota = 10 * 1024LL * 1024LL * 1024LL; + m_list.push_back(d2); + } + + //---------------------------------------------------------------------------- + //! Provide bytes to erase from dir quota listed in a text file + //---------------------------------------------------------------------------- + virtual long long GetBytesToRecover(XrdPfc::DirState *ds) + { + // setup disuusage for each dir path + InitDirStatesForLocalPaths(ds); + + long long totalToRemove = 0; + // get bytes to remove + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + long long cv = it->dirState->get_usage() - it->nBytesQuota; + if (cv > 0) + it->nBytesToRecover = cv; + else + it->nBytesToRecover = 0; + + totalToRemove += it->nBytesToRecover; + } + + return totalToRemove; + } +}; + +#endif diff --git a/src/XrdPfc/XrdPfcFPurgeState.cc b/src/XrdPfc/XrdPfcFPurgeState.cc index 91d28ab67fe..d12ce842fe5 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.cc +++ b/src/XrdPfc/XrdPfcFPurgeState.cc @@ -9,7 +9,7 @@ // Temporary, extensive purge tracing // #define TRACE_PURGE(x) TRACE(Debug, x) -// #define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" +#define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" #define TRACE_PURGE(x) using namespace XrdPfc; diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index 0de01e37b53..07b532ec841 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -423,6 +423,51 @@ void Cache::Purge() } } + ///////////////////////////////////////////////////////////// + /// + /// TEST PLUGIN begin + /// + ///////////////////////////////////////////////////////////// + + XrdPfcDirPurgeFileCfg testPlg; + // set dir stat for each path and calculate nBytes to rocover for each path + // return total bytes to recover within the plugin + long long clearVal = testPlg.GetBytesToRecover(m_fs_state->get_root()); + if (clearVal) + { + DirPurgeRequest::list_t &dpl = testPlg.refDirInfos(); + // iterate through the plugin paths + for (DirPurgeRequest::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) + { + XrdOssDF *dh_plg = m_oss->newDir(m_configuration.m_username.c_str()); + FPurgeState purgeState_plg(ppit->nBytesToRecover, *m_oss); + if (dh_plg->Opendir(ppit->path.c_str(), env) == XrdOssOK) + { + DirState* plg_dirState = ppit->dirState; + purgeState_plg.begin_traversal(plg_dirState); + purgeState_plg.TraverseNamespace(dh_plg); + purgeState_plg.end_traversal(); + dh_plg->Close(); + } + + // fill central map from the plugin entry + for (FPurgeState::map_i it = purgeState_plg.refMap().begin(); it != purgeState_plg.refMap().end(); ++it) + { + it->second.path = ppit->path + it->second.path; + printf("AMT found %s bytes %lld \n", it->second.path.c_str(), it->second.nBytes); + purgeState.refMap().insert(std::make_pair(0, it->second)); // set atime to zero to make sure this is deleted + } + } + bytesToRemove = std::max(clearVal, bytesToRemove); + purge_required = true; // set the falg! + } + + ///////////////////////////////////////////////////////////// + /// + /// TEST PLUGIN end + /// + ///////////////////////////////////////////////////////////// + // Dump statistcs before actual purging so maximum usage values get recorded. // Should really go to gstream --- and should really go from Heartbeat. if (m_configuration.is_dir_stat_reporting_on()) @@ -439,6 +484,7 @@ void Cache::Purge() long long protected_sum = 0; for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) { + printf("AMT attempting to unlink file %s \n", it->second.path.c_str()); // Finish when enough space has been freed but not while age-based purging is in progress. // Those files are marked with time-stamp = 0. if (bytesToRemove <= 0 && ! (enforce_age_based_purge && it->first == 0)) From a3a2c02ecc8a4e0263e2cea42a056162a0454cb9 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Thu, 8 Feb 2024 16:09:16 -0800 Subject: [PATCH 03/43] Add created files to the statistics. --- src/XrdPfc/XrdPfcCommand.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index 60d0ca3f19b..ba03e716e19 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -255,6 +255,13 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) m_writeQ.writes_between_purges += file_size; } + { + XrdSysCondVarHelper lock(&m_active_cond); + + XrdPfc::Stats stats; + stats.m_BytesWritten = file_size; + m_closed_files_stats.insert({file_path, stats}); + } } } From 8993804fb14aa421852d9fec772340e9eded4b1e Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Fri, 9 Feb 2024 11:56:25 -0800 Subject: [PATCH 04/43] Correction/addition to the previous commit: rename plugin class --- src/XrdPfc/XrdPfcDirPurge.hh | 26 +++++++++++++------------- src/XrdPfc/XrdPfcDirPurgeFileCfg.hh | 4 ++-- src/XrdPfc/XrdPfcFPurgeState.cc | 2 +- src/XrdPfc/XrdPfcFPurgeState.hh | 1 + src/XrdPfc/XrdPfcPurge.cc | 5 +++-- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/XrdPfc/XrdPfcDirPurge.hh b/src/XrdPfc/XrdPfcDirPurge.hh index 3fefbcc71d2..a45bab3bc9d 100644 --- a/src/XrdPfc/XrdPfcDirPurge.hh +++ b/src/XrdPfc/XrdPfcDirPurge.hh @@ -18,29 +18,29 @@ class DirPurge public: struct DirInfo { - std::string path; - long long nBytesQuota{0}; - DirState* dirState{nullptr};// cached - long long nBytesToRecover{0}; + std::string path; + long long nBytesQuota{0}; + DirState *dirState{nullptr}; // currently cached and shared within the purge thread + long long nBytesToRecover{0}; }; typedef std::vector list_t; - typedef list_t::iterator list_i; + typedef list_t::iterator list_i; protected: list_t m_list; public: - virtual ~DirPurgeRequest() {} + virtual ~DirPurge() {} void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) { - for (list_i it = m_list.begin(); it != m_list.end(); ++it) - { - it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); - } + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); + } } - + //--------------------------------------------------------------------- //! Provide erase information from directory statistics //! @@ -48,9 +48,9 @@ public: //! //! @return total number of bytes //--------------------------------------------------------------------- - virtual long long GetBytesToRecover(XrdPfc::DirState*) = 0; + virtual long long GetBytesToRecover(XrdPfc::DirState *) = 0; - list_t& refDirInfos() { return m_list; } + list_t &refDirInfos() { return m_list; } }; } diff --git a/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh b/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh index 7660645e6e9..bcd11e59fe8 100644 --- a/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh +++ b/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh @@ -7,7 +7,7 @@ class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge { public: - { + XrdPfcDirPurgeFileCfg () { DirInfo d1; d1.path = "/test/a_LEVEL_1/"; d1.nBytesQuota = 30 * 1024LL * 1024LL * 1024LL; @@ -24,7 +24,7 @@ public: //---------------------------------------------------------------------------- virtual long long GetBytesToRecover(XrdPfc::DirState *ds) { - // setup disuusage for each dir path + // setup diskusage for each dir path InitDirStatesForLocalPaths(ds); long long totalToRemove = 0; diff --git a/src/XrdPfc/XrdPfcFPurgeState.cc b/src/XrdPfc/XrdPfcFPurgeState.cc index d12ce842fe5..91d28ab67fe 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.cc +++ b/src/XrdPfc/XrdPfcFPurgeState.cc @@ -9,7 +9,7 @@ // Temporary, extensive purge tracing // #define TRACE_PURGE(x) TRACE(Debug, x) -#define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" +// #define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" #define TRACE_PURGE(x) using namespace XrdPfc; diff --git a/src/XrdPfc/XrdPfcFPurgeState.hh b/src/XrdPfc/XrdPfcFPurgeState.hh index 5cd7313e256..413e8435dc3 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.hh +++ b/src/XrdPfc/XrdPfcFPurgeState.hh @@ -1,6 +1,7 @@ #ifndef __XRDPFC_FPURGESTATE_HH__ #define __XRDPFC_FPURGESTATE_HH__ #include "XrdPfc.hh" +#include "XrdPfcStats.hh" #include "XrdPfcDirState.hh" #include "XrdPfcFPurgeState.hh" #include "XrdPfcTrace.hh" diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index 07b532ec841..dd2e4c5ccd9 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -1,6 +1,7 @@ #include "XrdPfc.hh" #include "XrdPfcDirState.hh" #include "XrdPfcFPurgeState.hh" +#include "XrdPfcDirPurgeFileCfg.hh" #include "XrdPfcTrace.hh" #include @@ -435,9 +436,9 @@ void Cache::Purge() long long clearVal = testPlg.GetBytesToRecover(m_fs_state->get_root()); if (clearVal) { - DirPurgeRequest::list_t &dpl = testPlg.refDirInfos(); + DirPurge::list_t &dpl = testPlg.refDirInfos(); // iterate through the plugin paths - for (DirPurgeRequest::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) + for (DirPurge::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) { XrdOssDF *dh_plg = m_oss->newDir(m_configuration.m_username.c_str()); FPurgeState purgeState_plg(ppit->nBytesToRecover, *m_oss); From 14ba19beb23b53a77250d966adcfa95184bbf029 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Fri, 16 Feb 2024 14:20:24 -0800 Subject: [PATCH 05/43] Instantiate plugin at the time when read configuration --- src/XrdPfc/XrdPfc.cc | 1 + src/XrdPfc/XrdPfc.hh | 3 + src/XrdPfc/XrdPfcConfiguration.cc | 62 ++++++++++++++++++ src/XrdPfc/XrdPfcDirPurge.hh | 32 ++++++---- src/XrdPfc/XrdPfcDirPurgeFileCfg.hh | 98 ++++++++++++++++++++++++++--- src/XrdPfc/XrdPfcPurge.cc | 58 ++++++++--------- 6 files changed, 204 insertions(+), 50 deletions(-) diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index d3766daee64..7a7954002e7 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -189,6 +189,7 @@ Cache::Cache(XrdSysLogger *logger, XrdOucEnv *env) : m_traceID("Cache"), m_oss(0), m_gstream(0), + m_dirpurge(0), m_prefetch_condVar(0), m_prefetch_enabled(false), m_RAM_used(0), diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index fdc4093dd70..8e102dde747 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -31,6 +31,7 @@ #include "XrdPfcFile.hh" #include "XrdPfcDecision.hh" +#include "XrdPfcDirPurge.hh" class XrdOucStream; class XrdSysError; @@ -408,6 +409,7 @@ private: bool ConfigXeq(char *, XrdOucStream &); bool xcschk(XrdOucStream &); bool xdlib(XrdOucStream &); + bool xplib(XrdOucStream &); bool xtrace(XrdOucStream &); bool cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name); @@ -424,6 +426,7 @@ private: XrdXrootdGStream *m_gstream; std::vector m_decisionpoints; //!< decision plugins + XrdPfc::DirPurge* m_dirpurge; //!< purge plugin Configuration m_configuration; //!< configurable parameters diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index 2cdb5869394..ad9d6767cb0 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -2,6 +2,8 @@ #include "XrdPfcTrace.hh" #include "XrdPfcInfo.hh" +#include "XrdPfcDirPurgeFileCfg.hh" + #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" @@ -228,6 +230,62 @@ bool Cache::xdlib(XrdOucStream &Config) return true; } +/* Function: xplib + + Purpose: To parse the directive: purgelib [] + + the path of the decision library to be used. + optional parameters to be passed. + + + Output: true upon success or false upon failure. + */ +bool Cache::xplib(XrdOucStream &Config) +{ + const char* val; + + std::string libp; + if (! (val = Config.GetWord()) || ! val[0]) + { + TRACE(Info," Cache::Config() decisionlib not specified; always caching files"); + return true; + } + else + { + libp = val; + } + + char params[4096]; + if (val[0]) + Config.GetRest(params, 4096); + else + params[0] = 0; + +/* + XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "purgelib", + libp.c_str()); + + DirPurge *(*ep)(XrdSysError&); + ep = (DirPurge *(*)(XrdSysError&))myLib->Resolve("XrdPfcGetDirPurge"); + if (! ep) {myLib->Unload(true); return false; } + + DirPurge * dp = ep(m_log); + if (! dp) + { + TRACE(Error, "Config() decisionlib was not able to create a directory purge object"); + return false; + } + m_dirpurge = dp; + */ + + m_dirpurge = new XrdPfcDirPurgeFileCfg(); + if (params[0]) + m_dirpurge->ConfigDirPurge(params); + + + return true; +} + /* Function: xtrace Purpose: To parse the directive: trace @@ -334,6 +392,10 @@ bool Cache::Config(const char *config_filename, const char *parameters) { retval = xdlib(Config); } + else if (! strcmp(var,"pfc.purgelib")) + { + retval = xplib(Config); + } else if (! strcmp(var,"pfc.trace")) { retval = xtrace(Config); diff --git a/src/XrdPfc/XrdPfcDirPurge.hh b/src/XrdPfc/XrdPfcDirPurge.hh index a45bab3bc9d..72b3e91c2d4 100644 --- a/src/XrdPfc/XrdPfcDirPurge.hh +++ b/src/XrdPfc/XrdPfcDirPurge.hh @@ -4,12 +4,12 @@ #include #include #include "XrdPfc.hh" - -class XrdPfcDirState; - +// #include "XrdPfcDirState.hh" namespace XrdPfc { +class DirState; + //---------------------------------------------------------------------------- //! Base class for reguesting directory space to obtain. //---------------------------------------------------------------------------- @@ -33,14 +33,6 @@ protected: public: virtual ~DirPurge() {} - void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) - { - for (list_i it = m_list.begin(); it != m_list.end(); ++it) - { - it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); - } - } - //--------------------------------------------------------------------- //! Provide erase information from directory statistics //! @@ -50,6 +42,24 @@ public: //--------------------------------------------------------------------- virtual long long GetBytesToRecover(XrdPfc::DirState *) = 0; + //------------------------------------------------------------------------------ + //! Parse configuration arguments. + //! + //! @param params configuration parameters + //! + //! @return status of configuration + //------------------------------------------------------------------------------ + virtual bool ConfigDirPurge(const char* params) // ?? AMT should this be abstract + { + (void) params; + return true; + } + + //----------------------------------------------- + //! + //! Get quotas for the given paths. Used in the XrdPfc:Cache::Purge() thread. + //! + //------------------------------------------------------------------------------ list_t &refDirInfos() { return m_list; } }; } diff --git a/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh b/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh index bcd11e59fe8..51af8d322c7 100644 --- a/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh +++ b/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh @@ -3,20 +3,29 @@ #include "XrdPfcDirPurge.hh" #include "XrdPfcDirState.hh" +// #include "XrdPfcTrace.hh" + +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucUtils.hh" +#include "XrdOuc/XrdOucStream.hh" +#include "XrdOuc/XrdOuca2x.hh" + +#include class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge { public: - XrdPfcDirPurgeFileCfg () { - DirInfo d1; - d1.path = "/test/a_LEVEL_1/"; - d1.nBytesQuota = 30 * 1024LL * 1024LL * 1024LL; - m_list.push_back(d1); - - DirInfo d2; - d2.path = "/test/b_LEVEL_1/"; - d2.nBytesQuota = 10 * 1024LL * 1024LL * 1024LL; - m_list.push_back(d2); + XrdPfcDirPurgeFileCfg () {} + + //---------------------------------------------------------------------------- + //! Set directory statistics + //---------------------------------------------------------------------------- + void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) + { + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); + } } //---------------------------------------------------------------------------- @@ -42,6 +51,75 @@ public: return totalToRemove; } + + //---------------------------------------------------------------------------- + //! Provide bytes to erase from dir quota listed in a text file + //---------------------------------------------------------------------------- + virtual bool ConfigDirPurge(const char *parms) + { + XrdSysError *log = Cache::GetInstance().GetLog(); + + // retrive configuration file name + if (!parms || !parms[0] || (strlen(parms) == 0)) + { + log->Emsg("ConfigDecision", "Quota file not specified."); + return false; + } + log->Emsg("ConfigDecision", "Using directory list", parms); + + // parse the file to get directory quotas + const char* config_filename = parms; + const char *theINS = getenv("XRDINSTANCE"); + XrdOucEnv myEnv; + XrdOucStream Config(log, theINS, &myEnv, "=====> PurgeFileCfg "); + + int fd; + if ((fd = open(config_filename, O_RDONLY, 0)) < 0) + { + log->Emsg("Config() can't open configuration file ", config_filename); + } + + Config.Attach(fd); + static const char *cvec[] = {"*** pfc purge plugin :", 0}; // ?? AMT where is this used + Config.Capture(cvec); + + char *var; + while ((var = Config.GetMyFirstWord())) + { + std::string dirpath = var; + const char* val; + + if (!(val = Config.GetWord())) + { + log->Emsg("PurgeFileCfg", "quota not specified"); + continue; + } + + std::string tmpc = val; + long long quota = 0; + if (::isalpha(*(tmpc.rbegin()))) + { + if (XrdOuca2x::a2sz(*log, "Error getting quota", tmpc.c_str(),"a)) + { + continue; + } + } + else + { + if (XrdOuca2x::a2ll(*log, "Error getting quota", tmpc.c_str(),"a)) + { + continue; + } + } + + DirInfo d; + d.path = dirpath; + d.nBytesQuota = quota; + m_list.push_back(d); + } + + return true; + } }; #endif diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index dd2e4c5ccd9..d1d441382e2 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -1,7 +1,7 @@ #include "XrdPfc.hh" #include "XrdPfcDirState.hh" #include "XrdPfcFPurgeState.hh" -#include "XrdPfcDirPurgeFileCfg.hh" +#include "XrdPfcDirPurge.hh" #include "XrdPfcTrace.hh" #include @@ -429,40 +429,40 @@ void Cache::Purge() /// TEST PLUGIN begin /// ///////////////////////////////////////////////////////////// - - XrdPfcDirPurgeFileCfg testPlg; - // set dir stat for each path and calculate nBytes to rocover for each path - // return total bytes to recover within the plugin - long long clearVal = testPlg.GetBytesToRecover(m_fs_state->get_root()); - if (clearVal) + if (m_dirpurge) { - DirPurge::list_t &dpl = testPlg.refDirInfos(); - // iterate through the plugin paths - for (DirPurge::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) + // set dir stat for each path and calculate nBytes to rocover for each path + // return total bytes to recover within the plugin + long long clearVal = m_dirpurge->GetBytesToRecover(m_fs_state->get_root()); + if (clearVal) { - XrdOssDF *dh_plg = m_oss->newDir(m_configuration.m_username.c_str()); - FPurgeState purgeState_plg(ppit->nBytesToRecover, *m_oss); - if (dh_plg->Opendir(ppit->path.c_str(), env) == XrdOssOK) + DirPurge::list_t &dpl = m_dirpurge->refDirInfos(); + // iterate through the plugin paths + for (DirPurge::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) { - DirState* plg_dirState = ppit->dirState; - purgeState_plg.begin_traversal(plg_dirState); - purgeState_plg.TraverseNamespace(dh_plg); - purgeState_plg.end_traversal(); - dh_plg->Close(); - } - - // fill central map from the plugin entry - for (FPurgeState::map_i it = purgeState_plg.refMap().begin(); it != purgeState_plg.refMap().end(); ++it) - { - it->second.path = ppit->path + it->second.path; - printf("AMT found %s bytes %lld \n", it->second.path.c_str(), it->second.nBytes); - purgeState.refMap().insert(std::make_pair(0, it->second)); // set atime to zero to make sure this is deleted + XrdOssDF *dh_plg = m_oss->newDir(m_configuration.m_username.c_str()); + FPurgeState purgeState_plg(ppit->nBytesToRecover, *m_oss); + if (dh_plg->Opendir(ppit->path.c_str(), env) == XrdOssOK) + { + DirState *plg_dirState = ppit->dirState; + purgeState_plg.begin_traversal(plg_dirState); + purgeState_plg.TraverseNamespace(dh_plg); + purgeState_plg.end_traversal(); + dh_plg->Close(); + } + + // fill central map from the plugin entry + for (FPurgeState::map_i it = purgeState_plg.refMap().begin(); it != purgeState_plg.refMap().end(); ++it) + { + it->second.path = ppit->path + it->second.path; + printf("AMT found %s bytes %lld \n", it->second.path.c_str(), it->second.nBytes); + purgeState.refMap().insert(std::make_pair(0, it->second)); // set atime to zero to make sure this is deleted + } } + bytesToRemove = std::max(clearVal, bytesToRemove); + purge_required = true; // set the falg! } - bytesToRemove = std::max(clearVal, bytesToRemove); - purge_required = true; // set the falg! } - ///////////////////////////////////////////////////////////// /// /// TEST PLUGIN end From 1b37add5fb8611f82bb9777856591eb6e336507c Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 13:28:38 -0800 Subject: [PATCH 06/43] Minimize includes in headers. --- src/XrdPfc/XrdPfc.hh | 5 ++--- src/XrdPfc/XrdPfcDirPurge.hh | 5 +---- src/XrdPfc/XrdPfcDirState.hh | 1 - src/XrdPfc/XrdPfcFile.hh | 1 - src/XrdPfc/XrdPfcIO.hh | 1 - src/XrdPfc/XrdPfcInfo.hh | 2 -- src/XrdPfc/XrdPfcTypes.hh | 1 + 7 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index 8e102dde747..1d5a622e10b 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -27,11 +27,10 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucCallBack.hh" -#include "XrdCl/XrdClDefaultEnv.hh" +// #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdPfcFile.hh" #include "XrdPfcDecision.hh" -#include "XrdPfcDirPurge.hh" class XrdOucStream; class XrdSysError; @@ -42,7 +41,7 @@ namespace XrdPfc { class File; class IO; - +class DirPurge; class DataFsState; } diff --git a/src/XrdPfc/XrdPfcDirPurge.hh b/src/XrdPfc/XrdPfcDirPurge.hh index 72b3e91c2d4..564283849ac 100644 --- a/src/XrdPfc/XrdPfcDirPurge.hh +++ b/src/XrdPfc/XrdPfcDirPurge.hh @@ -3,8 +3,6 @@ #include #include -#include "XrdPfc.hh" -// #include "XrdPfcDirState.hh" namespace XrdPfc { @@ -40,7 +38,7 @@ public: //! //! @return total number of bytes //--------------------------------------------------------------------- - virtual long long GetBytesToRecover(XrdPfc::DirState *) = 0; + virtual long long GetBytesToRecover(DirState *) = 0; //------------------------------------------------------------------------------ //! Parse configuration arguments. @@ -65,4 +63,3 @@ public: } #endif - diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh index 49d137bbb01..8e66bf54771 100644 --- a/src/XrdPfc/XrdPfcDirState.hh +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -1,7 +1,6 @@ #ifndef __XRDPFC_DIRSTATE_HH__ #define __XRDPFC_DIRSTATE_HH__ -// #include "XrdPfcTrace.hh" #include "XrdPfcInfo.hh" using namespace XrdPfc; diff --git a/src/XrdPfc/XrdPfcFile.hh b/src/XrdPfc/XrdPfcFile.hh index 0fcb7c0a021..6e8b74ae7de 100644 --- a/src/XrdPfc/XrdPfcFile.hh +++ b/src/XrdPfc/XrdPfcFile.hh @@ -19,7 +19,6 @@ //---------------------------------------------------------------------------------- #include "XrdCl/XrdClXRootDResponses.hh" -#include "XrdCl/XrdClDefaultEnv.hh" #include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucIOVec.hh" diff --git a/src/XrdPfc/XrdPfcIO.hh b/src/XrdPfc/XrdPfcIO.hh index 1e5f5885b71..731624ac54c 100644 --- a/src/XrdPfc/XrdPfcIO.hh +++ b/src/XrdPfc/XrdPfcIO.hh @@ -5,7 +5,6 @@ class XrdSysTrace; #include "XrdPfc.hh" #include "XrdOuc/XrdOucCache.hh" -#include "XrdCl/XrdClDefaultEnv.hh" #include "XrdSys/XrdSysPthread.hh" #include diff --git a/src/XrdPfc/XrdPfcInfo.hh b/src/XrdPfc/XrdPfcInfo.hh index 36a5bc37ad0..d5082e684ad 100644 --- a/src/XrdPfc/XrdPfcInfo.hh +++ b/src/XrdPfc/XrdPfcInfo.hh @@ -24,8 +24,6 @@ #include #include "XrdSys/XrdSysPthread.hh" -#include "XrdCl/XrdClConstants.hh" -#include "XrdCl/XrdClDefaultEnv.hh" #include "XrdPfcTypes.hh" diff --git a/src/XrdPfc/XrdPfcTypes.hh b/src/XrdPfc/XrdPfcTypes.hh index 5a9a45e62ff..459b4959b86 100644 --- a/src/XrdPfc/XrdPfcTypes.hh +++ b/src/XrdPfc/XrdPfcTypes.hh @@ -1,5 +1,6 @@ #ifndef __XRDPFC_TYPES_HH__ #define __XRDPFC_TYPES_HH__ +#include //---------------------------------------------------------------------------------- // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman From 2cba3b0db597c7eaf9dcf116756ceb2e710365bb Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 13:31:32 -0800 Subject: [PATCH 07/43] Change libxrdPfc.so from type MODULE to type SHARED in the add_library() statement. Add example of a purge plugin. --- src/XrdPfc.cmake | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index 42c12dcdf60..88f326352ac 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -5,6 +5,7 @@ set( LIB_XRD_FILECACHE XrdPfc-${PLUGIN_VERSION} ) set( LIB_XRD_FILECACHE_LEGACY XrdFileCache-${PLUGIN_VERSION} ) set( LIB_XRD_BLACKLIST XrdBlacklistDecision-${PLUGIN_VERSION} ) +set( LIB_XRD_DIRPURGE XrdDirPurge-${PLUGIN_VERSION} ) #------------------------------------------------------------------------------- # Shared library version @@ -15,7 +16,7 @@ set( LIB_XRD_BLACKLIST XrdBlacklistDecision-${PLUGIN_VERSION} ) #------------------------------------------------------------------------------- add_library( ${LIB_XRD_FILECACHE} - MODULE + SHARED XrdPfc/XrdPfcTypes.hh XrdPfc/XrdPfc.cc XrdPfc/XrdPfc.hh XrdPfc/XrdPfcConfiguration.cc @@ -55,6 +56,21 @@ target_link_libraries( XrdUtils ) +#------------------------------------------------------------------------------- +# The XrdDirPurgeFileCfg library +#------------------------------------------------------------------------------- +add_library( + ${LIB_XRD_DIRPURGE} + MODULE + XrdPfc/XrdPfcDirPurgeFileCfg.cc) + +target_link_libraries( + ${LIB_XRD_DIRPURGE} + PRIVATE + XrdUtils + ${LIB_XRD_FILECACHE} + ) + #------------------------------------------------------------------------------- # xrdpfc_print #------------------------------------------------------------------------------- From eb876954fca793be6605a0073e77856b399717ea Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 13:34:00 -0800 Subject: [PATCH 08/43] Initialize purge plugin from configuration --- src/XrdPfc/XrdPfcDirPurgeFileCfg.cc | 133 ++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 src/XrdPfc/XrdPfcDirPurgeFileCfg.cc diff --git a/src/XrdPfc/XrdPfcDirPurgeFileCfg.cc b/src/XrdPfc/XrdPfcDirPurgeFileCfg.cc new file mode 100644 index 00000000000..82f2ce3a28d --- /dev/null +++ b/src/XrdPfc/XrdPfcDirPurgeFileCfg.cc @@ -0,0 +1,133 @@ +#include "XrdPfc.hh" +#include "XrdPfcDirPurge.hh" +#include "XrdPfcDirState.hh" + +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucUtils.hh" +#include "XrdOuc/XrdOucStream.hh" +#include "XrdOuc/XrdOuca2x.hh" + +#include + +class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge +{ +public: + XrdPfcDirPurgeFileCfg () {} + + //---------------------------------------------------------------------------- + //! Set directory statistics + //---------------------------------------------------------------------------- + void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) + { + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); + } + } + + //---------------------------------------------------------------------------- + //! Provide bytes to erase from dir quota listed in a text file + //---------------------------------------------------------------------------- + virtual long long GetBytesToRecover(XrdPfc::DirState *ds) + { + // setup diskusage for each dir path + InitDirStatesForLocalPaths(ds); + + long long totalToRemove = 0; + // get bytes to remove + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + long long cv = it->dirState->get_usage() - it->nBytesQuota; + if (cv > 0) + it->nBytesToRecover = cv; + else + it->nBytesToRecover = 0; + + totalToRemove += it->nBytesToRecover; + } + + return totalToRemove; + } + + //---------------------------------------------------------------------------- + //! Provide bytes to erase from dir quota listed in a text file + //---------------------------------------------------------------------------- + virtual bool ConfigDirPurge(const char *parms) + { + XrdSysError *log = Cache::GetInstance().GetLog(); + + // retrive configuration file name + if (!parms || !parms[0] || (strlen(parms) == 0)) + { + log->Emsg("ConfigDecision", "Quota file not specified."); + return false; + } + log->Emsg("ConfigDecision", "Using directory list", parms); + + // parse the file to get directory quotas + const char* config_filename = parms; + const char *theINS = getenv("XRDINSTANCE"); + XrdOucEnv myEnv; + XrdOucStream Config(log, theINS, &myEnv, "=====> PurgeFileCfg "); + + int fd; + if ((fd = open(config_filename, O_RDONLY, 0)) < 0) + { + log->Emsg("Config() can't open configuration file ", config_filename); + } + + Config.Attach(fd); + static const char *cvec[] = {"*** pfc purge plugin :", 0}; // ?? AMT where is this used + Config.Capture(cvec); + + char *var; + while ((var = Config.GetMyFirstWord())) + { + std::string dirpath = var; + const char* val; + + if (!(val = Config.GetWord())) + { + log->Emsg("PurgeFileCfg", "quota not specified"); + continue; + } + + std::string tmpc = val; + long long quota = 0; + if (::isalpha(*(tmpc.rbegin()))) + { + if (XrdOuca2x::a2sz(*log, "Error getting quota", tmpc.c_str(),"a)) + { + continue; + } + } + else + { + if (XrdOuca2x::a2ll(*log, "Error getting quota", tmpc.c_str(),"a)) + { + continue; + } + } + + DirInfo d; + d.path = dirpath; + d.nBytesQuota = quota; + m_list.push_back(d); + } + + return true; + } +}; + +/******************************************************************************/ +/* XrdPfcGetDirPurge */ +/******************************************************************************/ + +// Return a purge object to use. +extern "C" +{ + XrdPfc::DirPurge *XrdPfcGetDirPurge(XrdSysError &) + { + return new XrdPfcDirPurgeFileCfg(); + } +} From 619cff314402218fb6483fb31e25e13ff195dad6 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 13:34:46 -0800 Subject: [PATCH 09/43] Initialize purge plugin from configuration (append to the previous commit) --- src/XrdPfc/XrdPfcConfiguration.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index ad9d6767cb0..e033de16da4 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -2,7 +2,8 @@ #include "XrdPfcTrace.hh" #include "XrdPfcInfo.hh" -#include "XrdPfcDirPurgeFileCfg.hh" +//#include "XrdPfcDirPurgeFileCfg.hh" +#include "XrdPfcDirPurge.hh" #include "XrdOss/XrdOss.hh" @@ -261,7 +262,6 @@ bool Cache::xplib(XrdOucStream &Config) else params[0] = 0; -/* XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "purgelib", libp.c_str()); @@ -276,9 +276,7 @@ bool Cache::xplib(XrdOucStream &Config) return false; } m_dirpurge = dp; - */ - m_dirpurge = new XrdPfcDirPurgeFileCfg(); if (params[0]) m_dirpurge->ConfigDirPurge(params); From dbc49cc28acf4fc747788e967319565e416e94db Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 13:35:07 -0800 Subject: [PATCH 10/43] File rename --- src/XrdPfc/XrdPfcDirPurgeFileCfg.hh | 125 ---------------------------- 1 file changed, 125 deletions(-) delete mode 100644 src/XrdPfc/XrdPfcDirPurgeFileCfg.hh diff --git a/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh b/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh deleted file mode 100644 index 51af8d322c7..00000000000 --- a/src/XrdPfc/XrdPfcDirPurgeFileCfg.hh +++ /dev/null @@ -1,125 +0,0 @@ -#ifndef __XRDPFC_FILEINSTRUCTPURGEPLG_HH__ -#define __XRDPFC_FILEINSTRUCTPURGEPLG_HH__ - -#include "XrdPfcDirPurge.hh" -#include "XrdPfcDirState.hh" -// #include "XrdPfcTrace.hh" - -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucUtils.hh" -#include "XrdOuc/XrdOucStream.hh" -#include "XrdOuc/XrdOuca2x.hh" - -#include - -class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge -{ -public: - XrdPfcDirPurgeFileCfg () {} - - //---------------------------------------------------------------------------- - //! Set directory statistics - //---------------------------------------------------------------------------- - void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) - { - for (list_i it = m_list.begin(); it != m_list.end(); ++it) - { - it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); - } - } - - //---------------------------------------------------------------------------- - //! Provide bytes to erase from dir quota listed in a text file - //---------------------------------------------------------------------------- - virtual long long GetBytesToRecover(XrdPfc::DirState *ds) - { - // setup diskusage for each dir path - InitDirStatesForLocalPaths(ds); - - long long totalToRemove = 0; - // get bytes to remove - for (list_i it = m_list.begin(); it != m_list.end(); ++it) - { - long long cv = it->dirState->get_usage() - it->nBytesQuota; - if (cv > 0) - it->nBytesToRecover = cv; - else - it->nBytesToRecover = 0; - - totalToRemove += it->nBytesToRecover; - } - - return totalToRemove; - } - - //---------------------------------------------------------------------------- - //! Provide bytes to erase from dir quota listed in a text file - //---------------------------------------------------------------------------- - virtual bool ConfigDirPurge(const char *parms) - { - XrdSysError *log = Cache::GetInstance().GetLog(); - - // retrive configuration file name - if (!parms || !parms[0] || (strlen(parms) == 0)) - { - log->Emsg("ConfigDecision", "Quota file not specified."); - return false; - } - log->Emsg("ConfigDecision", "Using directory list", parms); - - // parse the file to get directory quotas - const char* config_filename = parms; - const char *theINS = getenv("XRDINSTANCE"); - XrdOucEnv myEnv; - XrdOucStream Config(log, theINS, &myEnv, "=====> PurgeFileCfg "); - - int fd; - if ((fd = open(config_filename, O_RDONLY, 0)) < 0) - { - log->Emsg("Config() can't open configuration file ", config_filename); - } - - Config.Attach(fd); - static const char *cvec[] = {"*** pfc purge plugin :", 0}; // ?? AMT where is this used - Config.Capture(cvec); - - char *var; - while ((var = Config.GetMyFirstWord())) - { - std::string dirpath = var; - const char* val; - - if (!(val = Config.GetWord())) - { - log->Emsg("PurgeFileCfg", "quota not specified"); - continue; - } - - std::string tmpc = val; - long long quota = 0; - if (::isalpha(*(tmpc.rbegin()))) - { - if (XrdOuca2x::a2sz(*log, "Error getting quota", tmpc.c_str(),"a)) - { - continue; - } - } - else - { - if (XrdOuca2x::a2ll(*log, "Error getting quota", tmpc.c_str(),"a)) - { - continue; - } - } - - DirInfo d; - d.path = dirpath; - d.nBytesQuota = quota; - m_list.push_back(d); - } - - return true; - } -}; - -#endif From b9660f9e26b7d0d65d6f72da8c66f4f01c18bcc5 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 13:54:42 -0800 Subject: [PATCH 11/43] Rename purge plugin sources --- src/XrdPfc/XrdPfcConfiguration.cc | 1 - .../{XrdPfcDirPurgeFileCfg.cc => XrdPfcPurgeQuota.cc} | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) rename src/XrdPfc/{XrdPfcDirPurgeFileCfg.cc => XrdPfcPurgeQuota.cc} (93%) diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index e033de16da4..776a56d0e66 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -2,7 +2,6 @@ #include "XrdPfcTrace.hh" #include "XrdPfcInfo.hh" -//#include "XrdPfcDirPurgeFileCfg.hh" #include "XrdPfcDirPurge.hh" #include "XrdOss/XrdOss.hh" diff --git a/src/XrdPfc/XrdPfcDirPurgeFileCfg.cc b/src/XrdPfc/XrdPfcPurgeQuota.cc similarity index 93% rename from src/XrdPfc/XrdPfcDirPurgeFileCfg.cc rename to src/XrdPfc/XrdPfcPurgeQuota.cc index 82f2ce3a28d..439963da760 100644 --- a/src/XrdPfc/XrdPfcDirPurgeFileCfg.cc +++ b/src/XrdPfc/XrdPfcPurgeQuota.cc @@ -9,10 +9,10 @@ #include -class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge +class XrdPfcPurgeQuota : public XrdPfc::DirPurge { public: - XrdPfcDirPurgeFileCfg () {} + XrdPfcPurgeQuota () {} //---------------------------------------------------------------------------- //! Set directory statistics @@ -68,7 +68,7 @@ class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge const char* config_filename = parms; const char *theINS = getenv("XRDINSTANCE"); XrdOucEnv myEnv; - XrdOucStream Config(log, theINS, &myEnv, "=====> PurgeFileCfg "); + XrdOucStream Config(log, theINS, &myEnv, "=====> PurgeQuota "); int fd; if ((fd = open(config_filename, O_RDONLY, 0)) < 0) @@ -88,7 +88,7 @@ class XrdPfcDirPurgeFileCfg : public XrdPfc::DirPurge if (!(val = Config.GetWord())) { - log->Emsg("PurgeFileCfg", "quota not specified"); + log->Emsg("PurgeQuota plugin", "quota not specified"); continue; } @@ -128,6 +128,6 @@ extern "C" { XrdPfc::DirPurge *XrdPfcGetDirPurge(XrdSysError &) { - return new XrdPfcDirPurgeFileCfg(); + return new XrdPfcPurgeQuota(); } } From e60c0f1faff1b8609340f6084dc5ecf32724d49d Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 13:55:18 -0800 Subject: [PATCH 12/43] Rename purge plugin sources (append to the previous commit) --- src/XrdPfc.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index 88f326352ac..02cc498f094 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -5,7 +5,7 @@ set( LIB_XRD_FILECACHE XrdPfc-${PLUGIN_VERSION} ) set( LIB_XRD_FILECACHE_LEGACY XrdFileCache-${PLUGIN_VERSION} ) set( LIB_XRD_BLACKLIST XrdBlacklistDecision-${PLUGIN_VERSION} ) -set( LIB_XRD_DIRPURGE XrdDirPurge-${PLUGIN_VERSION} ) +set( LIB_XRD_DIRPURGE XrdPfcPurgeQuota-${PLUGIN_VERSION} ) #------------------------------------------------------------------------------- # Shared library version @@ -62,7 +62,7 @@ target_link_libraries( add_library( ${LIB_XRD_DIRPURGE} MODULE - XrdPfc/XrdPfcDirPurgeFileCfg.cc) + XrdPfc/XrdPfcPurgeQuota.cc) target_link_libraries( ${LIB_XRD_DIRPURGE} From 74ff271dce6e76c3ec15b4cc959ccea9f203b6ff Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 14:02:15 -0800 Subject: [PATCH 13/43] Remove line with a commented include --- src/XrdPfc/XrdPfc.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index 1d5a622e10b..2b85f096516 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -27,7 +27,6 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucCallBack.hh" -// #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdPfcFile.hh" #include "XrdPfcDecision.hh" From 342cbd022b831d07c5377cf4e7bddf0df48576bf Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 14:39:01 -0800 Subject: [PATCH 14/43] Remove obsolete comment --- src/XrdPfc/XrdPfcPurgeQuota.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPfc/XrdPfcPurgeQuota.cc b/src/XrdPfc/XrdPfcPurgeQuota.cc index 439963da760..409c262d41a 100644 --- a/src/XrdPfc/XrdPfcPurgeQuota.cc +++ b/src/XrdPfc/XrdPfcPurgeQuota.cc @@ -77,7 +77,7 @@ class XrdPfcPurgeQuota : public XrdPfc::DirPurge } Config.Attach(fd); - static const char *cvec[] = {"*** pfc purge plugin :", 0}; // ?? AMT where is this used + static const char *cvec[] = {"*** pfc purge plugin :", 0}; Config.Capture(cvec); char *var; From ca5c041b0701a2c36c63bdd7057abe91c2347750 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 14:39:26 -0800 Subject: [PATCH 15/43] Improve plugin log messages --- src/XrdPfc/XrdPfcPurge.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index d1d441382e2..561cbb69c39 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -426,7 +426,7 @@ void Cache::Purge() ///////////////////////////////////////////////////////////// /// - /// TEST PLUGIN begin + /// PurgePin begin /// ///////////////////////////////////////////////////////////// if (m_dirpurge) @@ -436,10 +436,12 @@ void Cache::Purge() long long clearVal = m_dirpurge->GetBytesToRecover(m_fs_state->get_root()); if (clearVal) { + TRACE(Debug, "PurgePin remove total " << clearVal << " bytes"); DirPurge::list_t &dpl = m_dirpurge->refDirInfos(); // iterate through the plugin paths for (DirPurge::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) { + TRACE(Debug, "\tPurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes"); XrdOssDF *dh_plg = m_oss->newDir(m_configuration.m_username.c_str()); FPurgeState purgeState_plg(ppit->nBytesToRecover, *m_oss); if (dh_plg->Opendir(ppit->path.c_str(), env) == XrdOssOK) @@ -455,7 +457,7 @@ void Cache::Purge() for (FPurgeState::map_i it = purgeState_plg.refMap().begin(); it != purgeState_plg.refMap().end(); ++it) { it->second.path = ppit->path + it->second.path; - printf("AMT found %s bytes %lld \n", it->second.path.c_str(), it->second.nBytes); + TRACE(Debug, "\t\tPurgePin found file " << it->second.path.c_str()<< " size " << it->second.nBytes); purgeState.refMap().insert(std::make_pair(0, it->second)); // set atime to zero to make sure this is deleted } } @@ -465,7 +467,7 @@ void Cache::Purge() } ///////////////////////////////////////////////////////////// /// - /// TEST PLUGIN end + /// PurgePin end /// ///////////////////////////////////////////////////////////// @@ -485,7 +487,6 @@ void Cache::Purge() long long protected_sum = 0; for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) { - printf("AMT attempting to unlink file %s \n", it->second.path.c_str()); // Finish when enough space has been freed but not while age-based purging is in progress. // Those files are marked with time-stamp = 0. if (bytesToRemove <= 0 && ! (enforce_age_based_purge && it->first == 0)) From b49da64aef5996bb267cee0ed9dcbab344201460 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 21 Feb 2024 14:56:59 -0800 Subject: [PATCH 16/43] Rename base class for purge plugin from PurgeDir to PurgePin (plugin interface --- src/XrdPfc.cmake | 8 ++-- src/XrdPfc/XrdPfc.cc | 2 +- src/XrdPfc/XrdPfc.hh | 6 +-- src/XrdPfc/XrdPfcConfiguration.cc | 12 +++--- src/XrdPfc/XrdPfcDirPurge.hh | 6 +-- src/XrdPfc/XrdPfcPurge.cc | 8 ++-- src/XrdPfc/XrdPfcPurgePin.hh | 65 +++++++++++++++++++++++++++++++ src/XrdPfc/XrdPfcPurgeQuota.cc | 10 ++--- 8 files changed, 91 insertions(+), 26 deletions(-) create mode 100644 src/XrdPfc/XrdPfcPurgePin.hh diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index 02cc498f094..292c065a95c 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -5,7 +5,7 @@ set( LIB_XRD_FILECACHE XrdPfc-${PLUGIN_VERSION} ) set( LIB_XRD_FILECACHE_LEGACY XrdFileCache-${PLUGIN_VERSION} ) set( LIB_XRD_BLACKLIST XrdBlacklistDecision-${PLUGIN_VERSION} ) -set( LIB_XRD_DIRPURGE XrdPfcPurgeQuota-${PLUGIN_VERSION} ) +set( LIB_XRD_PURGEQUOTA XrdPfcPurgeQuota-${PLUGIN_VERSION} ) #------------------------------------------------------------------------------- # Shared library version @@ -57,15 +57,15 @@ target_link_libraries( ) #------------------------------------------------------------------------------- -# The XrdDirPurgeFileCfg library +# The XrdPurgeQuota library #------------------------------------------------------------------------------- add_library( - ${LIB_XRD_DIRPURGE} + ${LIB_XRD_PURGEQUOTA} MODULE XrdPfc/XrdPfcPurgeQuota.cc) target_link_libraries( - ${LIB_XRD_DIRPURGE} + ${LIB_XRD_PURGEQUOTA} PRIVATE XrdUtils ${LIB_XRD_FILECACHE} diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index 7a7954002e7..b20e7f07f04 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -189,7 +189,7 @@ Cache::Cache(XrdSysLogger *logger, XrdOucEnv *env) : m_traceID("Cache"), m_oss(0), m_gstream(0), - m_dirpurge(0), + m_purge_pin(0), m_prefetch_condVar(0), m_prefetch_enabled(false), m_RAM_used(0), diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index 2b85f096516..94c44645771 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -9,7 +9,7 @@ // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// XRootD is distributed in the hope that it will be useful, +// XRootD is distributed in the hope that it will be useful,fm_pu // but WITHOUT ANY emacs WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. @@ -40,7 +40,7 @@ namespace XrdPfc { class File; class IO; -class DirPurge; +class PurgePin; class DataFsState; } @@ -424,7 +424,7 @@ private: XrdXrootdGStream *m_gstream; std::vector m_decisionpoints; //!< decision plugins - XrdPfc::DirPurge* m_dirpurge; //!< purge plugin + XrdPfc::PurgePin* m_purge_pin; //!< purge plugin Configuration m_configuration; //!< configurable parameters diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index 776a56d0e66..74885a4fc78 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -2,7 +2,7 @@ #include "XrdPfcTrace.hh" #include "XrdPfcInfo.hh" -#include "XrdPfcDirPurge.hh" +#include "XrdPfcPurgePin.hh" #include "XrdOss/XrdOss.hh" @@ -264,20 +264,20 @@ bool Cache::xplib(XrdOucStream &Config) XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "purgelib", libp.c_str()); - DirPurge *(*ep)(XrdSysError&); - ep = (DirPurge *(*)(XrdSysError&))myLib->Resolve("XrdPfcGetDirPurge"); + PurgePin *(*ep)(XrdSysError&); + ep = (PurgePin *(*)(XrdSysError&))myLib->Resolve("XrdPfcGetPurgePin"); if (! ep) {myLib->Unload(true); return false; } - DirPurge * dp = ep(m_log); + PurgePin * dp = ep(m_log); if (! dp) { TRACE(Error, "Config() decisionlib was not able to create a directory purge object"); return false; } - m_dirpurge = dp; + m_purge_pin = dp; if (params[0]) - m_dirpurge->ConfigDirPurge(params); + m_purge_pin->ConfigPurgePin(params); return true; diff --git a/src/XrdPfc/XrdPfcDirPurge.hh b/src/XrdPfc/XrdPfcDirPurge.hh index 564283849ac..67e48e42062 100644 --- a/src/XrdPfc/XrdPfcDirPurge.hh +++ b/src/XrdPfc/XrdPfcDirPurge.hh @@ -11,7 +11,7 @@ class DirState; //---------------------------------------------------------------------------- //! Base class for reguesting directory space to obtain. //---------------------------------------------------------------------------- -class DirPurge +class PurgePin { public: struct DirInfo @@ -29,7 +29,7 @@ protected: list_t m_list; public: - virtual ~DirPurge() {} + virtual ~PurgePin() {} //--------------------------------------------------------------------- //! Provide erase information from directory statistics @@ -47,7 +47,7 @@ public: //! //! @return status of configuration //------------------------------------------------------------------------------ - virtual bool ConfigDirPurge(const char* params) // ?? AMT should this be abstract + virtual bool ConfigPurgePin(const char* params) // ?? AMT should this be abstract { (void) params; return true; diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index 561cbb69c39..ffc5989e0e2 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -429,17 +429,17 @@ void Cache::Purge() /// PurgePin begin /// ///////////////////////////////////////////////////////////// - if (m_dirpurge) + if (m_purge_pin) { // set dir stat for each path and calculate nBytes to rocover for each path // return total bytes to recover within the plugin - long long clearVal = m_dirpurge->GetBytesToRecover(m_fs_state->get_root()); + long long clearVal = m_purge_pin->GetBytesToRecover(m_fs_state->get_root()); if (clearVal) { TRACE(Debug, "PurgePin remove total " << clearVal << " bytes"); - DirPurge::list_t &dpl = m_dirpurge->refDirInfos(); + PurgePin::list_t &dpl = m_purge_pin->refDirInfos(); // iterate through the plugin paths - for (DirPurge::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) + for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) { TRACE(Debug, "\tPurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes"); XrdOssDF *dh_plg = m_oss->newDir(m_configuration.m_username.c_str()); diff --git a/src/XrdPfc/XrdPfcPurgePin.hh b/src/XrdPfc/XrdPfcPurgePin.hh new file mode 100644 index 00000000000..67e48e42062 --- /dev/null +++ b/src/XrdPfc/XrdPfcPurgePin.hh @@ -0,0 +1,65 @@ +#ifndef __XRDPFC_PURGEPLG_HH__ +#define __XRDPFC_PURGEPLG_HH__ + +#include +#include + +namespace XrdPfc +{ +class DirState; + +//---------------------------------------------------------------------------- +//! Base class for reguesting directory space to obtain. +//---------------------------------------------------------------------------- +class PurgePin +{ +public: + struct DirInfo + { + std::string path; + long long nBytesQuota{0}; + DirState *dirState{nullptr}; // currently cached and shared within the purge thread + long long nBytesToRecover{0}; + }; + + typedef std::vector list_t; + typedef list_t::iterator list_i; + +protected: + list_t m_list; + +public: + virtual ~PurgePin() {} + + //--------------------------------------------------------------------- + //! Provide erase information from directory statistics + //! + //! @param & XrdPfcDirState + //! + //! @return total number of bytes + //--------------------------------------------------------------------- + virtual long long GetBytesToRecover(DirState *) = 0; + + //------------------------------------------------------------------------------ + //! Parse configuration arguments. + //! + //! @param params configuration parameters + //! + //! @return status of configuration + //------------------------------------------------------------------------------ + virtual bool ConfigPurgePin(const char* params) // ?? AMT should this be abstract + { + (void) params; + return true; + } + + //----------------------------------------------- + //! + //! Get quotas for the given paths. Used in the XrdPfc:Cache::Purge() thread. + //! + //------------------------------------------------------------------------------ + list_t &refDirInfos() { return m_list; } +}; +} + +#endif diff --git a/src/XrdPfc/XrdPfcPurgeQuota.cc b/src/XrdPfc/XrdPfcPurgeQuota.cc index 409c262d41a..2d5f18d688e 100644 --- a/src/XrdPfc/XrdPfcPurgeQuota.cc +++ b/src/XrdPfc/XrdPfcPurgeQuota.cc @@ -1,5 +1,5 @@ #include "XrdPfc.hh" -#include "XrdPfcDirPurge.hh" +#include "XrdPfcPurgePin.hh" #include "XrdPfcDirState.hh" #include "XrdOuc/XrdOucEnv.hh" @@ -9,7 +9,7 @@ #include -class XrdPfcPurgeQuota : public XrdPfc::DirPurge +class XrdPfcPurgeQuota : public XrdPfc::PurgePin { public: XrdPfcPurgeQuota () {} @@ -52,7 +52,7 @@ class XrdPfcPurgeQuota : public XrdPfc::DirPurge //---------------------------------------------------------------------------- //! Provide bytes to erase from dir quota listed in a text file //---------------------------------------------------------------------------- - virtual bool ConfigDirPurge(const char *parms) + virtual bool ConfigPurgePin(const char *parms) { XrdSysError *log = Cache::GetInstance().GetLog(); @@ -120,13 +120,13 @@ class XrdPfcPurgeQuota : public XrdPfc::DirPurge }; /******************************************************************************/ -/* XrdPfcGetDirPurge */ +/* XrdPfcGetPurgePin */ /******************************************************************************/ // Return a purge object to use. extern "C" { - XrdPfc::DirPurge *XrdPfcGetDirPurge(XrdSysError &) + XrdPfc::PurgePin *XrdPfcGetPurgePin(XrdSysError &) { return new XrdPfcPurgeQuota(); } From 9ee907e82aad199f426bf9cc45a3d80017603051 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Fri, 8 Mar 2024 14:37:16 -0800 Subject: [PATCH 17/43] Rename local variables in the purge plugin block --- src/XrdPfc/XrdPfcPurge.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index ffc5989e0e2..27f69612804 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -442,19 +442,19 @@ void Cache::Purge() for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) { TRACE(Debug, "\tPurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes"); - XrdOssDF *dh_plg = m_oss->newDir(m_configuration.m_username.c_str()); - FPurgeState purgeState_plg(ppit->nBytesToRecover, *m_oss); - if (dh_plg->Opendir(ppit->path.c_str(), env) == XrdOssOK) + XrdOssDF *ldh = m_oss->newDir(m_configuration.m_username.c_str()); + FPurgeState fps(ppit->nBytesToRecover, *m_oss); + if (ldh->Opendir(ppit->path.c_str(), env) == XrdOssOK) { - DirState *plg_dirState = ppit->dirState; - purgeState_plg.begin_traversal(plg_dirState); - purgeState_plg.TraverseNamespace(dh_plg); - purgeState_plg.end_traversal(); - dh_plg->Close(); + DirState *lds = ppit->dirState; + fps.begin_traversal(lds); + fps.TraverseNamespace(ldh); + fps.end_traversal(); + ldh->Close(); } // fill central map from the plugin entry - for (FPurgeState::map_i it = purgeState_plg.refMap().begin(); it != purgeState_plg.refMap().end(); ++it) + for (FPurgeState::map_i it = fps.refMap().begin(); it != fps.refMap().end(); ++it) { it->second.path = ppit->path + it->second.path; TRACE(Debug, "\t\tPurgePin found file " << it->second.path.c_str()<< " size " << it->second.nBytes); From b80e5c1d1e9755f614520730cee28e2712a8c511 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Tue, 12 Mar 2024 16:29:36 -0700 Subject: [PATCH 18/43] Move DirState function implementation to a new XrdPfcDirState.cc source file --- src/XrdPfc/XrdPfcDirState.cc | 190 +++++++++++++++++++++++++++++++++++ src/XrdPfc/XrdPfcDirState.hh | 135 +++++-------------------- 2 files changed, 216 insertions(+), 109 deletions(-) create mode 100644 src/XrdPfc/XrdPfcDirState.cc diff --git a/src/XrdPfc/XrdPfcDirState.cc b/src/XrdPfc/XrdPfcDirState.cc new file mode 100644 index 00000000000..dc90d9d05b4 --- /dev/null +++ b/src/XrdPfc/XrdPfcDirState.cc @@ -0,0 +1,190 @@ +#include "XrdPfcDirState.hh" +#include "XrdPfcStats.hh" + +#include + +namespace XrdPfc +{ +//---------------------------------------------------------------------------- +//! Constructor +//! @param int max subdir depth +//---------------------------------------------------------------------------- +DirState::DirState(int max_depth) : m_parent(0), m_depth(0), m_max_depth(max_depth) +{ + // init(); +} + +//---------------------------------------------------------------------------- +//! Constructor +//! @param DirState parent directory +//---------------------------------------------------------------------------- +DirState::DirState(DirState *parent) : m_parent(parent), m_depth(m_parent->m_depth + 1), m_max_depth(m_parent->m_max_depth) +{ + // init(); +} +/* +void DirState::init() +{ + m_usage = 0; + m_usage_extra = 0; + m_usage_purged = 0; +}*/ + +//---------------------------------------------------------------------------- +//! Internal function called from find_dir or find_path_tok +//! @param dir subdir name +//---------------------------------------------------------------------------- +DirState* DirState::create_child(const std::string &dir) +{ + std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this))); + return & ir.first->second; +} + + +//---------------------------------------------------------------------------- +//! Internal function called from find_path +//! @param dir subdir name +//---------------------------------------------------------------------------- +DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs) +{ + if (pos == pt.get_n_dirs()) return this; + + DsMap_i i = m_subdirs.find(pt.m_dirs[pos]); + + DirState *ds = 0; + + if (i != m_subdirs.end()) + { + ds = & i->second; + } + if (create_subdirs && m_depth < m_max_depth) + { + ds = create_child(pt.m_dirs[pos]); + } + if (ds) return ds->find_path_tok(pt, pos + 1, create_subdirs); + + return 0; +} + + + +//---------------------------------------------------------------------------- +//! Recursive function to find DireState with given absolute dir path +//! @param dir subdir name +//! @param max_depth +//! @param parse_as_lfn +//! @param create_subdirs +DirState* DirState::find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs) +{ + PathTokenizer pt(path, max_depth, parse_as_lfn); + + return find_path_tok(pt, 0, create_subdirs); +} + +//---------------------------------------------------------------------------- +//! Non recursive function +//! @param dir subdir name @param bool create the subdir in this DirsStat +DirState* DirState::find_dir(const std::string &dir, bool create_subdirs) +{ + DsMap_i i = m_subdirs.find(dir); + + if (i != m_subdirs.end()) return & i->second; + + if (create_subdirs && m_depth < m_max_depth) return create_child(dir); + + return 0; +} + +//---------------------------------------------------------------------------- +//! Reset current transaction statistics. +//! Called from Cache::copy_out_active_stats_and_update_data_fs_state() +//---------------------------------------------------------------------------- +void DirState::reset_stats() +{ + m_stats.Reset(); + + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.reset_stats(); + } +} + +//---------------------------------------------------------------------------- +//! Propagate stat to parents +//! Called from Cache::copy_out_active_stats_and_update_data_fs_state() +//---------------------------------------------------------------------------- +void DirState::upward_propagate_stats() +{ + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.upward_propagate_stats(); + + m_stats.AddUp(i->second.m_stats); + } + + m_usage_extra += m_stats.m_BytesWritten; +} + +//---------------------------------------------------------------------------- +//! Update statistics. +//! Called from Purge thread. +//---------------------------------------------------------------------------- +long long DirState::upward_propagate_usage_purged() +{ + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + m_usage_purged += i->second.upward_propagate_usage_purged(); + } + m_usage -= m_usage_purged; + + long long ret = m_usage_purged; + m_usage_purged = 0; + return ret; +} + +//---------------------------------------------------------------------------- +//! Recursive print of statistics. Called if defined in pfc configuration. +//! +//---------------------------------------------------------------------------- +void DirState::dump_recursively(const char *name) +{ + printf("%*d %s usage=%lld usage_extra=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", + 2 + 2*m_depth, m_depth, name, m_usage, m_usage_extra, m_usage + m_usage_extra, + m_stats.m_NumIos, m_stats.m_Duration, m_stats.m_BytesHit, m_stats.m_BytesMissed, m_stats.m_BytesBypassed, m_stats.m_BytesWritten); + + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.dump_recursively(i->first.c_str()); + } +} + +//---------------------------------------------------------------------------- +//! Set read usage +//! +//---------------------------------------------------------------------------- +void DirState::set_usage(long long u) +{ + m_usage = u; + m_usage_extra = 0; +} + + +//---------------------------------------------------------------------------- +//! Add to temporary Stat obj +//! +//---------------------------------------------------------------------------- +void DirState::add_up_stats(const Stats& stats) +{ + m_stats.AddUp(stats); +} + +//---------------------------------------------------------------------------- +//! Accumulate usage from purged files +//! +//---------------------------------------------------------------------------- +void DirState::add_usage_purged(long long up) +{ + m_usage_purged += up; +} + +} // end namespace diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh index 8e66bf54771..4989bd5473f 100644 --- a/src/XrdPfc/XrdPfcDirState.hh +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -1,7 +1,11 @@ #ifndef __XRDPFC_DIRSTATE_HH__ #define __XRDPFC_DIRSTATE_HH__ +#include "XrdPfc.hh" #include "XrdPfcInfo.hh" +#include "XrdPfcStats.hh" +#include +#include using namespace XrdPfc; @@ -18,9 +22,9 @@ class DirState Stats m_stats; // access stats from client reads in this directory (and subdirs) - long long m_usage; // collected / measured during purge traversal - long long m_usage_extra; // collected from write events in this directory and subdirs - long long m_usage_purged; // amount of data purged from this directory (and subdirectories for leaf nodes) + long long m_usage{0}; // collected / measured during purge traversal + long long m_usage_extra{0}; // collected from write events in this directory and subdirs + long long m_usage_purged{0}; // amount of data purged from this directory (and subdirectories for leaf nodes) // begin purge traversal usage \_ so we can have a good estimate of what came in during the traversal // end purge traversal usage / (should be small, presumably) @@ -36,124 +40,37 @@ class DirState DsMap_t m_subdirs; - void init() - { - m_usage = 0; - m_usage_extra = 0; - m_usage_purged = 0; - } + void init(); - DirState* create_child(const std::string &dir) - { - std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this))); - return & ir.first->second; - } + DirState* create_child(const std::string &dir); - DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs) - { - if (pos == pt.get_n_dirs()) return this; + DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs); - DsMap_i i = m_subdirs.find(pt.m_dirs[pos]); +public: - DirState *ds = 0; + DirState(int max_depth); - if (i != m_subdirs.end()) - { - ds = & i->second; - } - if (create_subdirs && m_depth < m_max_depth) - { - ds = create_child(pt.m_dirs[pos]); - } - if (ds) return ds->find_path_tok(pt, pos + 1, create_subdirs); + DirState(DirState *parent); - return 0; - } + DirState* get_parent() { return m_parent; } -public: + long long get_usage() { return m_usage; } - DirState(int max_depth) : m_parent(0), m_depth(0), m_max_depth(max_depth) - { - init(); - } + void set_usage(long long u); + void add_up_stats(const Stats& stats); + void add_usage_purged(long long up); - DirState(DirState *parent) : m_parent(parent), m_depth(m_parent->m_depth + 1), m_max_depth(m_parent->m_max_depth) - { - init(); - } + DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs); - DirState* get_parent() { return m_parent; } + DirState* find_dir(const std::string &dir, bool create_subdirs); - long long get_usage() { return m_usage; } + void reset_stats(); + + void upward_propagate_stats(); + + long long upward_propagate_usage_purged(); - void set_usage(long long u) { m_usage = u; m_usage_extra = 0; } - void add_up_stats(const Stats& stats) { m_stats.AddUp(stats); } - void add_usage_purged(long long up) { m_usage_purged += up; } - - DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs) - { - PathTokenizer pt(path, max_depth, parse_as_lfn); - - return find_path_tok(pt, 0, create_subdirs); - } - - DirState* find_dir(const std::string &dir, bool create_subdirs) - { - DsMap_i i = m_subdirs.find(dir); - - if (i != m_subdirs.end()) return & i->second; - - if (create_subdirs && m_depth < m_max_depth) return create_child(dir); - - return 0; - } - - void reset_stats() - { - m_stats.Reset(); - - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.reset_stats(); - } - } - - void upward_propagate_stats() - { - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.upward_propagate_stats(); - - m_stats.AddUp(i->second.m_stats); - } - - m_usage_extra += m_stats.m_BytesWritten; - } - - long long upward_propagate_usage_purged() - { - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - m_usage_purged += i->second.upward_propagate_usage_purged(); - } - m_usage -= m_usage_purged; - - long long ret = m_usage_purged; - m_usage_purged = 0; - return ret; - } - - void dump_recursively(const char *name) - { - printf("%*d %s usage=%lld usage_extra=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", - 2 + 2*m_depth, m_depth, name, m_usage, m_usage_extra, m_usage + m_usage_extra, - m_stats.m_NumIos, m_stats.m_Duration, m_stats.m_BytesHit, m_stats.m_BytesMissed, m_stats.m_BytesBypassed, m_stats.m_BytesWritten); - - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.dump_recursively(i->first.c_str()); - } - } + void dump_recursively(const char *name); }; } From f68b5105ea261798c0bd024bf4c2c5ffb8d66412 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 13 Mar 2024 12:02:11 -0700 Subject: [PATCH 19/43] Add new source file XrdPfcDirState.cc --- src/XrdPfc.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index 292c065a95c..2bb86a961f7 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -20,7 +20,7 @@ add_library( XrdPfc/XrdPfcTypes.hh XrdPfc/XrdPfc.cc XrdPfc/XrdPfc.hh XrdPfc/XrdPfcConfiguration.cc - XrdPfc/XrdPfcDirState.hh + XrdPfc/XrdPfcDirState.cc XrdPfc/XrdPfcDirState.hh XrdPfc/XrdPfcFPurgeState.cc XrdPfc/XrdPfcFPurgeState.hh XrdPfc/XrdPfcPurge.cc XrdPfc/XrdPfcCommand.cc From b7145fdb20adf892f10fa48dd61f9357b82f3109 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Wed, 27 Mar 2024 11:52:49 -0700 Subject: [PATCH 20/43] Prototype implementation of XrdPfc::ResourceMonitor -- the new monitoring and purge manager. - Some code for path/lfn parsing and namespace traversal has been moved into separate files and refactored to allow for reuse in initial scanning, purge traversal, and, eventually, in user plugins. - DirState now holds full information about stats/changes/deltas during the current monitoring period as well as summed up information about current usage of resources in / under this directory. Both Stats and Usages are split into "here" part (in this directory only) and "recursive_subdir" part (summed up information from all the subdirs). - ResourceMonintor holds queues where file open/close/purge events are reported and periodically processed. At the same time, Stats from currently opened files are extracted. This is used for periodic updates of the DirState tree. - High-level processing in ResourceMonitor still needs to be finalized and put together with purge-plugin API. DirState tree will be periodically exported (snapshotted) into vector form and provided for monitoring (as g-stream or as on-disk binary-data, convertable to JSON) and space management (purge plugin). --- src/XrdOss/XrdOssAt.hh | 2 + src/XrdPfc.cmake | 4 + src/XrdPfc/XrdPfc.cc | 116 +++++---- src/XrdPfc/XrdPfc.hh | 157 +----------- src/XrdPfc/XrdPfcCommand.cc | 16 +- src/XrdPfc/XrdPfcConfiguration.cc | 14 +- src/XrdPfc/XrdPfcDecision.hh | 4 +- src/XrdPfc/XrdPfcDirPurge.hh | 65 ----- src/XrdPfc/XrdPfcDirState.cc | 115 +++++---- src/XrdPfc/XrdPfcDirState.hh | 112 +++++++-- src/XrdPfc/XrdPfcFPurgeState.cc | 342 ++++++++++----------------- src/XrdPfc/XrdPfcFPurgeState.hh | 69 ++---- src/XrdPfc/XrdPfcFile.cc | 64 +++-- src/XrdPfc/XrdPfcFile.hh | 26 +- src/XrdPfc/XrdPfcFsTraversal.cc | 241 +++++++++++++++++++ src/XrdPfc/XrdPfcFsTraversal.hh | 80 +++++++ src/XrdPfc/XrdPfcIOFile.cc | 15 +- src/XrdPfc/XrdPfcIOFileBlock.cc | 17 +- src/XrdPfc/XrdPfcInfo.hh | 10 +- src/XrdPfc/XrdPfcPathParseTools.hh | 138 +++++++++++ src/XrdPfc/XrdPfcPurge.cc | 265 ++------------------- src/XrdPfc/XrdPfcPurgePin.hh | 8 +- src/XrdPfc/XrdPfcPurgeQuota.cc | 7 +- src/XrdPfc/XrdPfcResourceMonitor.cc | 355 ++++++++++++++++++++++++++++ src/XrdPfc/XrdPfcResourceMonitor.hh | 189 +++++++++++++++ src/XrdPfc/XrdPfcStats.hh | 112 +++++---- src/XrdPfc/XrdPfcTypes.hh | 5 +- 27 files changed, 1596 insertions(+), 952 deletions(-) delete mode 100644 src/XrdPfc/XrdPfcDirPurge.hh create mode 100644 src/XrdPfc/XrdPfcFsTraversal.cc create mode 100644 src/XrdPfc/XrdPfcFsTraversal.hh create mode 100644 src/XrdPfc/XrdPfcPathParseTools.hh create mode 100644 src/XrdPfc/XrdPfcResourceMonitor.cc create mode 100644 src/XrdPfc/XrdPfcResourceMonitor.hh diff --git a/src/XrdOss/XrdOssAt.hh b/src/XrdOss/XrdOssAt.hh index 6bcfb17e694..2c0dcfe2151 100644 --- a/src/XrdOss/XrdOssAt.hh +++ b/src/XrdOss/XrdOssAt.hh @@ -30,6 +30,8 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ +#include "XrdOuc/XrdOucEnv.hh" + #include #include diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index 2bb86a961f7..ce7784aa99d 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -23,6 +23,10 @@ add_library( XrdPfc/XrdPfcDirState.cc XrdPfc/XrdPfcDirState.hh XrdPfc/XrdPfcFPurgeState.cc XrdPfc/XrdPfcFPurgeState.hh XrdPfc/XrdPfcPurge.cc + XrdPfc/XrdPfcPurgePin.hh + XrdPfc/XrdPfcResourceMonitor.cc XrdPfc/XrdPfcResourceMonitor.hh + XrdPfc/XrdPfcPathParseTools.hh + XrdPfc/XrdPfcFsTraversal.cc XrdPfc/XrdPfcFsTraversal.hh XrdPfc/XrdPfcCommand.cc XrdPfc/XrdPfcFile.cc XrdPfc/XrdPfcFile.hh XrdPfc/XrdPfcFSctl.cc XrdPfc/XrdPfcFSctl.hh diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index b20e7f07f04..f2c390d9c13 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -41,17 +41,18 @@ #include "XrdPfcInfo.hh" #include "XrdPfcIOFile.hh" #include "XrdPfcIOFileBlock.hh" +#include "XrdPfcResourceMonitor.hh" using namespace XrdPfc; -Cache * Cache::m_instance = 0; - -XrdScheduler *Cache::schedP = 0; +Cache *Cache::m_instance = nullptr; +XrdScheduler *Cache::schedP = nullptr; void *ResourceMonitorHeartBeatThread(void*) { - Cache::GetInstance().ResourceMonitorHeartBeat(); + // Cache::GetInstance().ResourceMonitorHeartBeat(); + Cache::ResMon().heart_beat(); return 0; } @@ -99,7 +100,7 @@ XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, err.Say("Config Proxy file cache initialization failed."); return 0; } - err.Say("------ Proxy file cache initialization completed."); + err.Say("++++++ Proxy file cache initialization completed."); { pthread_t tid; @@ -116,6 +117,8 @@ XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, XrdSysThread::Run(&tid, ResourceMonitorHeartBeatThread, 0, 0, "XrdPfc ResourceMonitorHeartBeat"); + // XXXX This will be called from ResMon::heart_beat, as and if needed (maybe as XrdJob). + // Now on for testing of the FPurgeState traversal / old-style (default?) purge. XrdSysThread::Run(&tid, PurgeThread, 0, 0, "XrdPfc Purge"); } @@ -156,9 +159,10 @@ Cache &Cache::CreateInstance(XrdSysLogger *logger, XrdOucEnv *env) return *m_instance; } - Cache& Cache::GetInstance() { return *m_instance; } -const Cache& Cache::TheOne() { return *m_instance; } -const Configuration& Cache::Conf() { return m_instance->RefConfiguration(); } + Cache& Cache::GetInstance() { return *m_instance; } +const Cache& Cache::TheOne() { return *m_instance; } +const Configuration& Cache::Conf() { return m_instance->RefConfiguration(); } + ResourceMonitor& Cache::ResMon() { return m_instance->RefResMon(); } bool Cache::Decide(XrdOucCacheIO* io) { @@ -197,12 +201,7 @@ Cache::Cache(XrdSysLogger *logger, XrdOucEnv *env) : m_RAM_std_size(0), m_isClient(false), m_in_purge(false), - m_active_cond(0), - m_stats_n_purge_cond(0), - m_fs_state(0), - m_last_scan_duration(0), - m_last_purge_duration(0), - m_spt_state(SPTS_Idle) + m_active_cond(0) { // Default log level is Warning. m_trace->What = 2; @@ -504,7 +503,24 @@ void Cache::ReleaseFile(File* f, IO* io) dec_ref_cnt(f, true); } - +int Cache::CopyOutActiveStats(std::vector> &store) +{ + XrdSysCondVarHelper lock(&m_active_cond); + int n = 0; + for (ActiveMap_i i = m_active.begin(); i != m_active.end(); ++i) + { + File *f = i->second; + if (f != 0) { + store.emplace_back(std::make_pair(f->GetResMonToken(), f->DeltaStatsFromLastCall())); + ++n; + } + } + return n; +} + + +//============================================================================== + namespace { @@ -627,6 +643,7 @@ void Cache::dec_ref_cnt(File* f, bool high_debug) } } + bool finished_p = false; { XrdSysCondVarHelper lock(&m_active_cond); @@ -637,37 +654,40 @@ void Cache::dec_ref_cnt(File* f, bool high_debug) ActiveMap_i it = m_active.find(f->GetLocalPath()); m_active.erase(it); - m_closed_files_stats.insert(std::make_pair(f->GetLocalPath(), f->DeltaStatsFromLastCall())); + finished_p = true; + } + } - if (m_gstream) + if (finished_p) + { + if (m_gstream) + { + const Stats &st = f->RefStats(); + const Info::AStat *as = f->GetLastAccessStats(); + + char buf[4096]; + int len = snprintf(buf, 4096, "{\"event\":\"file_close\"," + "\"lfn\":\"%s\",\"size\":%lld,\"blk_size\":%d,\"n_blks\":%d,\"n_blks_done\":%d," + "\"access_cnt\":%lu,\"attach_t\":%lld,\"detach_t\":%lld,\"remotes\":%s," + "\"b_hit\":%lld,\"b_miss\":%lld,\"b_bypass\":%lld,\"n_cks_errs\":%d}", + f->GetLocalPath().c_str(), f->GetFileSize(), f->GetBlockSize(), + f->GetNBlocks(), f->GetNDownloadedBlocks(), + (unsigned long) f->GetAccessCnt(), (long long) as->AttachTime, (long long) as->DetachTime, + f->GetRemoteLocations().c_str(), + as->BytesHit, as->BytesMissed, as->BytesBypassed, st.m_NCksumErrors + ); + bool suc = false; + if (len < 4096) { - const Stats &st = f->RefStats(); - const Info::AStat *as = f->GetLastAccessStats(); - - char buf[4096]; - int len = snprintf(buf, 4096, "{\"event\":\"file_close\"," - "\"lfn\":\"%s\",\"size\":%lld,\"blk_size\":%d,\"n_blks\":%d,\"n_blks_done\":%d," - "\"access_cnt\":%lu,\"attach_t\":%lld,\"detach_t\":%lld,\"remotes\":%s," - "\"b_hit\":%lld,\"b_miss\":%lld,\"b_bypass\":%lld,\"n_cks_errs\":%d}", - f->GetLocalPath().c_str(), f->GetFileSize(), f->GetBlockSize(), - f->GetNBlocks(), f->GetNDownloadedBlocks(), - (unsigned long) f->GetAccessCnt(), (long long) as->AttachTime, (long long) as->DetachTime, - f->GetRemoteLocations().c_str(), - as->BytesHit, as->BytesMissed, as->BytesBypassed, st.m_NCksumErrors - ); - bool suc = false; - if (len < 4096) - { - suc = m_gstream->Insert(buf, len + 1); - } - if ( ! suc) - { - TRACE(Error, "Failed g-stream insertion of file_close record, len=" << len); - } + suc = m_gstream->Insert(buf, len + 1); + } + if ( ! suc) + { + TRACE(Error, "Failed g-stream insertion of file_close record, len=" << len); } - - delete f; } + + delete f; } } @@ -1132,6 +1152,7 @@ int Cache::Unlink(const char *curl) int Cache::UnlinkFile(const std::string& f_name, bool fail_if_open) { + static const char* trc_pfx = "UnlinkFile "; ActiveMap_i it; File *file = 0; { @@ -1143,7 +1164,7 @@ int Cache::UnlinkFile(const std::string& f_name, bool fail_if_open) { if (fail_if_open) { - TRACE(Info, "UnlinkCommon " << f_name << ", file currently open and force not requested - denying request"); + TRACE(Info, trc_pfx << f_name << ", file currently open and force not requested - denying request"); return -EBUSY; } @@ -1151,7 +1172,7 @@ int Cache::UnlinkFile(const std::string& f_name, bool fail_if_open) // Attach() with possible File::Open(). Ask for retry. if (it->second == 0) { - TRACE(Info, "UnlinkCommon " << f_name << ", an operation on this file is ongoing - denying request"); + TRACE(Info, trc_pfx << f_name << ", an operation on this file is ongoing - denying request"); return -EAGAIN; } @@ -1173,10 +1194,15 @@ int Cache::UnlinkFile(const std::string& f_name, bool fail_if_open) std::string i_name = f_name + Info::s_infoExtension; // Unlink file & cinfo + struct stat f_stat; + bool stat_ok = (m_oss->Stat(f_name.c_str(), &f_stat) == XrdOssOK); int f_ret = m_oss->Unlink(f_name.c_str()); int i_ret = m_oss->Unlink(i_name.c_str()); - TRACE(Debug, "UnlinkCommon " << f_name << ", f_ret=" << f_ret << ", i_ret=" << i_ret); + if (stat_ok) + m_res_mon->register_file_purge(f_name, f_stat.st_blocks * 512); + + TRACE(Debug, trc_pfx << f_name << ", f_ret=" << f_ret << ", i_ret=" << i_ret); { XrdSysCondVarHelper lock(&m_active_cond); diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index 94c44645771..75cefa27503 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -31,6 +31,7 @@ #include "XrdPfcFile.hh" #include "XrdPfcDecision.hh" +class XrdOss; class XrdOucStream; class XrdSysError; class XrdSysTrace; @@ -42,6 +43,7 @@ class File; class IO; class PurgePin; class DataFsState; +class ResourceMonitor; } @@ -132,128 +134,6 @@ struct TmpConfiguration {} }; -//============================================================================== - -struct SplitParser -{ - char *f_str; - const char *f_delim; - char *f_state; - bool f_first; - - SplitParser(const std::string &s, const char *d) : - f_str(strdup(s.c_str())), f_delim(d), f_state(0), f_first(true) - {} - ~SplitParser() { free(f_str); } - - char* get_token() - { - if (f_first) { f_first = false; return strtok_r(f_str, f_delim, &f_state); } - else { return strtok_r(0, f_delim, &f_state); } - } - - char* get_reminder_with_delim() - { - if (f_first) { return f_str; } - else { *(f_state - 1) = f_delim[0]; return f_state - 1; } - } - - char *get_reminder() - { - return f_first ? f_str : f_state; - } - - int fill_argv(std::vector &argv) - { - if (!f_first) return 0; - int dcnt = 0; { char *p = f_str; while (*p) { if (*(p++) == f_delim[0]) ++dcnt; } } - argv.reserve(dcnt + 1); - int argc = 0; - char *i = strtok_r(f_str, f_delim, &f_state); - while (i) - { - ++argc; - argv.push_back(i); - // printf(" arg %d : '%s'\n", argc, i); - i = strtok_r(0, f_delim, &f_state); - } - return argc; - } -}; - -struct PathTokenizer : private SplitParser -{ - std::vector m_dirs; - const char *m_reminder; - int m_n_dirs; - - PathTokenizer(const std::string &path, int max_depth, bool parse_as_lfn) : - SplitParser(path, "/"), - m_reminder (0), - m_n_dirs (0) - { - // If parse_as_lfn is true store final token into m_reminder, regardless of maxdepth. - // This assumes the last token is a file name (and full path is lfn, including the file name). - - m_dirs.reserve(max_depth); - - char *t = 0; - for (int i = 0; i < max_depth; ++i) - { - t = get_token(); - if (t == 0) break; - m_dirs.emplace_back(t); - } - if (parse_as_lfn && *get_reminder() == 0 && ! m_dirs.empty()) - { - m_reminder = m_dirs.back(); - m_dirs.pop_back(); - } - else - { - m_reminder = get_reminder(); - } - m_n_dirs = (int) m_dirs.size(); - } - - int get_n_dirs() - { - return m_n_dirs; - } - - const char *get_dir(int pos) - { - if (pos >= m_n_dirs) return 0; - return m_dirs[pos]; - } - - std::string make_path() - { - std::string res; - for (std::vector::iterator i = m_dirs.begin(); i != m_dirs.end(); ++i) - { - res += "/"; - res += *i; - } - if (m_reminder != 0) - { - res += "/"; - res += m_reminder; - } - return res; - } - - void deboog() - { - printf("PathTokenizer::deboog size=%d\n", m_n_dirs); - for (int i = 0; i < m_n_dirs; ++i) - { - printf(" %2d: %s\n", i, m_dirs[i]); - } - printf(" rem: %s\n", m_reminder); - } -}; - //============================================================================== // Cache @@ -335,16 +215,13 @@ public: static const Cache &TheOne(); static const Configuration &Conf(); + static ResourceMonitor &ResMon(); + //--------------------------------------------------------------------- //! Version check. //--------------------------------------------------------------------- static bool VCheck(XrdVersionInfo &urVersion) { return true; } - //--------------------------------------------------------------------- - //! Thread function checking resource usage periodically. - //--------------------------------------------------------------------- - void ResourceMonitorHeartBeat(); - //--------------------------------------------------------------------- //! Thread function invoked to scan and purge files from disk when needed. //--------------------------------------------------------------------- @@ -396,8 +273,12 @@ public: XrdSysError* GetLog() { return &m_log; } XrdSysTrace* GetTrace() { return m_trace; } + ResourceMonitor& RefResMon() { return *m_res_mon; } XrdXrootdGStream* GetGStream() { return m_gstream; } + int CopyOutActiveStats(std::vector> &store); + + void ExecuteCommandUrl(const std::string& command_url); static XrdScheduler *schedP; @@ -412,7 +293,7 @@ private: bool cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name); - static Cache *m_instance; //!< this object + static Cache *m_instance; //!< this object XrdOucEnv *m_env; //!< environment passed in at creation XrdSysError m_log; //!< XrdPfc namespace logger @@ -423,6 +304,8 @@ private: XrdXrootdGStream *m_gstream; + ResourceMonitor *m_res_mon; + std::vector m_decisionpoints; //!< decision plugins XrdPfc::PurgePin* m_purge_pin; //!< purge plugin @@ -454,12 +337,9 @@ private: // active map, purge delay set typedef std::map ActiveMap_t; typedef ActiveMap_t::iterator ActiveMap_i; - typedef std::multimap StatsMMap_t; - typedef StatsMMap_t::iterator StatsMMap_i; typedef std::set FNameSet_t; ActiveMap_t m_active; //!< Map of currently active / open files. - StatsMMap_t m_closed_files_stats; FNameSet_t m_purge_delay_set; bool m_in_purge; XrdSysCondVar m_active_cond; //!< Cond-var protecting active file data structures. @@ -472,21 +352,6 @@ private: // prefetching typedef std::vector PrefetchList; PrefetchList m_prefetchList; - - //--------------------------------------------------------------------------- - // Statistics, heart-beat, scan-and-purge - - enum ScanAndPurgeThreadState_e { SPTS_Idle, SPTS_Scan, SPTS_Purge, SPTS_Done }; - - XrdSysCondVar m_stats_n_purge_cond; //!< communication between heart-beat and scan-purge threads - - DataFsState *m_fs_state; //!< directory state for access / usage info and quotas - - int m_last_scan_duration; - int m_last_purge_duration; - ScanAndPurgeThreadState_e m_spt_state; - - void copy_out_active_stats_and_update_data_fs_state(); }; } diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index ba03e716e19..a483063fbf3 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -19,6 +19,8 @@ #include "XrdPfcInfo.hh" #include "XrdPfc.hh" #include "XrdPfcTrace.hh" +#include "XrdPfcPathParseTools.hh" +#include "XrdPfcResourceMonitor.hh" #include "XrdOfs/XrdOfsConfigPI.hh" #include "XrdOss/XrdOss.hh" @@ -245,6 +247,15 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) myInfo.Write(myInfoFile, cinfo_path.c_str()); + // Fake last modified time to the last access_time + { + time_t last_detach; + myInfo.GetLatestDetachTime(last_detach); + struct timespec acc_mod_time[2] = { {last_detach, UTIME_OMIT}, {last_detach, 0} }; + + futimens(myInfoFile->getFD(), acc_mod_time); + } + myInfoFile->Close(); delete myInfoFile; myFile->Close(); delete myFile; @@ -256,11 +267,10 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) m_writeQ.writes_between_purges += file_size; } { - XrdSysCondVarHelper lock(&m_active_cond); - + int token = m_res_mon->register_file_open(file_path, time_now); XrdPfc::Stats stats; stats.m_BytesWritten = file_size; - m_closed_files_stats.insert({file_path, stats}); + m_res_mon->register_file_close(token, stats, time(0)); } } } diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index 74885a4fc78..022a0fcb38d 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -2,6 +2,7 @@ #include "XrdPfcTrace.hh" #include "XrdPfcInfo.hh" +#include "XrdPfcResourceMonitor.hh" #include "XrdPfcPurgePin.hh" #include "XrdOss/XrdOss.hh" @@ -611,9 +612,18 @@ bool Cache::Config(const char *config_filename, const char *parameters) m_gstream = (XrdXrootdGStream*) m_env->GetPtr("pfc.gStream*"); - m_log.Say("Config Proxy File Cache g-stream has", m_gstream ? "" : " NOT", " been configured via xrootd.monitor directive"); + m_log.Say(" pfc g-stream has", m_gstream ? "" : " NOT", " been configured via xrootd.monitor directive\n"); - m_log.Say("------ Proxy File Cache configuration parsing ", aOK ? "completed" : "failed"); + // Create the ResourceMonitor object and perform initial scan. + if (aOK) + { + m_log.Say("-----> Proxy file cache performing initial directory scan"); + m_res_mon = new ResourceMonitor(*m_oss); + aOK = m_res_mon->perform_initial_scan(); + m_log.Say("-----> Proxy File Cache initial directory scan finished"); + } + + m_log.Say("=====> Proxy file cache configuration parsing ", aOK ? "completed" : "failed"); if (ofsCfg) delete ofsCfg; diff --git a/src/XrdPfc/XrdPfcDecision.hh b/src/XrdPfc/XrdPfcDecision.hh index 3f62c87764b..9e85f8651fb 100644 --- a/src/XrdPfc/XrdPfcDecision.hh +++ b/src/XrdPfc/XrdPfcDecision.hh @@ -19,10 +19,8 @@ //---------------------------------------------------------------------------------- #include -#include -#include -#include "XrdOss/XrdOss.hh" +class XrdOss; class XrdSysError; namespace XrdPfc diff --git a/src/XrdPfc/XrdPfcDirPurge.hh b/src/XrdPfc/XrdPfcDirPurge.hh deleted file mode 100644 index 67e48e42062..00000000000 --- a/src/XrdPfc/XrdPfcDirPurge.hh +++ /dev/null @@ -1,65 +0,0 @@ -#ifndef __XRDPFC_PURGEPLG_HH__ -#define __XRDPFC_PURGEPLG_HH__ - -#include -#include - -namespace XrdPfc -{ -class DirState; - -//---------------------------------------------------------------------------- -//! Base class for reguesting directory space to obtain. -//---------------------------------------------------------------------------- -class PurgePin -{ -public: - struct DirInfo - { - std::string path; - long long nBytesQuota{0}; - DirState *dirState{nullptr}; // currently cached and shared within the purge thread - long long nBytesToRecover{0}; - }; - - typedef std::vector list_t; - typedef list_t::iterator list_i; - -protected: - list_t m_list; - -public: - virtual ~PurgePin() {} - - //--------------------------------------------------------------------- - //! Provide erase information from directory statistics - //! - //! @param & XrdPfcDirState - //! - //! @return total number of bytes - //--------------------------------------------------------------------- - virtual long long GetBytesToRecover(DirState *) = 0; - - //------------------------------------------------------------------------------ - //! Parse configuration arguments. - //! - //! @param params configuration parameters - //! - //! @return status of configuration - //------------------------------------------------------------------------------ - virtual bool ConfigPurgePin(const char* params) // ?? AMT should this be abstract - { - (void) params; - return true; - } - - //----------------------------------------------- - //! - //! Get quotas for the given paths. Used in the XrdPfc:Cache::Purge() thread. - //! - //------------------------------------------------------------------------------ - list_t &refDirInfos() { return m_list; } -}; -} - -#endif diff --git a/src/XrdPfc/XrdPfcDirState.cc b/src/XrdPfc/XrdPfcDirState.cc index dc90d9d05b4..ed59a3de67c 100644 --- a/src/XrdPfc/XrdPfcDirState.cc +++ b/src/XrdPfc/XrdPfcDirState.cc @@ -1,34 +1,36 @@ #include "XrdPfcDirState.hh" -#include "XrdPfcStats.hh" +#include "XrdPfcPathParseTools.hh" #include namespace XrdPfc { + //---------------------------------------------------------------------------- //! Constructor -//! @param int max subdir depth //---------------------------------------------------------------------------- -DirState::DirState(int max_depth) : m_parent(0), m_depth(0), m_max_depth(max_depth) -{ - // init(); -} +DirState::DirState() : m_parent(0), m_depth(0) +{} //---------------------------------------------------------------------------- //! Constructor //! @param DirState parent directory //---------------------------------------------------------------------------- -DirState::DirState(DirState *parent) : m_parent(parent), m_depth(m_parent->m_depth + 1), m_max_depth(m_parent->m_max_depth) -{ - // init(); -} -/* -void DirState::init() -{ - m_usage = 0; - m_usage_extra = 0; - m_usage_purged = 0; -}*/ +DirState::DirState(DirState *parent) : + m_parent(parent), + m_depth(m_parent->m_depth + 1) +{} + +//---------------------------------------------------------------------------- +//! Constructor +//! @param parent parent DirState object +//! @param dname name of this directory only, no slashes, no extras. +//---------------------------------------------------------------------------- +DirState::DirState(DirState *parent, const std::string& dname) : + m_parent(parent), + m_dir_name(dname), + m_depth(m_parent->m_depth + 1) +{} //---------------------------------------------------------------------------- //! Internal function called from find_dir or find_path_tok @@ -36,11 +38,10 @@ void DirState::init() //---------------------------------------------------------------------------- DirState* DirState::create_child(const std::string &dir) { - std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this))); - return & ir.first->second; + std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this, dir))); + return & ir.first->second; } - //---------------------------------------------------------------------------- //! Internal function called from find_path //! @param dir subdir name @@ -57,7 +58,7 @@ DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdir { ds = & i->second; } - if (create_subdirs && m_depth < m_max_depth) + if (create_subdirs) { ds = create_child(pt.m_dirs[pos]); } @@ -66,15 +67,14 @@ DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdir return 0; } - - //---------------------------------------------------------------------------- -//! Recursive function to find DireState with given absolute dir path -//! @param dir subdir name -//! @param max_depth +//! Recursive function to find DirState with given absolute dir path +//! @param path full path to parse +//! @param max_depth directory depth to which to descend (value < 0 means full descent) //! @param parse_as_lfn //! @param create_subdirs -DirState* DirState::find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs) +DirState* DirState::find_path(const std::string &path, int max_depth, bool parse_as_lfn, + bool create_subdirs) { PathTokenizer pt(path, max_depth, parse_as_lfn); @@ -82,15 +82,18 @@ DirState* DirState::find_path(const std::string &path, int max_depth, bool parse } //---------------------------------------------------------------------------- -//! Non recursive function +//! Non recursive function to find an entry in this directory only. //! @param dir subdir name @param bool create the subdir in this DirsStat -DirState* DirState::find_dir(const std::string &dir, bool create_subdirs) +//! @param create_subdirs if true and the dir is not found, a new DirState +//! child is created +DirState* DirState::find_dir(const std::string &dir, + bool create_subdirs) { DsMap_i i = m_subdirs.find(dir); if (i != m_subdirs.end()) return & i->second; - if (create_subdirs && m_depth < m_max_depth) return create_child(dir); + if (create_subdirs) return create_child(dir); return 0; } @@ -101,7 +104,8 @@ DirState* DirState::find_dir(const std::string &dir, bool create_subdirs) //---------------------------------------------------------------------------- void DirState::reset_stats() { - m_stats.Reset(); + m_here_stats.Reset(); + m_recursive_subdirs_stats.Reset(); for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) { @@ -119,10 +123,9 @@ void DirState::upward_propagate_stats() { i->second.upward_propagate_stats(); - m_stats.AddUp(i->second.m_stats); + // XXXXX m_stats.AddUp(i->second.m_stats); + // fix these for here_stats, there_stats } - - m_usage_extra += m_stats.m_BytesWritten; } //---------------------------------------------------------------------------- @@ -131,6 +134,8 @@ void DirState::upward_propagate_stats() //---------------------------------------------------------------------------- long long DirState::upward_propagate_usage_purged() { + // XXXXX what's with this? +/* for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) { m_usage_purged += i->second.upward_propagate_usage_purged(); @@ -140,51 +145,43 @@ long long DirState::upward_propagate_usage_purged() long long ret = m_usage_purged; m_usage_purged = 0; return ret; +*/ + return 0; } //---------------------------------------------------------------------------- //! Recursive print of statistics. Called if defined in pfc configuration. //! //---------------------------------------------------------------------------- -void DirState::dump_recursively(const char *name) +void DirState::dump_recursively(const char *name, int max_depth) { printf("%*d %s usage=%lld usage_extra=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", - 2 + 2*m_depth, m_depth, name, m_usage, m_usage_extra, m_usage + m_usage_extra, - m_stats.m_NumIos, m_stats.m_Duration, m_stats.m_BytesHit, m_stats.m_BytesMissed, m_stats.m_BytesBypassed, m_stats.m_BytesWritten); + 2 + 2*m_depth, m_depth, name, + // XXXXX decide what goes here + // m_usage, m_usage_extra, m_usage + m_usage_extra, + 0ll, 0ll, 0ll, + // XXXXX here_stats or sum up? + m_here_stats.m_NumIos, m_here_stats.m_Duration, + m_here_stats.m_BytesHit, m_here_stats.m_BytesMissed, m_here_stats.m_BytesBypassed, + m_here_stats.m_BytesWritten); + + if (m_depth >= max_depth) + return; for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) { - i->second.dump_recursively(i->first.c_str()); + i->second.dump_recursively(i->first.c_str(), max_depth); } } -//---------------------------------------------------------------------------- -//! Set read usage -//! -//---------------------------------------------------------------------------- -void DirState::set_usage(long long u) -{ - m_usage = u; - m_usage_extra = 0; -} - - //---------------------------------------------------------------------------- //! Add to temporary Stat obj //! //---------------------------------------------------------------------------- void DirState::add_up_stats(const Stats& stats) { - m_stats.AddUp(stats); -} - -//---------------------------------------------------------------------------- -//! Accumulate usage from purged files -//! -//---------------------------------------------------------------------------- -void DirState::add_usage_purged(long long up) -{ - m_usage_purged += up; + m_here_stats.AddUp(stats); + // XXXX propagate to parent done at the end. } } // end namespace diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh index 4989bd5473f..e9e30be8c56 100644 --- a/src/XrdPfc/XrdPfcDirState.hh +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -1,30 +1,70 @@ #ifndef __XRDPFC_DIRSTATE_HH__ #define __XRDPFC_DIRSTATE_HH__ -#include "XrdPfc.hh" -#include "XrdPfcInfo.hh" #include "XrdPfcStats.hh" + +#include #include #include -using namespace XrdPfc; - namespace XrdPfc { +class PathTokenizer; + +//============================================================================== +// Manifest: +// 1. class DirState -- state of a directory, including current delta-stats +// 2. class DataFSState -- manager of the DirState tree, starting from root (as in "/"). + + //============================================================================== // DirState //============================================================================== +struct DirUsage +{ + time_t m_last_open_time = 0; + time_t m_last_close_time = 0; + long long m_bytes_on_disk = 0; + int m_num_files_open = 0; + int m_num_files = 0; + int m_num_subdirs = 0; +}; + class DirState { - DirState *m_parent; +public: + DirState *m_parent = nullptr; + std::string m_dir_name; - Stats m_stats; // access stats from client reads in this directory (and subdirs) + DirStats m_here_stats; + DirStats m_recursive_subdirs_stats; - long long m_usage{0}; // collected / measured during purge traversal - long long m_usage_extra{0}; // collected from write events in this directory and subdirs - long long m_usage_purged{0}; // amount of data purged from this directory (and subdirectories for leaf nodes) + DirUsage m_here_usage; + DirUsage m_recursive_subdir_usage; + + // XXX int m_possible_discrepancy; // num detected possible inconsistencies. here, subdirs? + + // flag - can potentially be inaccurate -- plus timestamp of it (min or max, if several for subdirs)? + + // + the same kind for resursive sums of all subdirs (but not in this dir). + // Thus, to get recursive totals of here+all_subdirs, one need to add them up. + // Have extra members or define an intermediate structure? + + // Do we need running averages of these, too, not just traffic? + // Snapshot should be fine, no? + + // Do we need all-time stats? Files/dirs created, deleted; files opened/closed; + // Well, those would be like Stats-running-average-infinity, just adding stuff in. + + // min/max open (or close ... or both?) time-stamps + + // Do we need string name? Probably yes, if we want to construct PFN from a given + // inner node upwards. Also, in this case, is it really the best idea to have + // map as daughter container? It keeps them sorted for export :) + + // m_stats should be separated by "here" and "daughters_recursively", too. // begin purge traversal usage \_ so we can have a good estimate of what came in during the traversal // end purge traversal usage / (should be small, presumably) @@ -32,7 +72,6 @@ class DirState // quota info, enabled? int m_depth; - int m_max_depth; // XXXX Do we need this? Should it be passed in to find functions? bool m_stat_report; // not used yet - storing of stats requested typedef std::map DsMap_t; @@ -48,17 +87,15 @@ class DirState public: - DirState(int max_depth); + DirState(); DirState(DirState *parent); - DirState* get_parent() { return m_parent; } + DirState(DirState *parent, const std::string& dname); - long long get_usage() { return m_usage; } + DirState* get_parent() { return m_parent; } - void set_usage(long long u); void add_up_stats(const Stats& stats); - void add_usage_purged(long long up); DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs); @@ -70,8 +107,51 @@ public: long long upward_propagate_usage_purged(); - void dump_recursively(const char *name); + void dump_recursively(const char *name, int max_depth); +}; + + +//============================================================================== +// DataFsState +//============================================================================== + +class DataFsState +{ + DirState m_root; + time_t m_prev_time; + +public: + DataFsState() : + m_root (), + m_prev_time (time(0)) + {} + + DirState* get_root() { return & m_root; } + + DirState* find_dirstate_for_lfn(const std::string& lfn) + { + return m_root.find_path(lfn, -1, true, true); + } + + void reset_stats() { m_root.reset_stats(); } + void upward_propagate_stats() { m_root.upward_propagate_stats(); } + void upward_propagate_usage_purged() { m_root.upward_propagate_usage_purged(); } + + void dump_recursively(int max_depth) + { + if (max_depth < 0) + max_depth = 4096; + time_t now = time(0); + + printf("DataFsState::dump_recursively epoch = %lld delta_t = %lld, max_dump_depth = %d\n", + (long long) now, (long long) (now - m_prev_time), max_depth); + + m_prev_time = now; + + m_root.dump_recursively("root", max_depth); + } }; + } #endif diff --git a/src/XrdPfc/XrdPfcFPurgeState.cc b/src/XrdPfc/XrdPfcFPurgeState.cc index 91d28ab67fe..b71da82b811 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.cc +++ b/src/XrdPfc/XrdPfcFPurgeState.cc @@ -1,11 +1,13 @@ #include "XrdPfcFPurgeState.hh" +#include "XrdPfcFsTraversal.hh" +#include "XrdPfcInfo.hh" +#include "XrdPfc.hh" +#include "XrdPfcTrace.hh" -#include "XrdPfcDirState.hh" -#include "XrdPfcFPurgeState.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucUtils.hh" +#include "XrdOss/XrdOss.hh" #include "XrdOss/XrdOssAt.hh" -#include "XrdSys/XrdSysTrace.hh" // Temporary, extensive purge tracing // #define TRACE_PURGE(x) TRACE(Debug, x) @@ -14,266 +16,167 @@ using namespace XrdPfc; -namespace XrdPfc -{ - -XrdSysTrace* GetTrace() +namespace { - // needed for logging macros - return Cache::GetInstance().GetTrace(); + XrdSysTrace* GetTrace() { return Cache::GetInstance().GetTrace(); } } +const char *FPurgeState::m_traceID = "Purge"; //---------------------------------------------------------------------------- //! Constructor. //---------------------------------------------------------------------------- FPurgeState::FPurgeState(long long iNBytesReq, XrdOss &oss) : - m_nBytesReq(iNBytesReq), m_nBytesAccum(0), m_nBytesTotal(0), m_tMinTimeStamp(0), m_tMinUVKeepTimeStamp(0), - m_oss_at(oss), - m_dir_state(0), m_dir_level(0), - m_max_dir_level_for_stat_collection(Cache::Conf().m_dirStatsStoreDepth), - m_info_ext(XrdPfc::Info::s_infoExtension), - m_info_ext_len(strlen(XrdPfc::Info::s_infoExtension)), - m_trace(Cache::GetInstance().GetTrace()) -{ - m_current_path.reserve(256); - m_dir_names_stack.reserve(32); - m_dir_usage_stack.reserve(m_max_dir_level_for_stat_collection + 1); -} - -//---------------------------------------------------------------------------- -//! Initiate DirState for traversal. -//! @param directory statistics -//! @param path relative to caching proxy OSS -//---------------------------------------------------------------------------- -void FPurgeState::begin_traversal(DirState *root, const char *root_path) -{ - m_dir_state = root; - m_dir_level = 0; - m_current_path = std::string(root_path); - m_dir_usage_stack.push_back(0); - - TRACE_PURGE("FPurgeState::begin_traversal cur_path '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", level=" << m_dir_level); -} - -//---------------------------------------------------------------------------- -//! Finalize DirState at the end of traversal. -//---------------------------------------------------------------------------- -void FPurgeState::end_traversal() -{ - TRACE_PURGE("FPurgeState::end_traversal reporting for '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", nBytesTotal=" << m_nBytesTotal << ", level=" << m_dir_level); - - m_dir_state->set_usage(m_dir_usage_stack.back()); - - m_dir_state = 0; -} - -//---------------------------------------------------------------------------- -//! Move to child directory. -//! @param relative name of subdirectory -//---------------------------------------------------------------------------- -void FPurgeState::cd_down(const std::string &dir_name) + m_oss(oss), + m_nBytesReq(iNBytesReq), m_nBytesAccum(0), m_nBytesTotal(0), + m_tMinTimeStamp(0), m_tMinUVKeepTimeStamp(0) { - ++m_dir_level; - - if (m_dir_level <= m_max_dir_level_for_stat_collection) - { - m_dir_usage_stack.push_back(0); - m_dir_state = m_dir_state->find_dir(dir_name, true); - } - - m_dir_names_stack.push_back(dir_name); - m_current_path.append(dir_name); - m_current_path.append("/"); -} - -//---------------------------------------------------------------------------- -//! Move to parent directory and set disk usage. -//---------------------------------------------------------------------------- -void FPurgeState::cd_up() -{ - if (m_dir_level <= m_max_dir_level_for_stat_collection) - { - long long tail = m_dir_usage_stack.back(); - m_dir_usage_stack.pop_back(); - - TRACE_PURGE("FPurgeState::cd_up reporting for '" << m_current_path << "', usage=" << tail << ", level=" << m_dir_level); - - m_dir_state->set_usage(tail); - m_dir_state = m_dir_state->get_parent(); - - m_dir_usage_stack.back() += tail; - } - - // remove trailing / and last dir but keep the new trailing / in place. - m_current_path.erase(m_current_path.find_last_of('/', m_current_path.size() - 2) + 1); - m_dir_names_stack.pop_back(); - - --m_dir_level; + // XXXX init traversal, pass oss? Note, do NOT use DirState (it would have to come from elsewhere) + // well, depends how it's going to be called, eventually } //---------------------------------------------------------------------------- //! Move remaing entires to the member map. -//! This is used for cold files and for files collected from purge plugin. +//! This is used for cold files and for files collected from purge plugin (really?). //---------------------------------------------------------------------------- void FPurgeState::MoveListEntriesToMap() { - for (list_i i = m_flist.begin(); i != m_flist.end(); ++i) - { - m_fmap.insert(std::make_pair(i->time, *i)); - } - m_flist.clear(); + for (list_i i = m_flist.begin(); i != m_flist.end(); ++i) + { + m_fmap.insert(std::make_pair(i->time, *i)); + } + m_flist.clear(); } //---------------------------------------------------------------------------- //! Open info file. Look at the UV stams and last access time. //! Store the file in sorted map or in a list.s -//! @param name of the cached file +//! @param fname name of cache-info file //! @param Info object //! @param stat of the given file //! //---------------------------------------------------------------------------- -void FPurgeState::CheckFile(const char *fname, Info &info, struct stat &fstat /*, XrdOssDF *iOssDF*/) +void FPurgeState::CheckFile(const FsTraversal &fst, const char *fname, Info &info, struct stat &fstat) { - static const char *trc_pfx = "FPurgeState::CheckFile "; + static const char *trc_pfx = "FPurgeState::CheckFile "; - long long nbytes = info.GetNDownloadedBytes(); - time_t atime; - if (!info.GetLatestDetachTime(atime)) - { - // cinfo file does not contain any known accesses, use fstat.mtime instead. - TRACE(Debug, trc_pfx << "could not get access time for " << m_current_path << fname << ", using mtime from stat instead."); - atime = fstat.st_mtime; - } - // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime); + long long nbytes = info.GetNDownloadedBytes(); + time_t atime; + if (!info.GetLatestDetachTime(atime)) + { + // cinfo file does not contain any known accesses, use fstat.mtime instead. + TRACE(Debug, trc_pfx << "could not get access time for " << fst.m_current_path << fname << ", using mtime from stat instead."); + atime = fstat.st_mtime; + } + // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime); - m_nBytesTotal += nbytes; + m_nBytesTotal += nbytes; - m_dir_usage_stack.back() += nbytes; + // XXXX Should remove aged-out files here ... but I have trouble getting + // the DirState and purge report set up consistently. + // Need some serious code reorganization here. + // Biggest problem is maintaining overall state a traversal state consistently. + // Sigh. - // XXXX Should remove aged-out files here ... but I have trouble getting - // the DirState and purge report set up consistently. - // Need some serious code reorganization here. - // Biggest problem is maintaining overall state a traversal state consistently. - // Sigh. + // This can be done right with transactional DirState. Also for uvkeep, it seems. - // In first two cases we lie about FS time (set to 0) to get them all removed early. - // The age-based purge atime would also be good as there should be nothing - // before that time in the map anyway. - // But we use 0 as a test in purge loop to make sure we continue even if enough - // disk-space has been freed. + // In first two cases we lie about PurgeCandidate time (set to 0) to get them all removed early. + // The age-based purge atime would also be good as there should be nothing + // before that time in the map anyway. + // But we use 0 as a test in purge loop to make sure we continue even if enough + // disk-space has been freed. - if (m_tMinTimeStamp > 0 && atime < m_tMinTimeStamp) - { - m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); - m_nBytesAccum += nbytes; - } - else if (m_tMinUVKeepTimeStamp > 0 && - Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) && - info.GetNoCkSumTimeForUVKeep() < m_tMinUVKeepTimeStamp) - { - m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); - m_nBytesAccum += nbytes; - } - else if (m_nBytesAccum < m_nBytesReq || (!m_fmap.empty() && atime < m_fmap.rbegin()->first)) - { - m_fmap.insert(std::make_pair(atime, FS(m_current_path, fname, nbytes, atime, m_dir_state))); - m_nBytesAccum += nbytes; + if (m_tMinTimeStamp > 0 && atime < m_tMinTimeStamp) + { + m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nbytes, 0)); + m_nBytesAccum += nbytes; + } + else if (m_tMinUVKeepTimeStamp > 0 && + Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) && + info.GetNoCkSumTimeForUVKeep() < m_tMinUVKeepTimeStamp) + { + m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nbytes, 0)); + m_nBytesAccum += nbytes; + } + else if (m_nBytesAccum < m_nBytesReq || (!m_fmap.empty() && atime < m_fmap.rbegin()->first)) + { + m_fmap.insert(std::make_pair(atime, PurgeCandidate(fst.m_current_path, fname, nbytes, atime))); + m_nBytesAccum += nbytes; - // remove newest files from map if necessary - while (!m_fmap.empty() && m_nBytesAccum - m_fmap.rbegin()->second.nBytes >= m_nBytesReq) - { - m_nBytesAccum -= m_fmap.rbegin()->second.nBytes; - m_fmap.erase(--(m_fmap.rbegin().base())); - } - } + // remove newest files from map if necessary + while (!m_fmap.empty() && m_nBytesAccum - m_fmap.rbegin()->second.nBytes >= m_nBytesReq) + { + m_nBytesAccum -= m_fmap.rbegin()->second.nBytes; + m_fmap.erase(--(m_fmap.rbegin().base())); + } + } } -//---------------------------------------------------------------------------- -//! Recursively traverse directory. Build DirState statistics and sort files. -//! @param XrdOssDF handle -//---------------------------------------------------------------------------- -void FPurgeState::TraverseNamespace(XrdOssDF *iOssDF) +void FPurgeState::ProcessDirAndRecurse(FsTraversal &fst) { - static const char *trc_pfx = "FPurgeState::TraverseNamespace "; + static const char *trc_pfx = "FPurgeState::ProcessDirAndRecurse "; - char fname[256]; - struct stat fstat; - XrdOucEnv env; - - TRACE_PURGE("Starting to read dir [" << m_current_path << "], iOssDF->getFD()=" << iOssDF->getFD() << "."); - - iOssDF->StatRet(&fstat); - - while (true) - { - int rc = iOssDF->Readdir(fname, 256); + for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it) + { + // Check if the file is currently opened / purge-protected is done before unlinking of the file. + const std::string &f_name = it->first; + const std::string i_name = f_name + Info::s_infoExtension; - if (rc == -ENOENT) - { - TRACE_PURGE(" Skipping ENOENT dir entry [" << fname << "]."); - continue; - } - if (rc != XrdOssOK) - { - TRACE(Error, trc_pfx << "Readdir error at " << m_current_path << ", err " << XrdSysE2T(-rc) << "."); - break; - } + XrdOssDF *fh = nullptr; + struct stat fstat; + Info cinfo(GetTrace()); - TRACE_PURGE(" Readdir [" << fname << "]"); + // XXX Note, the initial scan now uses stat information only! - if (fname[0] == 0) - { - TRACE_PURGE(" Finished reading dir [" << m_current_path << "]. Break loop."); - break; - } - if (fname[0] == '.' && (fname[1] == 0 || (fname[1] == '.' && fname[2] == 0))) - { - TRACE_PURGE(" Skipping here or parent dir [" << fname << "]. Continue loop."); - continue; - } + if (fst.open_at_ro(i_name.c_str(), fh) == XrdOssOK && + cinfo.Read(fh, fst.m_current_path.c_str(), i_name.c_str())) + { + CheckFile(fst, i_name.c_str(), cinfo, fstat); + } + else + { + TRACE(Warning, trc_pfx << "can't open or read " << fst.m_current_path << i_name << ", err " << XrdSysE2T(errno) << "; purging."); + fst.unlink_at(i_name.c_str()); + fst.unlink_at(f_name.c_str()); + // generate purge event or not? or just flag possible discrepancy? + // should this really be done in some other consistency-check traversal? + } - size_t fname_len = strlen(fname); - XrdOssDF *dfh = 0; + // XXX ? What do we do with the data-only / cinfo only ? + // Protected top-directories are skipped. + } - if (S_ISDIR(fstat.st_mode)) - { - if (m_oss_at.Opendir(*iOssDF, fname, env, dfh) == XrdOssOK) - { - cd_down(fname); - TRACE_PURGE(" cd_down -> [" << m_current_path << "]."); - TraverseNamespace(dfh); - cd_up(); - TRACE_PURGE(" cd_up -> [" << m_current_path << "]."); - } - else - TRACE(Warning, trc_pfx << "could not opendir [" << m_current_path << fname << "], " << XrdSysE2T(errno)); - } - else if (fname_len > m_info_ext_len && strncmp(&fname[fname_len - m_info_ext_len], m_info_ext, m_info_ext_len) == 0) - { - // Check if the file is currently opened / purge-protected is done before unlinking of the file. + std::vector dirs; + dirs.swap(fst.m_current_dirs); + for (auto &dname : dirs) + { + if (fst.cd_down(dname)) + { + ProcessDirAndRecurse(fst); + fst.cd_up(); + } + } +} - Info cinfo(m_trace); +bool FPurgeState::TraverseNamespace(const char *root_path) +{ + bool success_p = true; - if (m_oss_at.OpenRO(*iOssDF, fname, env, dfh) == XrdOssOK && cinfo.Read(dfh, m_current_path.c_str(), fname)) - { - CheckFile(fname, cinfo, fstat); - } - else - { - TRACE(Warning, trc_pfx << "can't open or read " << m_current_path << fname << ", err " << XrdSysE2T(errno) << "; purging."); - m_oss_at.Unlink(*iOssDF, fname); - fname[fname_len - m_info_ext_len] = 0; - m_oss_at.Unlink(*iOssDF, fname); - } - } - else // XXXX devel debug only, to be removed - { - TRACE_PURGE(" Ignoring [" << fname << "], not a dir or cinfo."); - } + FsTraversal fst(m_oss); + fst.m_protected_top_dirs.insert("pfc-stats"); // XXXX This should come from config. Also: N2N? + // Also ... this onoly applies to /, not any root_path + if (fst.begin_traversal(root_path)) + { + ProcessDirAndRecurse(fst); + } + else + { + // Fail startup, can't open /. + success_p = false; + } + fst.end_traversal(); - delete dfh; - } + return success_p; } /* @@ -301,6 +204,3 @@ void FPurgeState::UnlinkInfoAndData(const char *fname, long long nbytes, XrdOssD m_oss_at.Unlink(*iOssDF, fname); } */ - -} // namespace XrdPfc - diff --git a/src/XrdPfc/XrdPfcFPurgeState.hh b/src/XrdPfc/XrdPfcFPurgeState.hh index 413e8435dc3..e2b8a39dd90 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.hh +++ b/src/XrdPfc/XrdPfcFPurgeState.hh @@ -1,18 +1,20 @@ #ifndef __XRDPFC_FPURGESTATE_HH__ #define __XRDPFC_FPURGESTATE_HH__ -#include "XrdPfc.hh" -#include "XrdPfcStats.hh" -#include "XrdPfcDirState.hh" -#include "XrdPfcFPurgeState.hh" -#include "XrdPfcTrace.hh" -#include "XrdOss/XrdOssAt.hh" -#include "XrdSys/XrdSysTrace.hh" -#include "XrdOuc/XrdOucUtils.hh" -#include "XrdOuc/XrdOucEnv.hh" +#include +#include +#include +#include + +#include + +class XrdOss; namespace XrdPfc { +class Info; +class FsTraversal; + //============================================================================== // FPurgeState //============================================================================== @@ -20,47 +22,33 @@ namespace XrdPfc { class FPurgeState { public: - struct FS + struct PurgeCandidate // unknown meaning, "file that is candidate for purge", PurgeCandidate would be better. { std::string path; long long nBytes; time_t time; - DirState *dirState; - FS(const std::string &dname, const char *fname, long long n, time_t t, DirState *ds) : - path(dname + fname), nBytes(n), time(t), dirState(ds) + PurgeCandidate(const std::string &dname, const char *fname, long long n, time_t t) : + path(dname + fname), nBytes(n), time(t) {} }; - typedef std::list list_t; - typedef list_t::iterator list_i; - typedef std::multimap map_t; - typedef map_t::iterator map_i; + using list_t = std::list; + using list_i = list_t::iterator; + using map_t = std::multimap; + using map_i = map_t::iterator; private: + XrdOss &m_oss; + long long m_nBytesReq; long long m_nBytesAccum; long long m_nBytesTotal; time_t m_tMinTimeStamp; time_t m_tMinUVKeepTimeStamp; - std::vector m_dir_names_stack; - std::vector m_dir_usage_stack; - - - XrdOssAt m_oss_at; - - DirState *m_dir_state; - std::string m_current_path; // Includes trailing '/' - int m_dir_level; - const int m_max_dir_level_for_stat_collection; // until we honor globs from pfc.dirstats - - const char *m_info_ext; - const size_t m_info_ext_len; - XrdSysTrace *m_trace; static const char *m_traceID; - list_t m_flist; // list of files to be removed unconditionally map_t m_fmap; // map of files that are purge candidates @@ -70,18 +58,6 @@ public: map_t &refMap() { return m_fmap; } list_t &refList() { return m_flist; } - // ------------------------------------ - // Directory handling & stat collection - // ------------------------------------ - - void begin_traversal(DirState *root, const char *root_path = "/"); - - void end_traversal(); - - void cd_down(const std::string& dir_name); - - void cd_up(); - void setMinTime(time_t min_time) { m_tMinTimeStamp = min_time; } time_t getMinTime() const { return m_tMinTimeStamp; } void setUVKeepMinTime(time_t min_time) { m_tMinUVKeepTimeStamp = min_time; } @@ -89,9 +65,10 @@ public: void MoveListEntriesToMap(); - void CheckFile(const char *fname, Info &info, struct stat &fstat /*, XrdOssDF *iOssDF*/); + void CheckFile(const FsTraversal &fst, const char *fname, Info &info, struct stat &fstat); - void TraverseNamespace(XrdOssDF *iOssDF); + void ProcessDirAndRecurse(FsTraversal &fst); + bool TraverseNamespace(const char *root_path); }; } // namespace XrdPfc diff --git a/src/XrdPfc/XrdPfcFile.cc b/src/XrdPfc/XrdPfcFile.cc index d51cac51ff8..f615c050022 100644 --- a/src/XrdPfc/XrdPfcFile.cc +++ b/src/XrdPfc/XrdPfcFile.cc @@ -18,12 +18,11 @@ #include "XrdPfcFile.hh" +#include "XrdPfc.hh" +#include "XrdPfcResourceMonitor.hh" #include "XrdPfcIO.hh" #include "XrdPfcTrace.hh" -#include -#include -#include -#include + #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClFile.hh" @@ -32,7 +31,11 @@ #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdSfs/XrdSfsInterface.hh" -#include "XrdPfc.hh" + +#include +#include +#include +#include using namespace XrdPfc; @@ -67,6 +70,7 @@ File::File(const std::string& path, long long iOffset, long long iFileSize) : m_state_cond(0), m_block_size(0), m_num_blocks(0), + m_resmon_token(-1), m_prefetch_state(kOff), m_prefetch_read_cnt(0), m_prefetch_hit_cnt(0), @@ -91,6 +95,10 @@ File::~File() m_data_file = NULL; } + if (m_resmon_token >= 0) { + Cache::ResMon().register_file_close(m_resmon_token, DeltaStatsFromLastCall(), time(0)); + } + TRACEF(Debug, "~File() ended, prefetch score = " << m_prefetch_score); } @@ -134,21 +142,21 @@ void File::initiate_emergency_shutdown() cache()->DeRegisterPrefetchFile(this); } } - } //------------------------------------------------------------------------------ Stats File::DeltaStatsFromLastCall() { - // Not locked, only used from Cache / Purge thread. - - Stats delta = m_last_stats; - - m_last_stats = m_stats.Clone(); + // Used for ResourceMonitor thread. + Stats delta; + { + XrdSysCondVarHelper _lck(m_state_cond); + delta = m_last_stats; + m_last_stats = m_stats; + } delta.DeltaToReference(m_last_stats); - return delta; } @@ -278,8 +286,7 @@ bool File::FinalizeSyncBeforeExit() { if ( ! m_writes_during_sync.empty() || m_non_flushed_cnt > 0 || ! m_detach_time_logged) { - Stats loc_stats = m_stats.Clone(); - m_cfi.WriteIOStatDetach(loc_stats); + m_cfi.WriteIOStatDetach(m_stats); m_detach_time_logged = true; m_in_sync = true; TRACEF(Debug, "FinalizeSyncBeforeExit requesting sync to write detach stats"); @@ -481,6 +488,9 @@ bool File::Open() m_block_size = m_cfi.GetBufferSize(); m_num_blocks = m_cfi.GetNBlocks(); m_prefetch_state = (m_cfi.IsComplete()) ? kComplete : kStopped; // Will engage in AddIO(). + m_resmon_token = Cache::ResMon().register_file_open(m_filename, time(0)); + // XXXX have some reporting counter that will trigger inter open stat reporting??? + // Or keep pull mode? Hmmh, requires Cache::active_cond lock ... and can desync with close. m_state_cond.UnLock(); return true; @@ -660,7 +670,10 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize, ReadR { m_state_cond.UnLock(); int ret = m_data_file->Read(iUserBuff, iUserOff, iUserSize); - if (ret > 0) m_stats.AddBytesHit(ret); + if (ret > 0) { + XrdSysCondVarHelper _lck(m_state_cond); + m_stats.AddBytesHit(ret); + } return ret; } @@ -689,7 +702,10 @@ int File::ReadV(IO *io, const XrdOucIOVec *readV, int readVnum, ReadReqRH *rh) { m_state_cond.UnLock(); int ret = m_data_file->ReadV(const_cast(readV), readVnum); - if (ret > 0) m_stats.AddBytesHit(ret); + if (ret > 0) { + XrdSysCondVarHelper _lck(m_state_cond); + m_stats.AddBytesHit(ret); + } return ret; } @@ -906,10 +922,10 @@ int File::ReadOpusCoalescere(IO *io, const XrdOucIOVec *readV, int readVnum, if (read_req->is_complete()) { // Almost like FinalizeReadRequest(read_req) -- but no callout! - m_state_cond.UnLock(); - m_stats.AddReadStats(read_req->m_stats); + m_state_cond.UnLock(); + int ret = read_req->return_value(); delete read_req; return ret; @@ -1027,7 +1043,11 @@ void File::Sync() bool errorp = false; if (ret == XrdOssOK) { - Stats loc_stats = m_stats.Clone(); + Stats loc_stats; + { + XrdSysCondVarHelper _lck(&m_state_cond); + loc_stats = m_stats; + } m_cfi.WriteIOStat(loc_stats); m_cfi.Write(m_info_file, m_filename.c_str()); int cret = m_info_file->Fsync(); @@ -1241,8 +1261,10 @@ void File::FinalizeReadRequest(ReadRequest *rreq) { // called from ProcessBlockResponse() // NOT under lock -- does callout - - m_stats.AddReadStats(rreq->m_stats); + { + XrdSysCondVarHelper _lck(m_state_cond); + m_stats.AddReadStats(rreq->m_stats); + } rreq->m_rh->Done(rreq->return_value()); delete rreq; diff --git a/src/XrdPfc/XrdPfcFile.hh b/src/XrdPfc/XrdPfcFile.hh index 6e8b74ae7de..501e8d8cd9d 100644 --- a/src/XrdPfc/XrdPfcFile.hh +++ b/src/XrdPfc/XrdPfcFile.hh @@ -18,14 +18,15 @@ // along with XRootD. If not, see . //---------------------------------------------------------------------------------- +#include "XrdPfcTypes.hh" +#include "XrdPfcInfo.hh" +#include "XrdPfcStats.hh" + #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucIOVec.hh" -#include "XrdPfcInfo.hh" -#include "XrdPfcStats.hh" - #include #include #include @@ -41,6 +42,7 @@ class Log; namespace XrdPfc { +class File; class BlockResponseHandler; class DirectResponseHandler; class IO; @@ -49,12 +51,6 @@ struct ReadVBlockListRAM; struct ReadVChunkListRAM; struct ReadVBlockListDisk; struct ReadVChunkListDisk; -} - - -namespace XrdPfc -{ -class File; struct ReadReqRH : public XrdOucCacheIOCB { @@ -270,12 +266,12 @@ public: //! Log path const char* lPath() const; - std::string& GetLocalPath() { return m_filename; } + const std::string& GetLocalPath() const { return m_filename; } XrdSysError* GetLog(); XrdSysTrace* GetTrace(); - long long GetFileSize() { return m_file_size; } + long long GetFileSize() const { return m_file_size; } void AddIO(IO *io); int GetPrefetchCountOnIO(IO *io); @@ -283,6 +279,7 @@ public: void RemoveIO(IO *io); Stats DeltaStatsFromLastCall(); + int GetResMonToken() const { return m_resmon_token; } std::string GetRemoteLocations() const; const Info::AStat* GetLastAccessStats() const { return m_cfi.GetLastAccessStats(); } @@ -315,9 +312,9 @@ private: XrdOssDF *m_info_file; //!< file handle for data-info file on disk Info m_cfi; //!< download status of file blocks and access statistics - std::string m_filename; //!< filename of data file on disk - long long m_offset; //!< offset of cached file for block-based / hdfs operation - long long m_file_size; //!< size of cached disk file for block-based operation + const std::string m_filename; //!< filename of data file on disk + const long long m_offset; //!< offset of cached file for block-based / hdfs operation + const long long m_file_size; //!< size of cached disk file for block-based operation // IO objects attached to this file. @@ -353,6 +350,7 @@ private: Stats m_stats; //!< cache statistics for this instance Stats m_last_stats; //!< copy of cache stats during last purge cycle, used for per directory stat reporting + int m_resmon_token; //!< token used in communication with the ResourceMonitor std::set m_remote_locations; //!< Gathered in AddIO / ioUpdate / ioActive. void insert_remote_location(const std::string &loc); diff --git a/src/XrdPfc/XrdPfcFsTraversal.cc b/src/XrdPfc/XrdPfcFsTraversal.cc new file mode 100644 index 00000000000..f163f81961b --- /dev/null +++ b/src/XrdPfc/XrdPfcFsTraversal.cc @@ -0,0 +1,241 @@ +#include "XrdPfcFsTraversal.hh" +#include "XrdPfcDirState.hh" +#include "XrdPfc.hh" +#include "XrdPfcTrace.hh" + +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOss/XrdOssApi.hh" + +// #define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" +#define TRACE_PURGE(x) + +using namespace XrdPfc; + +namespace +{ + XrdSysTrace* GetTrace() { return Cache::GetInstance().GetTrace(); } +} + +const char *FsTraversal::m_traceID = "FsTraversal"; + +//---------------------------------------------------------------------------- + +FsTraversal::FsTraversal(XrdOss &oss) : + m_oss(oss), m_oss_at(oss) +{} + +FsTraversal::~FsTraversal() +{} + +//---------------------------------------------------------------------------- + +bool FsTraversal::begin_traversal(DirState *root, const char *root_path) +{ + m_maintain_dirstate = true; + m_root_dir_state = m_dir_state = root; + + bool ret = begin_traversal(root_path); + + m_maintain_dirstate = false; + + return ret; +} + +bool FsTraversal::begin_traversal(const char *root_path) +{ + static const char *trc_pfx = "FsTraversal::begin_traversal "; + + assert(root_path && strlen(root_path) > 0 && root_path[strlen(root_path) - 1] == '/'); + + m_rel_dir_level = 0; + m_current_path = root_path; + + XrdOssDF* dhp = m_oss.newDir("PfcFsTraversal"); + if (dhp->Opendir(root_path, m_env) != XrdOssOK) { + delete dhp; + TRACE(Error, trc_pfx << "could not opendir [" << root_path << "], " << XrdSysE2T(errno)); + return false; + } + m_dir_handle_stack.push_back(dhp); + + TRACE_PURGE("FPurgeState::begin_traversal cur_path '" << m_current_path << "', rel_level=" << m_rel_dir_level); + + slurp_current_dir(); + return true; +} + +void FsTraversal::end_traversal() +{ + TRACE_PURGE("FPurgeState::end_traversal reporting for '" << m_current_path << "', re_level=" << m_rel_dir_level); + + for (auto &dhp : m_dir_handle_stack) { + dhp->Close(); + delete dhp; + } + m_dir_handle_stack.clear(); + m_current_path.clear(); + m_current_dirs.clear(); + m_current_files.clear(); + + m_rel_dir_level = -1; + m_root_dir_state = m_dir_state = nullptr; +} + +//---------------------------------------------------------------------------- + +bool FsTraversal::cd_down(const std::string &dir_name) +{ + static const char *trc_pfx = "FsTraversal::cd_down "; + + XrdOssDF *dhp = 0; + if (m_oss_at.Opendir(*m_dir_handle_stack.back(), dir_name.c_str(), m_env, dhp) != XrdOssOK) { + delete dhp; + TRACE(Error, trc_pfx << "could not opendir [" << m_current_path << dir_name << "], " << XrdSysE2T(errno)); + return false; + } + m_dir_handle_stack.push_back(dhp); + + ++m_rel_dir_level; + m_current_path.append(dir_name); + m_current_path.append("/"); + + if (m_maintain_dirstate) + m_dir_state = m_dir_state->find_dir(dir_name, true); + + slurp_current_dir(); + return true; +} + +void FsTraversal::cd_up() +{ + m_current_dirs.clear(); + m_current_files.clear(); + + m_dir_handle_stack.back()->Close(); + delete m_dir_handle_stack.back(); + m_dir_handle_stack.pop_back(); + + if (m_maintain_dirstate) + m_dir_state = m_dir_state->get_parent(); + + m_current_path.erase(m_current_path.find_last_of('/', m_current_path.size() - 2) + 1); + --m_rel_dir_level; +} + +//---------------------------------------------------------------------------- + +void FsTraversal::slurp_current_dir() +{ + static const char *trc_pfx = "FsTraversal::slurp_current_dir "; + + char fname[256]; + struct stat fstat; + XrdOucEnv env; + + XrdOssDF &dh = *m_dir_handle_stack.back(); + dh.StatRet(&fstat); + + const char *info_ext = Info::s_infoExtension; + const size_t info_ext_len = Info::s_infoExtensionLen; + + m_current_dirs.clear(); + m_current_files.clear(); + + while (true) + { + int rc = dh.Readdir(fname, 256); + + if (rc == -ENOENT) + { + TRACE_PURGE(" Skipping ENOENT dir entry [" << fname << "]."); + continue; + } + if (rc != XrdOssOK) + { + TRACE(Error, trc_pfx << "Readdir error at " << m_current_path << ", err " << XrdSysE2T(-rc) << "."); + break; + } + + TRACE_PURGE(" Readdir [" << fname << "]"); + + if (fname[0] == 0) + { + TRACE_PURGE(" Finished reading dir [" << m_current_path << "]. Break loop."); + break; + } + if (fname[0] == '.' && (fname[1] == 0 || (fname[1] == '.' && fname[2] == 0))) + { + TRACE_PURGE(" Skipping here or parent dir [" << fname << "]. Continue loop."); + continue; + } + + if (S_ISDIR(fstat.st_mode)) + { + if (m_rel_dir_level == 0 && m_protected_top_dirs.find(fname) != m_protected_top_dirs.end()) + { + // Skip protected top-directories. + continue; + } + m_current_dirs.push_back(fname); + } + else + { + size_t fname_len = strlen(fname); + + if (fname_len > info_ext_len && strncmp(&fname[fname_len - info_ext_len], info_ext, info_ext_len) == 0) + { + // truncate ".cinfo" away + fname[fname_len - info_ext_len] = 0; + m_current_files[fname].set_cinfo(fstat); + } + else + { + m_current_files[fname].set_data(fstat); + } + } + + /* + size_t fname_len = strlen(fname); + XrdOssDF *dfh = 0; + + if (S_ISDIR(fstat.st_mode)) + { + if (m_oss_at.Opendir(*iOssDF, fname, env, dfh) == XrdOssOK) + { + cd_down(fname); + TRACE_PURGE(" cd_down -> [" << m_current_path << "]."); + TraverseNamespace(dfh); + cd_up(); + TRACE_PURGE(" cd_up -> [" << m_current_path << "]."); + } + else + TRACE(Warning, trc_pfx << "could not opendir [" << m_current_path << fname << "], " << XrdSysE2T(errno)); + } + else if (fname_len > info_ext_len && strncmp(&fname[fname_len - info_ext_len], info_ext, info_ext_len) == 0) + { + // Check if the file is currently opened / purge-protected is done before unlinking of the file. + + Info cinfo(GetTrace()); + + if (m_oss_at.OpenRO(*iOssDF, fname, env, dfh) == XrdOssOK && cinfo.Read(dfh, m_current_path.c_str(), fname)) + { + CheckFile(fname, cinfo, fstat); + } + else + { + TRACE(Warning, trc_pfx << "can't open or read " << m_current_path << fname << ", err " << XrdSysE2T(errno) << "; purging."); + m_oss_at.Unlink(*iOssDF, fname); + fname[fname_len - info_ext_len] = 0; + m_oss_at.Unlink(*iOssDF, fname); + // generate purge event or not? or just flag possible discrepancy? + } + } + else // XXXX devel debug only, to be removed + { + TRACE_PURGE(" Ignoring [" << fname << "], not a dir or cinfo."); + } + + delete dfh; + */ + } +} diff --git a/src/XrdPfc/XrdPfcFsTraversal.hh b/src/XrdPfc/XrdPfcFsTraversal.hh new file mode 100644 index 00000000000..baec350f084 --- /dev/null +++ b/src/XrdPfc/XrdPfcFsTraversal.hh @@ -0,0 +1,80 @@ +#ifndef __XRDPFC_FSTRAVERSAL_HH__ +#define __XRDPFC_FSTRAVERSAL_HH__ + +#include "XrdOss/XrdOssAt.hh" +#include "XrdOuc/XrdOucEnv.hh" + +#include +#include +#include +#include +#include + +class XrdOss; +class XrdOssDF; + +namespace XrdPfc { + +class DirState; + +class FsTraversal +{ +public: + struct FilePairStat { + struct stat stat_data, stat_cinfo; + bool has_data = false; + bool has_cinfo = false; + + void set_data (const struct stat &s) { stat_data = s; has_data = true; } + void set_cinfo(const struct stat &s) { stat_cinfo = s; has_cinfo = true; } + }; + +protected: + XrdOss &m_oss; + XrdOssAt m_oss_at; + XrdOucEnv m_env; + + bool m_maintain_dirstate = false; + +public: + DirState *m_root_dir_state = nullptr; + DirState *m_dir_state = nullptr; // current DirState + + int m_rel_dir_level = -1; // dir level relative to root, 0 ~ at root + std::string m_current_path; // Includes trailing '/' -- needed for printouts and PurgeCandidate creation. + + // Hmmh ... need a stack of those ... or not, if doing tail recursion. + // Can not, OpenDirAt descend can not be like that, ie, i will need the old handle. + std::vector m_dir_handle_stack; + + std::vector m_current_dirs; // swap out into local scope before recursion + std::map m_current_files; // clear when done + + std::set m_protected_top_dirs; // directories that will NOT be traversed at relative level 0. + + static const char *m_traceID; + + void slurp_current_dir(); + +public: + FsTraversal(XrdOss &oss); + ~FsTraversal(); + + bool begin_traversal(DirState *root, const char *root_path); + bool begin_traversal(const char *root_path); + void end_traversal(); + + bool cd_down(const std::string &dir_name); + void cd_up(); + + int open_at_ro(const char* fname, XrdOssDF *&ossDF) { + return m_oss_at.OpenRO(*m_dir_handle_stack.back(), fname, m_env, ossDF); + } + int unlink_at(const char* fname) { + return m_oss_at.Unlink(*m_dir_handle_stack.back(), fname); + } +}; + +} + +#endif diff --git a/src/XrdPfc/XrdPfcIOFile.cc b/src/XrdPfc/XrdPfcIOFile.cc index d0b3ebef169..bca6bda0f3f 100644 --- a/src/XrdPfc/XrdPfcIOFile.cc +++ b/src/XrdPfc/XrdPfcIOFile.cc @@ -16,20 +16,21 @@ // along with XRootD. If not, see . //---------------------------------------------------------------------------------- -#include -#include - -#include "XrdSys/XrdSysError.hh" -#include "XrdSfs/XrdSfsInterface.hh" -#include "XrdSys/XrdSysPthread.hh" - #include "XrdPfcIOFile.hh" #include "XrdPfcStats.hh" #include "XrdPfcTrace.hh" +#include "XrdOss/XrdOss.hh" +#include "XrdSfs/XrdSfsInterface.hh" +#include "XrdSys/XrdSysError.hh" +#include "XrdSys/XrdSysPthread.hh" + #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucPgrwUtils.hh" +#include +#include + using namespace XrdPfc; //______________________________________________________________________________ diff --git a/src/XrdPfc/XrdPfcIOFileBlock.cc b/src/XrdPfc/XrdPfcIOFileBlock.cc index deb7ee2a38c..7b72500c0b5 100644 --- a/src/XrdPfc/XrdPfcIOFileBlock.cc +++ b/src/XrdPfc/XrdPfcIOFileBlock.cc @@ -16,23 +16,24 @@ // along with XRootD. If not, see . //---------------------------------------------------------------------------------- -#include -#include -#include -#include -#include -#include - #include "XrdPfcIOFileBlock.hh" #include "XrdPfc.hh" #include "XrdPfcStats.hh" #include "XrdPfcTrace.hh" -#include "XrdSys/XrdSysError.hh" +#include "XrdOss/XrdOss.hh" #include "XrdSfs/XrdSfsInterface.hh" +#include "XrdSys/XrdSysError.hh" #include "XrdOuc/XrdOucEnv.hh" +#include +#include +#include +#include +#include +#include + using namespace XrdPfc; //______________________________________________________________________________ diff --git a/src/XrdPfc/XrdPfcInfo.hh b/src/XrdPfc/XrdPfcInfo.hh index d5082e684ad..b9f965bcb23 100644 --- a/src/XrdPfc/XrdPfcInfo.hh +++ b/src/XrdPfc/XrdPfcInfo.hh @@ -18,14 +18,14 @@ // along with XRootD. If not, see . //---------------------------------------------------------------------------------- -#include -#include -#include -#include +#include "XrdPfcTypes.hh" #include "XrdSys/XrdSysPthread.hh" -#include "XrdPfcTypes.hh" +#include +#include +#include +#include class XrdOssDF; class XrdCksCalc; diff --git a/src/XrdPfc/XrdPfcPathParseTools.hh b/src/XrdPfc/XrdPfcPathParseTools.hh new file mode 100644 index 00000000000..be33eaabdaa --- /dev/null +++ b/src/XrdPfc/XrdPfcPathParseTools.hh @@ -0,0 +1,138 @@ +#ifndef __XRDPFC_PATHPARSETOOLS_HH__ +#define __XRDPFC_PATHPARSETOOLS_HH__ + +#include +#include + +#include +#include + +namespace XrdPfc { + +struct SplitParser +{ + char *f_str; + const char *f_delim; + char *f_state; + bool f_first; + + SplitParser(const std::string &s, const char *d) : + f_str(strdup(s.c_str())), f_delim(d), f_state(0), f_first(true) + {} + ~SplitParser() { free(f_str); } + + char* get_token() + { + if (f_first) { f_first = false; return strtok_r(f_str, f_delim, &f_state); } + else { return strtok_r(0, f_delim, &f_state); } + } + + char* get_reminder_with_delim() + { + if (f_first) { return f_str; } + else { *(f_state - 1) = f_delim[0]; return f_state - 1; } + } + + char *get_reminder() + { + return f_first ? f_str : f_state; + } + + int fill_argv(std::vector &argv) + { + if (!f_first) return 0; + int dcnt = 0; { char *p = f_str; while (*p) { if (*(p++) == f_delim[0]) ++dcnt; } } + argv.reserve(dcnt + 1); + int argc = 0; + char *i = strtok_r(f_str, f_delim, &f_state); + while (i) + { + ++argc; + argv.push_back(i); + // printf(" arg %d : '%s'\n", argc, i); + i = strtok_r(0, f_delim, &f_state); + } + return argc; + } +}; + +struct PathTokenizer : private SplitParser +{ + std::vector m_dirs; + const char *m_reminder; + int m_n_dirs; + + PathTokenizer(const std::string &path, int max_depth, bool parse_as_lfn) : + SplitParser(path, "/"), + m_reminder (0), + m_n_dirs (0) + { + // max_depth - maximum number of directories to extract. If < 0, all path elements + // are extracted (well, up to 4096). The rest is in m_reminder. + // If parse_as_lfn is true store final token into m_reminder, regardless of maxdepth. + // This assumes the last token is a file name (and full path is lfn, including the file name). + + if (max_depth < 0) + max_depth = 4096; + m_dirs.reserve(std::min(8, max_depth)); + + char *t = 0; + for (int i = 0; i < max_depth; ++i) + { + t = get_token(); + if (t == 0) break; + m_dirs.emplace_back(t); + } + if (parse_as_lfn && *get_reminder() == 0 && ! m_dirs.empty()) + { + m_reminder = m_dirs.back(); + m_dirs.pop_back(); + } + else + { + m_reminder = get_reminder(); + } + m_n_dirs = (int) m_dirs.size(); + } + + int get_n_dirs() + { + return m_n_dirs; + } + + const char *get_dir(int pos) + { + if (pos >= m_n_dirs) return 0; + return m_dirs[pos]; + } + + std::string make_path() + { + std::string res; + for (std::vector::iterator i = m_dirs.begin(); i != m_dirs.end(); ++i) + { + res += "/"; + res += *i; + } + if (m_reminder != 0) + { + res += "/"; + res += m_reminder; + } + return res; + } + + void print_debug() + { + printf("PathTokenizer::print_debug size=%d\n", m_n_dirs); + for (int i = 0; i < m_n_dirs; ++i) + { + printf(" %2d: %s\n", i, m_dirs[i]); + } + printf(" rem: %s\n", m_reminder); + } +}; + +} + +#endif diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index 27f69612804..8d3f07d22a2 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -1,7 +1,8 @@ #include "XrdPfc.hh" +#include "XrdPfcPathParseTools.hh" #include "XrdPfcDirState.hh" #include "XrdPfcFPurgeState.hh" -#include "XrdPfcDirPurge.hh" +#include "XrdPfcPurgePin.hh" #include "XrdPfcTrace.hh" #include @@ -9,86 +10,14 @@ #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucUtils.hh" +#include "XrdOss/XrdOss.hh" #include "XrdOss/XrdOssAt.hh" #include "XrdSys/XrdSysTrace.hh" using namespace XrdPfc; -namespace XrdPfc -{ - -/* -XrdSysTrace* GetTrace() -{ - // needed for logging macros - return Cache::GetInstance().GetTrace(); -}*/ - -//============================================================================== -// DataFsState -//============================================================================== - -class DataFsState -{ - int m_max_depth; - DirState m_root; - time_t m_prev_time; - -public: - DataFsState() : - m_max_depth ( Cache::Conf().m_dirStatsStoreDepth ), - m_root ( m_max_depth ), - m_prev_time ( time(0) ) - {} - - int get_max_depth() const { return m_max_depth; } - - DirState* get_root() { return & m_root; } - - DirState* find_dirstate_for_lfn(const std::string& lfn) - { - return m_root.find_path(lfn, m_max_depth, true, true); - } - - void reset_stats() { m_root.reset_stats(); } - void upward_propagate_stats() { m_root.upward_propagate_stats(); } - void upward_propagate_usage_purged() { m_root.upward_propagate_usage_purged(); } - - void dump_recursively() - { - time_t now = time(0); - - printf("DataFsState::dump_recursively epoch = %lld delta_t = %lld max_depth = %d\n", - (long long) now, (long long) (now - m_prev_time), m_max_depth); - - m_prev_time = now; - - m_root.dump_recursively("root"); - } -}; - - - -const char *FPurgeState::m_traceID = "Purge"; - //============================================================================== -// ResourceMonitor -//============================================================================== - -// Encapsulates local variables used withing the previous mega-function Purge(). -// -// This will be used within the continuously/periodically ran heart-beat / breath -// function ... and then parts of it will be passed to invoked FS scan and purge -// jobs (which will be controlled throught this as well). - -class ResourceMonitor -{ - -}; - - -//============================================================================== -// +// anonymous //============================================================================== namespace @@ -108,137 +37,8 @@ class ScanAndPurgeJob : public XrdJob // Cache methods //============================================================================== -void Cache::copy_out_active_stats_and_update_data_fs_state() -{ - static const char *trc_pfx = "copy_out_active_stats_and_update_data_fs_state() "; - - StatsMMap_t updates; - { - XrdSysCondVarHelper lock(&m_active_cond); - - // Slurp in stats from files closed since last cycle. - updates.swap( m_closed_files_stats ); - - for (ActiveMap_i i = m_active.begin(); i != m_active.end(); ++i) - { - if (i->second != 0) - { - updates.insert(std::make_pair(i->first, i->second->DeltaStatsFromLastCall())); - } - } - } - - m_fs_state->reset_stats(); // XXXX-CKSUM rethink how to do this if we keep some purge entries for next time - - for (StatsMMap_i i = updates.begin(); i != updates.end(); ++i) - { - DirState *ds = m_fs_state->find_dirstate_for_lfn(i->first); - - if (ds == 0) - { - TRACE(Error, trc_pfx << "Failed finding DirState for file '" << i->first << "'."); - continue; - } - - ds->add_up_stats(i->second); - } - - m_fs_state->upward_propagate_stats(); -} -//============================================================================== - -void Cache::ResourceMonitorHeartBeat() -{ - // static const char *trc_pfx = "ResourceMonitorHeartBeat() "; - - // Pause before initial run - sleep(1); - - // XXXX Setup initial / constant stats (total RAM, total disk, ???) - - XrdOucCacheStats &S = Statistics; - XrdOucCacheStats::CacheStats &X = Statistics.X; - - S.Lock(); - - X.DiskSize = m_configuration.m_diskTotalSpace; - - X.MemSize = m_configuration.m_RamAbsAvailable; - - S.UnLock(); - - // XXXX Schedule initial disk scan, time it! - // - // TRACE(Info, trc_pfx << "scheduling intial disk scan."); - // schedP->Schedule( new ScanAndPurgeJob("XrdPfc::ScanAndPurge") ); - // - // bool scan_and_purge_running = true; - - // XXXX Could we really hold last-usage for all files in memory? - - // XXXX Think how to handle disk-full, scan/purge not finishing: - // - start dropping things out of write queue, but only when RAM gets near full; - // - monitoring this then becomes a high-priority job, inner loop with sleep of, - // say, 5 or 10 seconds. - - while (true) - { - time_t heartbeat_start = time(0); - - // TRACE(Info, trc_pfx << "HeartBeat starting ..."); - - // if sumary monitoring configured, pupulate OucCacheStats: - S.Lock(); - - // - available / used disk space (files usage calculated elsewhere (maybe)) - - // - RAM usage - { XrdSysMutexHelper lck(&m_RAM_mutex); - X.MemUsed = m_RAM_used; - X.MemWriteQ = m_RAM_write_queue; - } - // - files opened / closed etc - - // do estimate of available space - S.UnLock(); - - // if needed, schedule purge in a different thread. - // purge is: - // - deep scan + gather FSPurgeState - // - actual purge - // - // this thread can continue running and, if needed, stop writing to disk - // if purge is taking too long. - - // think how data is passed / synchronized between this and purge thread - - // !!!! think how stat collection is done and propgated upwards; - // until now it was done once per purge-interval. - // now stats will be added up more often, but purge will be done - // only occasionally. - // also, do we report cumulative values or deltas? cumulative should - // be easier and consistent with summary data. - // still, some are state - like disk usage, num of files. - - // Do we take care of directories that need to be newly added into DirState hierarchy? - // I.e., when user creates new directories and these are covered by either full - // spec or by root + depth declaration. - - int heartbeat_duration = time(0) - heartbeat_start; - - // TRACE(Info, trc_pfx << "HeartBeat finished, heartbeat_duration " << heartbeat_duration); - - // int sleep_time = m_configuration.m_purgeInterval - heartbeat_duration; - int sleep_time = 60 - heartbeat_duration; - if (sleep_time > 0) - { - sleep(sleep_time); - } - } -} - //============================================================================== void Cache::Purge() @@ -252,8 +52,6 @@ void Cache::Purge() // Pause before initial run sleep(1); - m_fs_state = new DataFsState; - // { PathTokenizer p("/a/b/c/f.root", 2, true); p.deboog(); } // { PathTokenizer p("/a/b/f.root", 2, true); p.deboog(); } // { PathTokenizer p("/a/f.root", 2, true); p.deboog(); } @@ -335,10 +133,11 @@ void Cache::Purge() } } + // XXXX this will happen in heart_beat() bool enforce_traversal_for_usage_collection = is_first; - // XXX Other conditions? Periodic checks? - copy_out_active_stats_and_update_data_fs_state(); + // XXXX as above + // copy_out_active_stats_and_update_data_fs_state(); TRACE(Debug, trc_pfx << "Precheck:"); TRACE(Debug, "\tbytes_to_remove_disk = " << bytesToRemove_d << " B"); @@ -358,7 +157,7 @@ void Cache::Purge() if (purge_required || enforce_traversal_for_usage_collection) { - // Make a sorted map of file paths sorted by access time. + // Make a map of file paths, sorted by access time. if (m_configuration.is_age_based_purge_in_effect()) { @@ -369,18 +168,11 @@ void Cache::Purge() purgeState.setUVKeepMinTime(time(0) - m_configuration.m_cs_UVKeep); } - XrdOssDF* dh = m_oss->newDir(m_configuration.m_username.c_str()); - if (dh->Opendir("/", env) == XrdOssOK) - { - purgeState.begin_traversal(m_fs_state->get_root()); - - purgeState.TraverseNamespace(dh); - - purgeState.end_traversal(); - - dh->Close(); + bool scan_ok = purgeState.TraverseNamespace("/"); + if ( ! scan_ok) { + TRACE(Error, trc_pfx << "namespace traversal failed at top-directory, this should not happen."); + // XXX once this runs in a job / independent thread, stop processing. } - delete dh; dh = 0; estimated_file_usage = purgeState.getNBytesTotal(); @@ -431,9 +223,9 @@ void Cache::Purge() ///////////////////////////////////////////////////////////// if (m_purge_pin) { - // set dir stat for each path and calculate nBytes to rocover for each path + // set dir stat for each path and calculate nBytes to recover for each path // return total bytes to recover within the plugin - long long clearVal = m_purge_pin->GetBytesToRecover(m_fs_state->get_root()); + long long clearVal = m_purge_pin->GetBytesToRecover(nullptr); // m_fs_state->get_root()); if (clearVal) { TRACE(Debug, "PurgePin remove total " << clearVal << " bytes"); @@ -442,15 +234,11 @@ void Cache::Purge() for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) { TRACE(Debug, "\tPurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes"); - XrdOssDF *ldh = m_oss->newDir(m_configuration.m_username.c_str()); + FPurgeState fps(ppit->nBytesToRecover, *m_oss); - if (ldh->Opendir(ppit->path.c_str(), env) == XrdOssOK) - { - DirState *lds = ppit->dirState; - fps.begin_traversal(lds); - fps.TraverseNamespace(ldh); - fps.end_traversal(); - ldh->Close(); + bool scan_ok = fps.TraverseNamespace(ppit->path.c_str()); + if ( ! scan_ok) { + continue; } // fill central map from the plugin entry @@ -471,18 +259,10 @@ void Cache::Purge() /// ///////////////////////////////////////////////////////////// - // Dump statistcs before actual purging so maximum usage values get recorded. - // Should really go to gstream --- and should really go from Heartbeat. - if (m_configuration.is_dir_stat_reporting_on()) - { - m_fs_state->dump_recursively(); - } - if (purge_required) { // Loop over map and remove files with oldest values of access time. struct stat fstat; - size_t info_ext_len = strlen(Info::s_infoExtension); int protected_cnt = 0; long long protected_sum = 0; for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) @@ -495,7 +275,7 @@ void Cache::Purge() } std::string &infoPath = it->second.path; - std::string dataPath = infoPath.substr(0, infoPath.size() - info_ext_len); + std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen); if (IsFileActiveOrPurgeProtected(dataPath)) { @@ -528,18 +308,19 @@ void Cache::Purge() m_oss->Unlink(dataPath.c_str()); TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << it->second.nBytes << ", time: " << it->first); + // XXXXX Report removal in some other way, aggregate by dirname, get DirState from somewhere else: + /* if (it->second.dirState != 0) // XXXX This should now always be true. it->second.dirState->add_usage_purged(it->second.nBytes); else TRACE(Error, trc_pfx << "DirState not set for file '" << dataPath << "'."); + */ } } if (protected_cnt > 0) { TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << protected_sum); } - - m_fs_state->upward_propagate_usage_purged(); } { @@ -561,5 +342,3 @@ void Cache::Purge() } } } - -} // end XrdPfc namespace diff --git a/src/XrdPfc/XrdPfcPurgePin.hh b/src/XrdPfc/XrdPfcPurgePin.hh index 67e48e42062..2dfb2dfdba9 100644 --- a/src/XrdPfc/XrdPfcPurgePin.hh +++ b/src/XrdPfc/XrdPfcPurgePin.hh @@ -18,8 +18,10 @@ public: { std::string path; long long nBytesQuota{0}; - DirState *dirState{nullptr}; // currently cached and shared within the purge thread long long nBytesToRecover{0}; + + // internal use by the Cache purge thread. to be revisited, maybe an access token is more appropriate. + DirState *dirState{nullptr}; }; typedef std::vector list_t; @@ -34,7 +36,9 @@ public: //--------------------------------------------------------------------- //! Provide erase information from directory statistics //! - //! @param & XrdPfcDirState + //! @param & XrdPfc::DirState vector, exported from the tree version. + // To be revisited -- can have a multi-step approach where + // cache periodically sends udates. //! //! @return total number of bytes //--------------------------------------------------------------------- diff --git a/src/XrdPfc/XrdPfcPurgeQuota.cc b/src/XrdPfc/XrdPfcPurgeQuota.cc index 2d5f18d688e..12994ab8d08 100644 --- a/src/XrdPfc/XrdPfcPurgeQuota.cc +++ b/src/XrdPfc/XrdPfcPurgeQuota.cc @@ -21,7 +21,7 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin { for (list_i it = m_list.begin(); it != m_list.end(); ++it) { - it->dirState = rootDS->find_path(it->path, Cache::Conf().m_dirStatsStoreDepth, false, false); + it->dirState = rootDS->find_path(it->path, XrdPfc::Cache::Conf().m_dirStatsStoreDepth, false, false); } } @@ -37,7 +37,8 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin // get bytes to remove for (list_i it = m_list.begin(); it != m_list.end(); ++it) { - long long cv = it->dirState->get_usage() - it->nBytesQuota; + // XXXXXX here we should have another mechanism. and probably add up here + subdirs + long long cv = it->dirState->m_recursive_subdir_usage.m_bytes_on_disk - it->nBytesQuota; if (cv > 0) it->nBytesToRecover = cv; else @@ -54,7 +55,7 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin //---------------------------------------------------------------------------- virtual bool ConfigPurgePin(const char *parms) { - XrdSysError *log = Cache::GetInstance().GetLog(); + XrdSysError *log = XrdPfc::Cache::GetInstance().GetLog(); // retrive configuration file name if (!parms || !parms[0] || (strlen(parms) == 0)) diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc new file mode 100644 index 00000000000..7a5efd4a7e7 --- /dev/null +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -0,0 +1,355 @@ +#include "XrdPfcResourceMonitor.hh" +#include "XrdPfc.hh" +#include "XrdPfcPathParseTools.hh" +#include "XrdPfcFsTraversal.hh" +#include "XrdPfcDirState.hh" +#include "XrdPfcTrace.hh" + +using namespace XrdPfc; + +namespace +{ + XrdSysTrace* GetTrace() { return Cache::GetInstance().GetTrace(); } + const char *m_traceID = "ResourceMonitor"; +} + +ResourceMonitor::ResourceMonitor(XrdOss& oss) : + m_fs_state(new DataFsState), + m_oss(oss) +{} + +ResourceMonitor::~ResourceMonitor() +{ + delete m_fs_state; +} + +//------------------------------------------------------------------------------ +// Initial scan +//------------------------------------------------------------------------------ + +void ResourceMonitor::scan_dir_and_recurse(FsTraversal &fst) +{ + printf("In scan_dir_and_recurse for '%s', size of dir_vec = %d, file_stat_map = %d\n", + fst.m_current_path.c_str(), + (int)fst.m_current_dirs.size(), (int)fst.m_current_files.size()); + + // Breadth first, accumulate into "here" + DirUsage &here = fst.m_dir_state->m_here_usage; + for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it) + { + printf("would be doing something with %s ... has_data=%d, has_cinfo=%d\n", + it->first.c_str(), it->second.has_data, it->second.has_cinfo); + + // XXX Make some of these optional? + // Remove files that do not have both cinfo and data? + // Remove empty directories before even descending? + // Leave this for some consistency pass? + // Note that FsTraversal supports ignored paths ... some details (cofig, N2N to be clarified). + + if (it->second.has_data && it->second.has_cinfo) { + here.m_bytes_on_disk += it->second.stat_data.st_blocks * 512; + here.m_num_files += 1; + } + } + + // Sub-dirs second, accumulate into "subdirs". + DirUsage &subdirs = fst.m_dir_state->m_recursive_subdir_usage; + std::vector dirs; + dirs.swap(fst.m_current_dirs); + for (auto &dname : dirs) + { + if (fst.cd_down(dname)) + { + DirState *daughter = fst.m_dir_state; + DirUsage &dhere = daughter->m_here_usage; + DirUsage &dsubdirs = daughter->m_recursive_subdir_usage; + + scan_dir_and_recurse(fst); + fst.cd_up(); + + here.m_num_subdirs += 1; + + subdirs.m_bytes_on_disk += dhere.m_bytes_on_disk + dsubdirs.m_bytes_on_disk; + subdirs.m_num_files += dhere.m_num_files + dsubdirs.m_num_files; + subdirs.m_num_subdirs += dhere.m_num_subdirs + dsubdirs.m_num_subdirs; + } + // XXX else try to remove it? + } +} + +bool ResourceMonitor::perform_initial_scan() +{ + // Called after PFC configuration is complete, but before full startup of the daemon. + // Base line usages are accumulated as part of the file-system, traversal. + + bool success_p = true; + + FsTraversal fst(m_oss); + fst.m_protected_top_dirs.insert("pfc-stats"); // XXXX This should come from config. Also: N2N? + + if (fst.begin_traversal(m_fs_state->get_root(), "/")) + { + scan_dir_and_recurse(fst); + } + else + { + // Fail startup, can't open /. + success_p = false; + } + fst.end_traversal(); + + return success_p; +} + +//------------------------------------------------------------------------------ +// Processing of queues +//------------------------------------------------------------------------------ + +int ResourceMonitor::process_queues() +{ + static const char *trc_pfx = "process_queues() "; + + // Assure that we pick up only entries that are present now. + // We really want all open records to be processed before file-stats so let's get + // the current snapshot of already opened files first. + // On the other hand, we do not care if we miss some updates in this pass. + int n_records = Cache::GetInstance().CopyOutActiveStats(m_file_stats_update_vec); + { + XrdSysMutexHelper _lock(&m_queue_mutex); + n_records += m_file_open_q.swap_queues(); + n_records += m_file_close_q.swap_queues(); + n_records += m_file_purge_q1.swap_queues(); + n_records += m_file_purge_q2.swap_queues(); + n_records += m_file_purge_q3.swap_queues(); + } + + for (auto &i : m_file_open_q.read_queue()) { + // time is in stat, fname is in the token slot; + int tid = i.id; + AccessToken &at = token(tid); + printf("process file open for token %d, time %ld -- %s\n", tid, i.record.m_open_time, at.m_filename.c_str()); + // At this point we need to resolve fname into DirState. + // We ould clear the filename after this ... or keep it, should we need it later on. + // - now just used for printing. + DirState *ds = m_fs_state->get_root()->find_path(at.m_filename, -1, true, true); + at.m_dir_state = ds; + ds->m_here_stats.m_NFilesOpened += 1; + + // XXXXXX Here (or, a bit before) we should figure out how many new directories were created here. + // find_path could take an optional arg, last-dir-found -- would also help with purge troubles. + // This should then set, if needed NFiles/Dirs_Created. + + ds->m_here_usage.m_last_open_time = i.record.m_open_time; + } + + for (auto &i : m_file_stats_update_vec) { + int tid = i.first; + AccessToken &at = token(tid); + // Stats + DirState *ds = at.m_dir_state; + printf("process file update for token %d, %p -- %s\n", + tid, ds, at.m_filename.c_str()); + + ds->m_here_stats.AddUp(i.second); + } + m_file_stats_update_vec.clear(); + + for (auto &i : m_file_close_q.read_queue()) { + // Stat and close_time are in the STAT == CloseRecord + // Remember to release the token !!! + int tid = i.id; + AccessToken &at = token(tid); + printf("process file close for token %d, time %ld -- %s\n", + tid, i.record.m_close_time, at.m_filename.c_str()); + + DirState *ds = at.m_dir_state; + ds->m_here_stats.m_NFilesClosed += 1; + + ds->m_here_usage.m_last_close_time = i.record.m_close_time; + + at.clear(); + m_access_tokens_free_slots.push_back(tid); + } + + for (auto &i : m_file_purge_q1.read_queue()) { + // ID is DirState*, multiple files + DirState *ds = i.id; + ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; + ds->m_here_stats.m_NFilesRemoved += i.record.n_files; + } + for (auto &i : m_file_purge_q2.read_queue()) { + // ID is directory-path, multiple files + DirState *ds = m_fs_state->get_root()->find_path(i.id, -1, false, false); + if ( ! ds) { + TRACE(Error, trc_pfx << "DirState not found for directory path '" << i.id << "'."); + // XXX Should have find_path return the last dir / depth still found? + continue; + } + ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; + ds->m_here_stats.m_NFilesRemoved += i.record.n_files; + } + for (auto &i : m_file_purge_q3.read_queue()) { + // ID is LFN, single file + DirState *ds = m_fs_state->get_root()->find_path(i.id, -1, true, false); + if ( ! ds) { + TRACE(Error, trc_pfx << "DirState not found for LFN path '" << i.id << "'."); + // XXX Should have find_path return the last dir / depth still found? + continue; + } + ds->m_here_stats.m_BytesRemoved += i.record; + ds->m_here_stats.m_NFilesRemoved += 1; + } + + // XXXX Upward-propagate here or in heart_beat, as neeed? + + // XXXX dir purge record processing --- seems it is not needed, auto-purge with some cool-off. + + // XXXX clear the read queue, re-capacity to 50% if usage is below 25% + + return n_records; +} + +//------------------------------------------------------------------------------ +// Heart beat +//------------------------------------------------------------------------------ + +void ResourceMonitor::heart_beat() +{ + printf("RMon entering heart_beat!\n"); + + // initial scan performed as part of config + + while (true) + { + int n_processed = process_queues(); + + printf("processed %d records\n", n_processed); + + // check if more to process (just sum of sizes, not swap)? + + // check time, is it time to apply the deltas + m_fs_state->upward_propagate_stats(); // XXXXX before or after export? + // XXXXX ???? One could, maybe, just prop upwards the subdirs-only part .. and reset them. + // No, how could this work? + + // check time, is it time to export into vector format, to disk /pfc-stats + + // Dump statistcs before actual purging so maximum usage values get recorded. + // Should really go to gstream --- and should really go from Heartbeat. + if (Cache::Conf().is_dir_stat_reporting_on()) + { + m_fs_state->dump_recursively(Cache::Conf().m_dirStatsStoreDepth); + } + + m_fs_state->reset_stats(); // XXXXXX this is for sure after export, otherwise they will be zero + + + // check time / diskusage --> purge condition? + // run purge as job or thread + // m_fs_state->upward_propagate_usage_purged(); // XXXX this is the old way + + + // if no work, sleep for N seconds + unsigned int t_sleep = 10; + printf("sleeping for %u seconds, to be improved ...\n", t_sleep); + sleep(t_sleep); + } +} + + +//============================================================================== +// Old prototype from Cache / Purge, now to go into heart_beat() here, above. +//============================================================================== + +void Proto_ResourceMonitorHeartBeat() +{ + // static const char *trc_pfx = "ResourceMonitorHeartBeat() "; + + // Pause before initial run + sleep(1); + + // XXXX Setup initial / constant stats (total RAM, total disk, ???) + + XrdOucCacheStats &S = Cache::GetInstance().Statistics; + XrdOucCacheStats::CacheStats &X = S.X; + + S.Lock(); + + X.DiskSize = Cache::Conf().m_diskTotalSpace; + + X.MemSize = Cache::Conf().m_RamAbsAvailable; + + S.UnLock(); + + // XXXX Schedule initial disk scan, time it! + // + // TRACE(Info, trc_pfx << "scheduling intial disk scan."); + // schedP->Schedule( new ScanAndPurgeJob("XrdPfc::ScanAndPurge") ); + // + // bool scan_and_purge_running = true; + + // XXXX Could we really hold last-usage for all files in memory? + + // XXXX Think how to handle disk-full, scan/purge not finishing: + // - start dropping things out of write queue, but only when RAM gets near full; + // - monitoring this then becomes a high-priority job, inner loop with sleep of, + // say, 5 or 10 seconds. + + while (true) + { + time_t heartbeat_start = time(0); + + // TRACE(Info, trc_pfx << "HeartBeat starting ..."); + + // if sumary monitoring configured, pupulate OucCacheStats: + S.Lock(); + + // - available / used disk space (files usage calculated elsewhere (maybe)) + + // - RAM usage + /* XXXX From Cache + { XrdSysMutexHelper lck(&m_RAM_mutex); + X.MemUsed = m_RAM_used; + X.MemWriteQ = m_RAM_write_queue; + } + */ + + // - files opened / closed etc + + // do estimate of available space + S.UnLock(); + + // if needed, schedule purge in a different thread. + // purge is: + // - deep scan + gather FSPurgeState + // - actual purge + // + // this thread can continue running and, if needed, stop writing to disk + // if purge is taking too long. + + // think how data is passed / synchronized between this and purge thread + + // !!!! think how stat collection is done and propgated upwards; + // until now it was done once per purge-interval. + // now stats will be added up more often, but purge will be done + // only occasionally. + // also, do we report cumulative values or deltas? cumulative should + // be easier and consistent with summary data. + // still, some are state - like disk usage, num of files. + + // Do we take care of directories that need to be newly added into DirState hierarchy? + // I.e., when user creates new directories and these are covered by either full + // spec or by root + depth declaration. + + int heartbeat_duration = time(0) - heartbeat_start; + + // TRACE(Info, trc_pfx << "HeartBeat finished, heartbeat_duration " << heartbeat_duration); + + // int sleep_time = m_configuration.m_purgeInterval - heartbeat_duration; + int sleep_time = 60 - heartbeat_duration; + if (sleep_time > 0) + { + sleep(sleep_time); + } + } +} diff --git a/src/XrdPfc/XrdPfcResourceMonitor.hh b/src/XrdPfc/XrdPfcResourceMonitor.hh new file mode 100644 index 00000000000..edf865d6498 --- /dev/null +++ b/src/XrdPfc/XrdPfcResourceMonitor.hh @@ -0,0 +1,189 @@ +#ifndef __XRDPFC_RESOURCEMONITOR_HH__ +#define __XRDPFC_RESOURCEMONITOR_HH__ + +#include "XrdPfcStats.hh" + +#include "XrdSys/XrdSysPthread.hh" + +#include +#include + +class XrdOss; + +namespace XrdPfc { + +class DataFsState; +class DirState; +class FsTraversal; + +//============================================================================== +// ResourceMonitor +//============================================================================== + +// Encapsulates local variables used withing the previous mega-function Purge(). +// +// This will be used within the continuously/periodically ran heart-beat / breath +// function ... and then parts of it will be passed to invoked FS scan and purge +// jobs (which will be controlled throught this as well). + +// Andy: XRDADMINPATH Is the directory for administrative files (i.e. all.adminpath) +// Also: XrdOucEnv::Export("XRDLOGDIR", logParms.logfn); (in XrdOucLogging::configLog) + +class ResourceMonitor +{ + template + class Queue { + public: + struct Entry { + ID id; + RECORD record; + }; + using queue_type = std::vector; + using iterator = typename queue_type::iterator; + + Queue() = default; + + int write_queue_size() const { return m_write_queue.size(); } + bool read_queue_empty() const { return m_read_queue.empty(); } + int read_queue_size() const { return m_read_queue.size(); } + + // Writer / producer access + void push(ID id, RECORD stat) { m_write_queue.push_back({ id, stat }); } + + // Reader / consumer access + int swap_queues() { m_read_queue.clear(); m_write_queue.swap(m_read_queue); return read_queue_size(); } + const queue_type& read_queue() const { return m_read_queue; } + iterator begin() const { return m_read_queue.begin(); } + iterator end() const { return m_read_queue.end(); } + + private: + queue_type m_write_queue, m_read_queue; + }; + + struct AccessToken { + std::string m_filename; + DirState *m_dir_state = nullptr; + + void clear() { m_filename.clear(); m_dir_state = nullptr; } + }; + std::vector m_access_tokens; + std::vector m_access_tokens_free_slots; + + struct OpenRecord { + // new file? complete? + time_t m_open_time; + }; + + struct CloseRecord { + Stats m_stats; + time_t m_close_time; + }; + + struct PurgeRecord { + long long m_total_size; + int n_files; + }; + + Queue m_file_open_q; // filename is also in the token + Queue m_file_close_q; + Queue m_file_purge_q1; + Queue m_file_purge_q2; + Queue m_file_purge_q3; + // DirPurge queue -- not needed? But we do need last-change timestamp in DirState. + + // Updates from active files are pulled in from Cache. + std::vector> m_file_stats_update_vec; + + XrdSysMutex m_queue_mutex; // mutex shared between queues + + DataFsState *m_fs_state; + XrdOss &m_oss; + +public: + ResourceMonitor(XrdOss& oss); + ~ResourceMonitor(); + + // --- Initial scan, building of DirState tree + + void scan_dir_and_recurse(FsTraversal &fst); + bool perform_initial_scan(); + + // --- Event registration + + int register_file_open(const std::string& filename, time_t open_timestamp) { + // Simply return a token, we will resolve it in the actual processing of the queue. + XrdSysMutexHelper _lock(&m_queue_mutex); + int token; + if ( ! m_access_tokens_free_slots.empty()) { + token = m_access_tokens_free_slots.back(); + m_access_tokens_free_slots.pop_back(); + m_access_tokens[token].m_filename = filename; + } else { + token = (int) m_access_tokens.size(); + m_access_tokens.push_back({filename}); + } + + m_file_open_q.push(token, {open_timestamp}); + return token; + } + + void register_file_close(int token, Stats stats, time_t close_timestamp) { + XrdSysMutexHelper _lock(&m_queue_mutex); + m_file_close_q.push(token, {stats, close_timestamp}); + } + + // deletions can come from purge and from direct requests (Cache::UnlinkFile), the latter + // also covering the emergency shutdown of a file. + void register_file_purge(DirState* target, long long file_size) { + XrdSysMutexHelper _lock(&m_queue_mutex); + m_file_purge_q1.push(target, {file_size, 1}); + } + void register_multi_file_purge(DirState* target, long long file_size, int n_files) { + XrdSysMutexHelper _lock(&m_queue_mutex); + m_file_purge_q1.push(target, {file_size, n_files}); + } + void register_multi_file_purge(const std::string& target, long long file_size, int n_files) { + XrdSysMutexHelper _lock(&m_queue_mutex); + m_file_purge_q2.push(target, {file_size, n_files}); + } + void register_file_purge(const std::string& filename, long long file_size) { + XrdSysMutexHelper _lock(&m_queue_mutex); + m_file_purge_q3.push(filename, file_size); + } + + // void register_dir_purge(DirState* target); + // target assumed to be empty at this point, triggered by a file_purge removing the last file in it. + // hmmh, this is actually tricky ... who will purge the dirs? we should now at export-to-vector time + // and can prune leaf directories. Tthis might fail if a file has been created in there in the meantime, which is ok. + // However, is there a race condition between rmdir and creation of a new file in that dir? Ask Andy. + + // --- Helpers for event processing and actions + + AccessToken& token(int i) { return m_access_tokens[i]; } + + // --- Actions + + int process_queues(); + + void heart_beat(); + + /* XXX Stuff from Cache, to be revisited. + enum ScanAndPurgeThreadState_e { SPTS_Idle, SPTS_Scan, SPTS_Purge, SPTS_Done }; + + XrdSysCondVar m_stats_n_purge_cond; //!< communication between heart-beat and scan-purge threads + + int m_last_scan_duration; + int m_last_purge_duration; + ScanAndPurgeThreadState_e m_spt_state; + // --- + m_stats_n_purge_cond(0), + m_last_scan_duration(0), + m_last_purge_duration(0), + m_spt_state(SPTS_Idle) + + */ +}; + +} + +#endif diff --git a/src/XrdPfc/XrdPfcStats.hh b/src/XrdPfc/XrdPfcStats.hh index ae06f194dd2..c0da213b85f 100644 --- a/src/XrdPfc/XrdPfcStats.hh +++ b/src/XrdPfc/XrdPfcStats.hh @@ -19,38 +19,34 @@ // along with XRootD. If not, see . //---------------------------------------------------------------------------------- -#include "XrdOuc/XrdOucCache.hh" -#include "XrdSys/XrdSysPthread.hh" - namespace XrdPfc { + //---------------------------------------------------------------------------- //! Statistics of cache utilisation by a File object. +// Used both as aggregation of usage by a single file as well as for +// collecting per-directory statistics on time-interval basis. In this second +// case they are used as "deltas" ... differences in respect to a previous +// reference value. +// For running averages / deltas, one might need a version with doubles, so +// it might make sense to template this. And add some timestamp. //---------------------------------------------------------------------------- class Stats { public: - int m_NumIos; //!< number of IO objects attached during this access - int m_Duration; //!< total duration of all IOs attached - long long m_BytesHit; //!< number of bytes served from disk - long long m_BytesMissed; //!< number of bytes served from remote and cached - long long m_BytesBypassed; //!< number of bytes served directly through XrdCl - long long m_BytesWritten; //!< number of bytes written to disk - int m_NCksumErrors; //!< number of checksum errors while getting data from remote + int m_NumIos = 0; //!< number of IO objects attached during this access + int m_Duration = 0; //!< total duration of all IOs attached + long long m_BytesHit = 0; //!< number of bytes served from disk + long long m_BytesMissed = 0; //!< number of bytes served from remote and cached + long long m_BytesBypassed = 0; //!< number of bytes served directly through XrdCl + long long m_BytesWritten = 0; //!< number of bytes written to disk + int m_NCksumErrors = 0; //!< number of checksum errors while getting data from remote //---------------------------------------------------------------------- - Stats() : - m_NumIos (0), m_Duration(0), - m_BytesHit(0), m_BytesMissed(0), m_BytesBypassed(0), - m_BytesWritten(0), m_NCksumErrors(0) - {} + Stats() = default; - Stats(const Stats& s) : - m_NumIos (s.m_NumIos), m_Duration(s.m_Duration), - m_BytesHit(s.m_BytesHit), m_BytesMissed(s.m_BytesMissed), m_BytesBypassed(s.m_BytesBypassed), - m_BytesWritten(s.m_BytesWritten), m_NCksumErrors(s.m_NCksumErrors) - {} + Stats(const Stats& s) = default; Stats& operator=(const Stats&) = default; @@ -58,8 +54,6 @@ public: void AddReadStats(const Stats &s) { - XrdSysMutexHelper _lock(&m_Mutex); - m_BytesHit += s.m_BytesHit; m_BytesMissed += s.m_BytesMissed; m_BytesBypassed += s.m_BytesBypassed; @@ -67,45 +61,30 @@ public: void AddBytesHit(long long bh) { - XrdSysMutexHelper _lock(&m_Mutex); - m_BytesHit += bh; } void AddWriteStats(long long bytes_written, int n_cks_errs) { - XrdSysMutexHelper _lock(&m_Mutex); - m_BytesWritten += bytes_written; m_NCksumErrors += n_cks_errs; } void IoAttach() { - XrdSysMutexHelper _lock(&m_Mutex); - ++m_NumIos; } void IoDetach(int duration) { - XrdSysMutexHelper _lock(&m_Mutex); - m_Duration += duration; } - Stats Clone() - { - XrdSysMutexHelper _lock(&m_Mutex); - - return Stats(*this); - } //---------------------------------------------------------------------- void DeltaToReference(const Stats& ref) { - // Not locked, only used from Cache / Purge thread. m_NumIos = ref.m_NumIos - m_NumIos; m_Duration = ref.m_Duration - m_Duration; m_BytesHit = ref.m_BytesHit - m_BytesHit; @@ -117,7 +96,6 @@ public: void AddUp(const Stats& s) { - // Not locked, only used from Cache / Purge thread. m_NumIos += s.m_NumIos; m_Duration += s.m_Duration; m_BytesHit += s.m_BytesHit; @@ -129,7 +107,6 @@ public: void Reset() { - // Not locked, only used from Cache / Purge thread. m_NumIos = 0; m_Duration = 0; m_BytesHit = 0; @@ -138,11 +115,62 @@ public: m_BytesWritten = 0; m_NCksumErrors = 0; } +}; + +//============================================================================== + +class DirStats : public Stats +{ +public: + long long m_BytesRemoved = 0; + int m_NFilesOpened = 0; + int m_NFilesClosed = 0; + int m_NFilesCreated = 0; + int m_NFilesRemoved = 0; // purged + int m_NDirectoriesCreated = 0; // this is hard, oss does it for us ... but we MUSt know it for DirState creation. + int m_NDirectoriesRemoved = 0; + + //---------------------------------------------------------------------- + + DirStats() = default; + + DirStats(const DirStats& s) = default; + + DirStats& operator=(const DirStats&) = default; + + //---------------------------------------------------------------------- + + // maybe some missing AddSomething functions, like for read/write + + //---------------------------------------------------------------------- + + using Stats::DeltaToReference; // activate overload based on arg + void DeltaToReference(const DirStats& ref) + { + Stats::DeltaToReference(ref); + m_BytesRemoved = ref.m_BytesRemoved - m_BytesRemoved; + m_NFilesOpened = ref.m_NFilesOpened - m_NFilesOpened; + m_NFilesClosed = ref.m_NFilesClosed - m_NFilesClosed; + m_NFilesCreated = ref.m_NFilesCreated - m_NFilesCreated; + m_NFilesRemoved = ref.m_NFilesRemoved - m_NFilesRemoved; + m_NDirectoriesCreated = ref.m_NDirectoriesCreated - m_NDirectoriesCreated; + m_NDirectoriesRemoved = ref.m_NDirectoriesRemoved - m_NDirectoriesRemoved; + } -private: - XrdSysMutex m_Mutex; + using Stats::AddUp; // activate overload based on arg + void AddUp(const DirStats& s) + { + Stats::AddUp(s); + m_BytesRemoved += s.m_BytesRemoved; + m_NFilesOpened += s.m_NFilesOpened; + m_NFilesClosed += s.m_NFilesClosed; + m_NFilesCreated += s.m_NFilesCreated; + m_NFilesRemoved += s.m_NFilesRemoved; + m_NDirectoriesCreated += s.m_NDirectoriesCreated; + m_NDirectoriesRemoved += s.m_NDirectoriesRemoved; + } }; + } #endif - diff --git a/src/XrdPfc/XrdPfcTypes.hh b/src/XrdPfc/XrdPfcTypes.hh index 459b4959b86..a5c632c8f07 100644 --- a/src/XrdPfc/XrdPfcTypes.hh +++ b/src/XrdPfc/XrdPfcTypes.hh @@ -1,6 +1,6 @@ #ifndef __XRDPFC_TYPES_HH__ #define __XRDPFC_TYPES_HH__ -#include + //---------------------------------------------------------------------------------- // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman @@ -19,6 +19,9 @@ // along with XRootD. If not, see . //---------------------------------------------------------------------------------- +#include +#include + namespace XrdPfc { enum CkSumCheck_e { CSChk_Unknown = -1, CSChk_None = 0, CSChk_Cache = 1, CSChk_Net = 2, CSChk_Both = 3, From 995aedeeadcb5066da5d6e96b0bad0477e9777fd Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Thu, 28 Mar 2024 11:16:58 -0700 Subject: [PATCH 21/43] Consolidate open/close/purge processing in ResourceMonitor. - OpenRecord now contains bool telling ResMon if the file existed before. - DirState::find_path() and PathTokenizer can now return the last existing directory DirState in the hierarchy --> this allows ResMon to detect and report how many new subdirectories got created. - Cleanup queue processing code. - Start working on heart_beat() processing logick. --- src/XrdPfc/XrdPfcCommand.cc | 2 +- src/XrdPfc/XrdPfcDirState.cc | 12 +-- src/XrdPfc/XrdPfcDirState.hh | 10 ++- src/XrdPfc/XrdPfcFile.cc | 2 +- src/XrdPfc/XrdPfcResourceMonitor.cc | 110 +++++++++++++++++----------- src/XrdPfc/XrdPfcResourceMonitor.hh | 6 +- 6 files changed, 87 insertions(+), 55 deletions(-) diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index a483063fbf3..e312462080c 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -267,7 +267,7 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) m_writeQ.writes_between_purges += file_size; } { - int token = m_res_mon->register_file_open(file_path, time_now); + int token = m_res_mon->register_file_open(file_path, time_now, false); XrdPfc::Stats stats; stats.m_BytesWritten = file_size; m_res_mon->register_file_close(token, stats, time(0)); diff --git a/src/XrdPfc/XrdPfcDirState.cc b/src/XrdPfc/XrdPfcDirState.cc index ed59a3de67c..c0ea5b5479d 100644 --- a/src/XrdPfc/XrdPfcDirState.cc +++ b/src/XrdPfc/XrdPfcDirState.cc @@ -46,19 +46,21 @@ DirState* DirState::create_child(const std::string &dir) //! Internal function called from find_path //! @param dir subdir name //---------------------------------------------------------------------------- -DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs) +DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs, + DirState **last_existing_dir) { if (pos == pt.get_n_dirs()) return this; DsMap_i i = m_subdirs.find(pt.m_dirs[pos]); - DirState *ds = 0; + DirState *ds = nullptr; if (i != m_subdirs.end()) { ds = & i->second; + if (last_existing_dir) *last_existing_dir = ds; } - if (create_subdirs) + else if (create_subdirs) { ds = create_child(pt.m_dirs[pos]); } @@ -74,11 +76,11 @@ DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdir //! @param parse_as_lfn //! @param create_subdirs DirState* DirState::find_path(const std::string &path, int max_depth, bool parse_as_lfn, - bool create_subdirs) + bool create_subdirs, DirState **last_existing_dir) { PathTokenizer pt(path, max_depth, parse_as_lfn); - return find_path_tok(pt, 0, create_subdirs); + return find_path_tok(pt, 0, create_subdirs, last_existing_dir); } //---------------------------------------------------------------------------- diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh index e9e30be8c56..5c389b44757 100644 --- a/src/XrdPfc/XrdPfcDirState.hh +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -83,7 +83,8 @@ public: DirState* create_child(const std::string &dir); - DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs); + DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs, + DirState **last_existing_dir = nullptr); public: @@ -97,7 +98,8 @@ public: void add_up_stats(const Stats& stats); - DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs); + DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, + DirState **last_existing_dir = nullptr); DirState* find_dir(const std::string &dir, bool create_subdirs); @@ -128,9 +130,9 @@ public: DirState* get_root() { return & m_root; } - DirState* find_dirstate_for_lfn(const std::string& lfn) + DirState* find_dirstate_for_lfn(const std::string& lfn, DirState **last_existing_dir = nullptr) { - return m_root.find_path(lfn, -1, true, true); + return m_root.find_path(lfn, -1, true, true, last_existing_dir); } void reset_stats() { m_root.reset_stats(); } diff --git a/src/XrdPfc/XrdPfcFile.cc b/src/XrdPfc/XrdPfcFile.cc index f615c050022..bbd3545d3fe 100644 --- a/src/XrdPfc/XrdPfcFile.cc +++ b/src/XrdPfc/XrdPfcFile.cc @@ -488,7 +488,7 @@ bool File::Open() m_block_size = m_cfi.GetBufferSize(); m_num_blocks = m_cfi.GetNBlocks(); m_prefetch_state = (m_cfi.IsComplete()) ? kComplete : kStopped; // Will engage in AddIO(). - m_resmon_token = Cache::ResMon().register_file_open(m_filename, time(0)); + m_resmon_token = Cache::ResMon().register_file_open(m_filename, time(0), data_existed); // XXXX have some reporting counter that will trigger inter open stat reporting??? // Or keep pull mode? Hmmh, requires Cache::active_cond lock ... and can desync with close. m_state_cond.UnLock(); diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 7a5efd4a7e7..e7b518deaf3 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -113,6 +113,7 @@ int ResourceMonitor::process_queues() // We really want all open records to be processed before file-stats so let's get // the current snapshot of already opened files first. // On the other hand, we do not care if we miss some updates in this pass. + m_file_stats_update_vec.clear(); int n_records = Cache::GetInstance().CopyOutActiveStats(m_file_stats_update_vec); { XrdSysMutexHelper _lock(&m_queue_mutex); @@ -123,26 +124,37 @@ int ResourceMonitor::process_queues() n_records += m_file_purge_q3.swap_queues(); } - for (auto &i : m_file_open_q.read_queue()) { - // time is in stat, fname is in the token slot; + for (auto &i : m_file_open_q.read_queue()) + { + // i.id: LFN, i.record: OpenRecord int tid = i.id; AccessToken &at = token(tid); printf("process file open for token %d, time %ld -- %s\n", tid, i.record.m_open_time, at.m_filename.c_str()); - // At this point we need to resolve fname into DirState. - // We ould clear the filename after this ... or keep it, should we need it later on. - // - now just used for printing. - DirState *ds = m_fs_state->get_root()->find_path(at.m_filename, -1, true, true); + + // Resolve fname into DirState. + // We could clear the filename after this ... or keep it, should we need it later on. + // For now it is just used for debug printouts. + DirState *last_existing_ds = nullptr; + DirState *ds = m_fs_state->find_dirstate_for_lfn(at.m_filename, &last_existing_ds); at.m_dir_state = ds; ds->m_here_stats.m_NFilesOpened += 1; - // XXXXXX Here (or, a bit before) we should figure out how many new directories were created here. - // find_path could take an optional arg, last-dir-found -- would also help with purge troubles. - // This should then set, if needed NFiles/Dirs_Created. + // If this is a new file figure out how many new parent dirs got created along the way. + if ( ! i.record.m_existing_file) { + ds->m_here_stats.m_NFilesCreated += 1; + DirState *pp = ds; + while (pp != last_existing_ds) { + pp = pp->get_parent(); + pp->m_here_stats.m_NDirectoriesCreated += 1; + } + } ds->m_here_usage.m_last_open_time = i.record.m_open_time; } - for (auto &i : m_file_stats_update_vec) { + for (auto &i : m_file_stats_update_vec) + { + // i.first: token, i.second: Stats int tid = i.first; AccessToken &at = token(tid); // Stats @@ -152,11 +164,10 @@ int ResourceMonitor::process_queues() ds->m_here_stats.AddUp(i.second); } - m_file_stats_update_vec.clear(); - for (auto &i : m_file_close_q.read_queue()) { - // Stat and close_time are in the STAT == CloseRecord - // Remember to release the token !!! + for (auto &i : m_file_close_q.read_queue()) + { + // i.id: token, i.record: CloseRecord int tid = i.id; AccessToken &at = token(tid); printf("process file close for token %d, time %ld -- %s\n", @@ -164,47 +175,48 @@ int ResourceMonitor::process_queues() DirState *ds = at.m_dir_state; ds->m_here_stats.m_NFilesClosed += 1; + ds->m_here_stats.AddUp(i.record.m_stats); ds->m_here_usage.m_last_close_time = i.record.m_close_time; + // Release the AccessToken! at.clear(); m_access_tokens_free_slots.push_back(tid); } - for (auto &i : m_file_purge_q1.read_queue()) { - // ID is DirState*, multiple files + for (auto &i : m_file_purge_q1.read_queue()) + { + // i.id: DirState*, i.record: PurgeRecord DirState *ds = i.id; ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; ds->m_here_stats.m_NFilesRemoved += i.record.n_files; } - for (auto &i : m_file_purge_q2.read_queue()) { - // ID is directory-path, multiple files + for (auto &i : m_file_purge_q2.read_queue()) + { + // i.id: directory-path, i.record: PurgeRecord DirState *ds = m_fs_state->get_root()->find_path(i.id, -1, false, false); if ( ! ds) { TRACE(Error, trc_pfx << "DirState not found for directory path '" << i.id << "'."); - // XXX Should have find_path return the last dir / depth still found? + // find_path can return the last dir found ... but this clearly isn't a valid purge record. continue; } ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; ds->m_here_stats.m_NFilesRemoved += i.record.n_files; } - for (auto &i : m_file_purge_q3.read_queue()) { - // ID is LFN, single file + for (auto &i : m_file_purge_q3.read_queue()) + { + // i.id: LFN, i.record: size of file DirState *ds = m_fs_state->get_root()->find_path(i.id, -1, true, false); if ( ! ds) { TRACE(Error, trc_pfx << "DirState not found for LFN path '" << i.id << "'."); - // XXX Should have find_path return the last dir / depth still found? continue; } ds->m_here_stats.m_BytesRemoved += i.record; ds->m_here_stats.m_NFilesRemoved += 1; } - // XXXX Upward-propagate here or in heart_beat, as neeed? - - // XXXX dir purge record processing --- seems it is not needed, auto-purge with some cool-off. - - // XXXX clear the read queue, re-capacity to 50% if usage is below 25% + // Read queues / vectors are cleared at swap time. + // We might consider reducing their capacity by half if, say, their usage is below 25%. return n_records; } @@ -219,20 +231,40 @@ void ResourceMonitor::heart_beat() // initial scan performed as part of config + time_t now = time(0); + time_t next_queue_proc_time = now + 10; + time_t next_up_prop_time = now + 60; + while (true) { - int n_processed = process_queues(); + time_t start = time(0); + time_t next_event = std::min(next_queue_proc_time, next_up_prop_time); + if (next_event > start) + { + unsigned int t_sleep = next_event - start; + printf("sleeping for %u seconds, to be improved ...\n", t_sleep); + sleep(t_sleep); + } + int n_processed = process_queues(); + next_queue_proc_time += 10; printf("processed %d records\n", n_processed); - // check if more to process (just sum of sizes, not swap)? + if (next_up_prop_time > time(0)) + continue; + + m_fs_state->upward_propagate_stats(); + next_up_prop_time += 60; + + // Here we can learn assumed file-based usage + // run the "disk-usage" + // decide if age-based purge needs to be run (or uvkeep one) + // decide if the standard / plugin purge needs to be called - // check time, is it time to apply the deltas - m_fs_state->upward_propagate_stats(); // XXXXX before or after export? - // XXXXX ???? One could, maybe, just prop upwards the subdirs-only part .. and reset them. - // No, how could this work? - // check time, is it time to export into vector format, to disk /pfc-stats + // Check time, is it time to export into vector format, to disk, into /pfc-stats. + // This one should really be rather timely ... as it will be used for calculation + // of averages of stuff going on. // Dump statistcs before actual purging so maximum usage values get recorded. // Should really go to gstream --- and should really go from Heartbeat. @@ -241,18 +273,14 @@ void ResourceMonitor::heart_beat() m_fs_state->dump_recursively(Cache::Conf().m_dirStatsStoreDepth); } - m_fs_state->reset_stats(); // XXXXXX this is for sure after export, otherwise they will be zero + // XXXX + // m_fs_state->apply_stats_to_usages_and_reset_stats(); + // m_fs_state->reset_stats(); // XXXXXX this is for sure after export, otherwise they will be zero // check time / diskusage --> purge condition? // run purge as job or thread // m_fs_state->upward_propagate_usage_purged(); // XXXX this is the old way - - - // if no work, sleep for N seconds - unsigned int t_sleep = 10; - printf("sleeping for %u seconds, to be improved ...\n", t_sleep); - sleep(t_sleep); } } diff --git a/src/XrdPfc/XrdPfcResourceMonitor.hh b/src/XrdPfc/XrdPfcResourceMonitor.hh index edf865d6498..c7834426518 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.hh +++ b/src/XrdPfc/XrdPfcResourceMonitor.hh @@ -70,8 +70,8 @@ class ResourceMonitor std::vector m_access_tokens_free_slots; struct OpenRecord { - // new file? complete? time_t m_open_time; + bool m_existing_file; }; struct CloseRecord { @@ -110,7 +110,7 @@ public: // --- Event registration - int register_file_open(const std::string& filename, time_t open_timestamp) { + int register_file_open(const std::string& filename, time_t open_timestamp, bool existing_file) { // Simply return a token, we will resolve it in the actual processing of the queue. XrdSysMutexHelper _lock(&m_queue_mutex); int token; @@ -123,7 +123,7 @@ public: m_access_tokens.push_back({filename}); } - m_file_open_q.push(token, {open_timestamp}); + m_file_open_q.push(token, {open_timestamp, existing_file}); return token; } From 4339814cb54a472bd44674d3eb9511ab68d63478 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Thu, 18 Apr 2024 17:06:29 -0700 Subject: [PATCH 22/43] Consolidate read/write Stats update between File and ResourceMonitor. Avoid looping over all opened files to collect the stats every N-seconds. - ResourceMonitor: introduce also a Queue for Stats updates. - File: periodically report Stats updates to ResourceMonitor, when total bytes read accumulates to 2% of file size and on every Sync() to disk call. This could be furhter tuned at runtime as it is known if a file is over- or under-reporting within given update/processing time-windows. --- src/XrdPfc/XrdPfc.cc | 16 +------- src/XrdPfc/XrdPfc.hh | 3 -- src/XrdPfc/XrdPfcCommand.cc | 3 +- src/XrdPfc/XrdPfcDirState.cc | 29 +++++-------- src/XrdPfc/XrdPfcDirState.hh | 5 ++- src/XrdPfc/XrdPfcFile.cc | 64 ++++++++++++++++++----------- src/XrdPfc/XrdPfcFile.hh | 11 ++--- src/XrdPfc/XrdPfcFsTraversal.cc | 3 +- src/XrdPfc/XrdPfcResourceMonitor.cc | 22 +++++----- src/XrdPfc/XrdPfcResourceMonitor.hh | 60 +++++++++++++++++++-------- src/XrdPfc/XrdPfcStats.hh | 5 +++ 11 files changed, 123 insertions(+), 98 deletions(-) diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index f2c390d9c13..5bfca537bc3 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -503,22 +503,8 @@ void Cache::ReleaseFile(File* f, IO* io) dec_ref_cnt(f, true); } -int Cache::CopyOutActiveStats(std::vector> &store) -{ - XrdSysCondVarHelper lock(&m_active_cond); - int n = 0; - for (ActiveMap_i i = m_active.begin(); i != m_active.end(); ++i) - { - File *f = i->second; - if (f != 0) { - store.emplace_back(std::make_pair(f->GetResMonToken(), f->DeltaStatsFromLastCall())); - ++n; - } - } - return n; -} - +//============================================================================== //============================================================================== namespace diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index 75cefa27503..c52ef87c583 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -276,9 +276,6 @@ public: ResourceMonitor& RefResMon() { return *m_res_mon; } XrdXrootdGStream* GetGStream() { return m_gstream; } - int CopyOutActiveStats(std::vector> &store); - - void ExecuteCommandUrl(const std::string& command_url); static XrdScheduler *schedP; diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index e312462080c..43205f38647 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -270,7 +270,8 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) int token = m_res_mon->register_file_open(file_path, time_now, false); XrdPfc::Stats stats; stats.m_BytesWritten = file_size; - m_res_mon->register_file_close(token, stats, time(0)); + m_res_mon->register_file_update_stats(token, stats); + m_res_mon->register_file_close(token, time(0)); } } } diff --git a/src/XrdPfc/XrdPfcDirState.cc b/src/XrdPfc/XrdPfcDirState.cc index c0ea5b5479d..b312504836e 100644 --- a/src/XrdPfc/XrdPfcDirState.cc +++ b/src/XrdPfc/XrdPfcDirState.cc @@ -64,9 +64,9 @@ DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdir { ds = create_child(pt.m_dirs[pos]); } - if (ds) return ds->find_path_tok(pt, pos + 1, create_subdirs); + if (ds) return ds->find_path_tok(pt, pos + 1, create_subdirs, last_existing_dir); - return 0; + return nullptr; } //---------------------------------------------------------------------------- @@ -80,6 +80,8 @@ DirState* DirState::find_path(const std::string &path, int max_depth, bool parse { PathTokenizer pt(path, max_depth, parse_as_lfn); + if (last_existing_dir) *last_existing_dir = this; + return find_path_tok(pt, 0, create_subdirs, last_existing_dir); } @@ -97,12 +99,12 @@ DirState* DirState::find_dir(const std::string &dir, if (create_subdirs) return create_child(dir); - return 0; + return nullptr; } //---------------------------------------------------------------------------- //! Reset current transaction statistics. -//! Called from Cache::copy_out_active_stats_and_update_data_fs_state() +//! Called from ... to be seen if needed at all XXXX //---------------------------------------------------------------------------- void DirState::reset_stats() { @@ -117,7 +119,7 @@ void DirState::reset_stats() //---------------------------------------------------------------------------- //! Propagate stat to parents -//! Called from Cache::copy_out_active_stats_and_update_data_fs_state() +//! Called from ResourceMonitor::heart_beat() //---------------------------------------------------------------------------- void DirState::upward_propagate_stats() { @@ -125,14 +127,15 @@ void DirState::upward_propagate_stats() { i->second.upward_propagate_stats(); - // XXXXX m_stats.AddUp(i->second.m_stats); - // fix these for here_stats, there_stats + m_recursive_subdirs_stats.AddUp(i->second.m_recursive_subdirs_stats); + m_recursive_subdirs_stats.AddUp(i->second.m_here_stats); + // nothing to do for m_here_stats. } } //---------------------------------------------------------------------------- //! Update statistics. -//! Called from Purge thread. +//! Called from ... to be seen XXXX //---------------------------------------------------------------------------- long long DirState::upward_propagate_usage_purged() { @@ -176,14 +179,4 @@ void DirState::dump_recursively(const char *name, int max_depth) } } -//---------------------------------------------------------------------------- -//! Add to temporary Stat obj -//! -//---------------------------------------------------------------------------- -void DirState::add_up_stats(const Stats& stats) -{ - m_here_stats.AddUp(stats); - // XXXX propagate to parent done at the end. -} - } // end namespace diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh index 5c389b44757..41886d38ff3 100644 --- a/src/XrdPfc/XrdPfcDirState.hh +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -96,8 +96,6 @@ public: DirState* get_parent() { return m_parent; } - void add_up_stats(const Stats& stats); - DirState* find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, DirState **last_existing_dir = nullptr); @@ -121,6 +119,9 @@ class DataFsState { DirState m_root; time_t m_prev_time; + // XXXX To specify what time-stamps are needed, likely: + // - last Stats update time + // - last Usage update time (and Stats reset) public: DataFsState() : diff --git a/src/XrdPfc/XrdPfcFile.cc b/src/XrdPfc/XrdPfcFile.cc index bbd3545d3fe..0229acf2535 100644 --- a/src/XrdPfc/XrdPfcFile.cc +++ b/src/XrdPfc/XrdPfcFile.cc @@ -84,7 +84,7 @@ File::~File() TRACEF(Debug, "~File() close info "); m_info_file->Close(); delete m_info_file; - m_info_file = NULL; + m_info_file = nullptr; } if (m_data_file) @@ -92,11 +92,13 @@ File::~File() TRACEF(Debug, "~File() close output "); m_data_file->Close(); delete m_data_file; - m_data_file = NULL; + m_data_file = nullptr; } - if (m_resmon_token >= 0) { - Cache::ResMon().register_file_close(m_resmon_token, DeltaStatsFromLastCall(), time(0)); + if (m_resmon_token >= 0) + { + // Last update of file stats has been sent from the final Sync. + Cache::ResMon().register_file_close(m_resmon_token, time(0)); } TRACEF(Debug, "~File() ended, prefetch score = " << m_prefetch_score); @@ -146,18 +148,20 @@ void File::initiate_emergency_shutdown() //------------------------------------------------------------------------------ -Stats File::DeltaStatsFromLastCall() +void File::check_delta_stats() { - // Used for ResourceMonitor thread. + // Called under m_state_cond lock. + // BytesWritten indirectly trigger an unconditional merge through periodic Sync(). + if (m_delta_stats.BytesRead() >= m_resmon_report_threshold) + report_and_merge_delta_stats(); +} - Stats delta; - { - XrdSysCondVarHelper _lck(m_state_cond); - delta = m_last_stats; - m_last_stats = m_stats; - } - delta.DeltaToReference(m_last_stats); - return delta; +void File::report_and_merge_delta_stats() +{ + // Called under m_state_cond lock. + Cache::ResMon().register_file_update_stats(m_resmon_token, m_delta_stats); + m_stats.AddUp(m_delta_stats); + m_delta_stats.Reset(); } //------------------------------------------------------------------------------ @@ -286,6 +290,7 @@ bool File::FinalizeSyncBeforeExit() { if ( ! m_writes_during_sync.empty() || m_non_flushed_cnt > 0 || ! m_detach_time_logged) { + report_and_merge_delta_stats(); m_cfi.WriteIOStatDetach(m_stats); m_detach_time_logged = true; m_in_sync = true; @@ -316,7 +321,7 @@ void File::AddIO(IO *io) { m_io_set.insert(io); io->m_attach_time = now; - m_stats.IoAttach(); + m_delta_stats.IoAttach(); insert_remote_location(loc); @@ -355,7 +360,7 @@ void File::RemoveIO(IO *io) ++m_current_io; } - m_stats.IoDetach(now - io->m_attach_time); + m_delta_stats.IoDetach(now - io->m_attach_time); m_io_set.erase(mi); --m_ios_in_detach; @@ -488,9 +493,12 @@ bool File::Open() m_block_size = m_cfi.GetBufferSize(); m_num_blocks = m_cfi.GetNBlocks(); m_prefetch_state = (m_cfi.IsComplete()) ? kComplete : kStopped; // Will engage in AddIO(). + m_resmon_token = Cache::ResMon().register_file_open(m_filename, time(0), data_existed); - // XXXX have some reporting counter that will trigger inter open stat reporting??? - // Or keep pull mode? Hmmh, requires Cache::active_cond lock ... and can desync with close. + m_resmon_report_threshold = std::min(std::max(200ll * 1024, m_file_size / 50), 500ll * 1024 * 1024); + // m_resmon_report_threshold_scaler; // something like 10% of original threshold, to adjust + // actual threshold based on return values from register_file_update_stats(). + m_state_cond.UnLock(); return true; @@ -672,7 +680,8 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize, ReadR int ret = m_data_file->Read(iUserBuff, iUserOff, iUserSize); if (ret > 0) { XrdSysCondVarHelper _lck(m_state_cond); - m_stats.AddBytesHit(ret); + m_delta_stats.AddBytesHit(ret); + check_delta_stats(); } return ret; } @@ -704,7 +713,8 @@ int File::ReadV(IO *io, const XrdOucIOVec *readV, int readVnum, ReadReqRH *rh) int ret = m_data_file->ReadV(const_cast(readV), readVnum); if (ret > 0) { XrdSysCondVarHelper _lck(m_state_cond); - m_stats.AddBytesHit(ret); + m_delta_stats.AddBytesHit(ret); + check_delta_stats(); } return ret; } @@ -922,8 +932,8 @@ int File::ReadOpusCoalescere(IO *io, const XrdOucIOVec *readV, int readVnum, if (read_req->is_complete()) { // Almost like FinalizeReadRequest(read_req) -- but no callout! - m_stats.AddReadStats(read_req->m_stats); - + m_delta_stats.AddReadStats(read_req->m_stats); + check_delta_stats(); m_state_cond.UnLock(); int ret = read_req->return_value(); @@ -938,7 +948,8 @@ int File::ReadOpusCoalescere(IO *io, const XrdOucIOVec *readV, int readVnum, } else { - m_stats.m_BytesHit += bytes_read; + m_delta_stats.m_BytesHit += bytes_read; + check_delta_stats(); m_state_cond.UnLock(); // !!! No callout. @@ -1046,6 +1057,7 @@ void File::Sync() Stats loc_stats; { XrdSysCondVarHelper _lck(&m_state_cond); + report_and_merge_delta_stats(); loc_stats = m_stats; } m_cfi.WriteIOStat(loc_stats); @@ -1263,7 +1275,8 @@ void File::FinalizeReadRequest(ReadRequest *rreq) // NOT under lock -- does callout { XrdSysCondVarHelper _lck(m_state_cond); - m_stats.AddReadStats(rreq->m_stats); + m_delta_stats.AddReadStats(rreq->m_stats); + check_delta_stats(); } rreq->m_rh->Done(rreq->return_value()); @@ -1333,7 +1346,8 @@ void File::ProcessBlockResponse(Block *b, int res) { // Increase ref-count for the writer. inc_ref_count(b); - m_stats.AddWriteStats(b->get_size(), b->get_n_cksum_errors()); + m_delta_stats.AddWriteStats(b->get_size(), b->get_n_cksum_errors()); + // No check for writes, report-and-merge forced during Sync(). cache()->AddWriteTask(b, true); } diff --git a/src/XrdPfc/XrdPfcFile.hh b/src/XrdPfc/XrdPfcFile.hh index 501e8d8cd9d..e010182ecf8 100644 --- a/src/XrdPfc/XrdPfcFile.hh +++ b/src/XrdPfc/XrdPfcFile.hh @@ -278,9 +278,6 @@ public: void StopPrefetchingOnIO(IO *io); void RemoveIO(IO *io); - Stats DeltaStatsFromLastCall(); - int GetResMonToken() const { return m_resmon_token; } - std::string GetRemoteLocations() const; const Info::AStat* GetLastAccessStats() const { return m_cfi.GetLastAccessStats(); } size_t GetAccessCnt() const { return m_cfi.GetAccessCnt(); } @@ -346,11 +343,15 @@ private: long long m_block_size; int m_num_blocks; - // Stats + // Stats and ResourceMonitor interface Stats m_stats; //!< cache statistics for this instance - Stats m_last_stats; //!< copy of cache stats during last purge cycle, used for per directory stat reporting + Stats m_delta_stats; //!< unreported updates to stats int m_resmon_token; //!< token used in communication with the ResourceMonitor + long long m_resmon_report_threshold; + + void check_delta_stats(); + void report_and_merge_delta_stats(); std::set m_remote_locations; //!< Gathered in AddIO / ioUpdate / ioActive. void insert_remote_location(const std::string &loc); diff --git a/src/XrdPfc/XrdPfcFsTraversal.cc b/src/XrdPfc/XrdPfcFsTraversal.cc index f163f81961b..26ea261fb6f 100644 --- a/src/XrdPfc/XrdPfcFsTraversal.cc +++ b/src/XrdPfc/XrdPfcFsTraversal.cc @@ -36,8 +36,6 @@ bool FsTraversal::begin_traversal(DirState *root, const char *root_path) bool ret = begin_traversal(root_path); - m_maintain_dirstate = false; - return ret; } @@ -79,6 +77,7 @@ void FsTraversal::end_traversal() m_rel_dir_level = -1; m_root_dir_state = m_dir_state = nullptr; + m_maintain_dirstate = false; } //---------------------------------------------------------------------------- diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index e7b518deaf3..5f7fe1adab9 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -110,18 +110,21 @@ int ResourceMonitor::process_queues() static const char *trc_pfx = "process_queues() "; // Assure that we pick up only entries that are present now. - // We really want all open records to be processed before file-stats so let's get - // the current snapshot of already opened files first. - // On the other hand, we do not care if we miss some updates in this pass. - m_file_stats_update_vec.clear(); - int n_records = Cache::GetInstance().CopyOutActiveStats(m_file_stats_update_vec); + // We really want all open records to be processed before file-stats updates + // and all those before the close records. + // Purges are sort of tangential as they really just modify bytes / number + // of files in a direcotry and do not deal with any persistent file id tokens. + + int n_records = 0; { XrdSysMutexHelper _lock(&m_queue_mutex); n_records += m_file_open_q.swap_queues(); + n_records += m_file_update_stats_q.swap_queues(); n_records += m_file_close_q.swap_queues(); n_records += m_file_purge_q1.swap_queues(); n_records += m_file_purge_q2.swap_queues(); n_records += m_file_purge_q3.swap_queues(); + ++m_queue_swap_u1; } for (auto &i : m_file_open_q.read_queue()) @@ -152,17 +155,17 @@ int ResourceMonitor::process_queues() ds->m_here_usage.m_last_open_time = i.record.m_open_time; } - for (auto &i : m_file_stats_update_vec) + for (auto &i : m_file_update_stats_q.read_queue()) { - // i.first: token, i.second: Stats - int tid = i.first; + // i.id: token, i.record: Stats + int tid = i.id; AccessToken &at = token(tid); // Stats DirState *ds = at.m_dir_state; printf("process file update for token %d, %p -- %s\n", tid, ds, at.m_filename.c_str()); - ds->m_here_stats.AddUp(i.second); + ds->m_here_stats.AddUp(i.record); } for (auto &i : m_file_close_q.read_queue()) @@ -175,7 +178,6 @@ int ResourceMonitor::process_queues() DirState *ds = at.m_dir_state; ds->m_here_stats.m_NFilesClosed += 1; - ds->m_here_stats.AddUp(i.record.m_stats); ds->m_here_usage.m_last_close_time = i.record.m_close_time; diff --git a/src/XrdPfc/XrdPfcResourceMonitor.hh b/src/XrdPfc/XrdPfcResourceMonitor.hh index c7834426518..e499fe653ad 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.hh +++ b/src/XrdPfc/XrdPfcResourceMonitor.hh @@ -49,6 +49,8 @@ class ResourceMonitor // Writer / producer access void push(ID id, RECORD stat) { m_write_queue.push_back({ id, stat }); } + // Existing entry access for updating Stats + RECORD& write_record(int pos) { return m_write_queue[pos].record; } // Reader / consumer access int swap_queues() { m_read_queue.clear(); m_write_queue.swap(m_read_queue); return read_queue_size(); } @@ -62,9 +64,16 @@ class ResourceMonitor struct AccessToken { std::string m_filename; + unsigned int m_last_queue_swap_u1 = 0xffffffff; + int m_last_write_queue_pos = -1; DirState *m_dir_state = nullptr; - void clear() { m_filename.clear(); m_dir_state = nullptr; } + void clear() { + m_filename.clear(); + m_last_queue_swap_u1 = 0xffffffff; + m_last_write_queue_pos = -1; + m_dir_state = nullptr; + } }; std::vector m_access_tokens; std::vector m_access_tokens_free_slots; @@ -75,7 +84,6 @@ class ResourceMonitor }; struct CloseRecord { - Stats m_stats; time_t m_close_time; }; @@ -84,17 +92,16 @@ class ResourceMonitor int n_files; }; - Queue m_file_open_q; // filename is also in the token - Queue m_file_close_q; + Queue m_file_open_q; + Queue m_file_update_stats_q; + Queue m_file_close_q; Queue m_file_purge_q1; Queue m_file_purge_q2; Queue m_file_purge_q3; // DirPurge queue -- not needed? But we do need last-change timestamp in DirState. - // Updates from active files are pulled in from Cache. - std::vector> m_file_stats_update_vec; - - XrdSysMutex m_queue_mutex; // mutex shared between queues + XrdSysMutex m_queue_mutex; // mutex shared between queues + unsigned int m_queue_swap_u1 = 0u; // identifier of current swap DataFsState *m_fs_state; XrdOss &m_oss; @@ -113,23 +120,42 @@ public: int register_file_open(const std::string& filename, time_t open_timestamp, bool existing_file) { // Simply return a token, we will resolve it in the actual processing of the queue. XrdSysMutexHelper _lock(&m_queue_mutex); - int token; + int token_id; if ( ! m_access_tokens_free_slots.empty()) { - token = m_access_tokens_free_slots.back(); + token_id = m_access_tokens_free_slots.back(); m_access_tokens_free_slots.pop_back(); - m_access_tokens[token].m_filename = filename; + m_access_tokens[token_id].m_filename = filename; + m_access_tokens[token_id].m_last_write_queue_pos = m_queue_swap_u1 - 1; } else { - token = (int) m_access_tokens.size(); - m_access_tokens.push_back({filename}); + token_id = (int) m_access_tokens.size(); + m_access_tokens.push_back({filename, m_queue_swap_u1 - 1}); } - m_file_open_q.push(token, {open_timestamp, existing_file}); - return token; + m_file_open_q.push(token_id, {open_timestamp, existing_file}); + return token_id; + } + + void register_file_update_stats(int token_id, const Stats& stats) { + XrdSysMutexHelper _lock(&m_queue_mutex); + AccessToken &at = token(token_id); + // Check if this is the first update within this queue swap cycle. + if (at.m_last_queue_swap_u1 != m_queue_swap_u1) { + m_file_update_stats_q.push(token_id, stats); + at.m_last_queue_swap_u1 = m_queue_swap_u1; + at.m_last_write_queue_pos = m_file_update_stats_q.write_queue_size() - 1; + } else { + Stats &existing_stats = m_file_update_stats_q.write_record(at.m_last_write_queue_pos); + existing_stats.AddUp(stats); + } + // Optionally, one could return "scaler" to moodify stat-reporting + // frequency in the file ... if it comes too often or too rarely. + // See also the logic for determining reporting interval (in N_bytes_read) + // in File::Open(). } - void register_file_close(int token, Stats stats, time_t close_timestamp) { + void register_file_close(int token_id, time_t close_timestamp) { XrdSysMutexHelper _lock(&m_queue_mutex); - m_file_close_q.push(token, {stats, close_timestamp}); + m_file_close_q.push(token_id, {close_timestamp}); } // deletions can come from purge and from direct requests (Cache::UnlinkFile), the latter diff --git a/src/XrdPfc/XrdPfcStats.hh b/src/XrdPfc/XrdPfcStats.hh index c0da213b85f..7f2e14d4612 100644 --- a/src/XrdPfc/XrdPfcStats.hh +++ b/src/XrdPfc/XrdPfcStats.hh @@ -83,6 +83,11 @@ public: //---------------------------------------------------------------------- + long long BytesRead() const + { + return m_BytesHit + m_BytesMissed + m_BytesBypassed; + } + void DeltaToReference(const Stats& ref) { m_NumIos = ref.m_NumIos - m_NumIos; From 4a77edd18f1a32404ab8fd5e6d5713c7e3002385 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Fri, 26 Apr 2024 06:45:51 -0700 Subject: [PATCH 23/43] Near final implementation of DirState stats and usages handling. - Includes export vector snapshot form. - Support writing of vector snapshot in JSON form (to be used in an outside utility program). - Some white-spaces fixes. --- src/XrdPfc.cmake | 1 + src/XrdPfc/README | 8 +- src/XrdPfc/XrdPfc.cc | 6 +- src/XrdPfc/XrdPfc.hh | 4 +- src/XrdPfc/XrdPfcConfiguration.cc | 2 +- src/XrdPfc/XrdPfcDirState.cc | 242 +++++++++++++++++---------- src/XrdPfc/XrdPfcDirState.hh | 164 +++++++++++------- src/XrdPfc/XrdPfcDirStateSnapshot.cc | 84 ++++++++++ src/XrdPfc/XrdPfcDirStateSnapshot.hh | 43 +++++ src/XrdPfc/XrdPfcFPurgeState.cc | 2 +- src/XrdPfc/XrdPfcFSctl.cc | 14 +- src/XrdPfc/XrdPfcFile.hh | 8 +- src/XrdPfc/XrdPfcIOFile.hh | 2 +- src/XrdPfc/XrdPfcIOFileBlock.hh | 2 +- src/XrdPfc/XrdPfcInfo.cc | 2 +- src/XrdPfc/XrdPfcInfo.hh | 2 +- src/XrdPfc/XrdPfcPurgeQuota.cc | 208 +++++++++++------------ src/XrdPfc/XrdPfcResourceMonitor.cc | 77 +++++++-- src/XrdPfc/XrdPfcResourceMonitor.hh | 11 +- src/XrdPfc/XrdPfcStats.hh | 4 +- 20 files changed, 590 insertions(+), 296 deletions(-) create mode 100644 src/XrdPfc/XrdPfcDirStateSnapshot.cc create mode 100644 src/XrdPfc/XrdPfcDirStateSnapshot.hh diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index ce7784aa99d..7c48ff9c822 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -21,6 +21,7 @@ add_library( XrdPfc/XrdPfc.cc XrdPfc/XrdPfc.hh XrdPfc/XrdPfcConfiguration.cc XrdPfc/XrdPfcDirState.cc XrdPfc/XrdPfcDirState.hh + XrdPfc/XrdPfcDirStateSnapshot.cc XrdPfc/XrdPfcDirStateSnapshot.hh XrdPfc/XrdPfcFPurgeState.cc XrdPfc/XrdPfcFPurgeState.hh XrdPfc/XrdPfcPurge.cc XrdPfc/XrdPfcPurgePin.hh diff --git a/src/XrdPfc/README b/src/XrdPfc/README index 488b31eeb28..673c728088c 100644 --- a/src/XrdPfc/README +++ b/src/XrdPfc/README @@ -114,7 +114,7 @@ CONFIGURATION pfc.blocksize: prefetch buffer size, default 1M -pfc.ram [bytes[g]]: maximum allowed RAM usage for caching proxy +pfc.ram [bytes[g]]: maximum allowed RAM usage for caching proxy pfc.prefetch : prefetch level, default is 10. Value zero disables prefetching. @@ -122,16 +122,16 @@ pfc.diskusage diskusage boundaries, can be specified relative in per pfc.user : username used by XrdOss plugin -pfc.filefragmentmode [fragmentsize ] -- enable prefetching a unit of a file, +pfc.filefragmentmode [fragmentsize ] -- enable prefetching a unit of a file, with default block size -pfc.osslib [] path to alternative plign for output file system +pfc.osslib [] path to alternative plign for output file system pfc.decisionlib [] path to decision library and plugin parameters pfc.trace default level is warning, xrootd option -d sets debug level -Examples +Examples a) Enable proxy file prefetching: pps.cachelib libXrdPfc.so diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index 5bfca537bc3..46dd75eed4d 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -411,7 +411,7 @@ void Cache::ReleaseRAM(char* buf, long long size) File* Cache::GetFile(const std::string& path, IO* io, long long off, long long filesize) { // Called from virtual IO::Attach - + TRACE(Debug, "GetFile " << path << ", io " << io); ActiveMap_i it; @@ -492,9 +492,9 @@ File* Cache::GetFile(const std::string& path, IO* io, long long off, long long f void Cache::ReleaseFile(File* f, IO* io) { // Called from virtual IO::DetachFinalize. - + TRACE(Debug, "ReleaseFile " << f->GetLocalPath() << ", io " << io); - + { XrdSysCondVarHelper lock(&m_active_cond); diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index c52ef87c583..a9d43d3d531 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -261,7 +261,7 @@ public: XrdOss* GetOss() const { return m_oss; } bool IsFileActiveOrPurgeProtected(const std::string&); - + File* GetFile(const std::string&, IO*, long long off = 0, long long filesize = 0); void ReleaseFile(File*, IO*); @@ -269,7 +269,7 @@ public: void ScheduleFileSync(File* f) { schedule_file_sync(f, false, false); } void FileSyncDone(File*, bool high_debug); - + XrdSysError* GetLog() { return &m_log; } XrdSysTrace* GetTrace() { return m_trace; } diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index 022a0fcb38d..bc9305aca69 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -516,7 +516,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) } // Setup number of standard-size blocks not released back to the system to 5% of total RAM. m_configuration.m_RamKeepStdBlocks = (m_configuration.m_RamAbsAvailable / m_configuration.m_bufferSize + 1) * 5 / 100; - + // Set tracing to debug if this is set in environment char* cenv = getenv("XRDDEBUG"); diff --git a/src/XrdPfc/XrdPfcDirState.cc b/src/XrdPfc/XrdPfcDirState.cc index b312504836e..c94a094d6b7 100644 --- a/src/XrdPfc/XrdPfcDirState.cc +++ b/src/XrdPfc/XrdPfcDirState.cc @@ -3,8 +3,7 @@ #include -namespace XrdPfc -{ +using namespace XrdPfc; //---------------------------------------------------------------------------- //! Constructor @@ -17,8 +16,8 @@ DirState::DirState() : m_parent(0), m_depth(0) //! @param DirState parent directory //---------------------------------------------------------------------------- DirState::DirState(DirState *parent) : - m_parent(parent), - m_depth(m_parent->m_depth + 1) + m_parent(parent), + m_depth(m_parent->m_depth + 1) {} //---------------------------------------------------------------------------- @@ -26,47 +25,51 @@ DirState::DirState(DirState *parent) : //! @param parent parent DirState object //! @param dname name of this directory only, no slashes, no extras. //---------------------------------------------------------------------------- -DirState::DirState(DirState *parent, const std::string& dname) : - m_parent(parent), - m_dir_name(dname), - m_depth(m_parent->m_depth + 1) +DirState::DirState(DirState *parent, const std::string &dname) : + DirStateBase(dname), + m_parent(parent), + m_depth(m_parent->m_depth + 1) {} //---------------------------------------------------------------------------- //! Internal function called from find_dir or find_path_tok //! @param dir subdir name //---------------------------------------------------------------------------- -DirState* DirState::create_child(const std::string &dir) +DirState *DirState::create_child(const std::string &dir) { - std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this, dir))); - return & ir.first->second; + std::pair ir = m_subdirs.insert(std::make_pair(dir, DirState(this, dir))); + return &ir.first->second; } //---------------------------------------------------------------------------- //! Internal function called from find_path //! @param dir subdir name //---------------------------------------------------------------------------- -DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs, +DirState *DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs, DirState **last_existing_dir) { - if (pos == pt.get_n_dirs()) return this; + if (pos == pt.get_n_dirs()) + return this; + + DirState *ds = nullptr; - DsMap_i i = m_subdirs.find(pt.m_dirs[pos]); + DsMap_i i = m_subdirs.find(pt.m_dirs[pos]); - DirState *ds = nullptr; + if (i != m_subdirs.end()) + { + ds = &i->second; + if (last_existing_dir) + *last_existing_dir = ds; + } + else if (create_subdirs) + { + ds = create_child(pt.m_dirs[pos]); + } - if (i != m_subdirs.end()) - { - ds = & i->second; - if (last_existing_dir) *last_existing_dir = ds; - } - else if (create_subdirs) - { - ds = create_child(pt.m_dirs[pos]); - } - if (ds) return ds->find_path_tok(pt, pos + 1, create_subdirs, last_existing_dir); + if (ds) + return ds->find_path_tok(pt, pos + 1, create_subdirs, last_existing_dir); - return nullptr; + return nullptr; } //---------------------------------------------------------------------------- @@ -74,15 +77,16 @@ DirState* DirState::find_path_tok(PathTokenizer &pt, int pos, bool create_subdir //! @param path full path to parse //! @param max_depth directory depth to which to descend (value < 0 means full descent) //! @param parse_as_lfn -//! @param create_subdirs -DirState* DirState::find_path(const std::string &path, int max_depth, bool parse_as_lfn, +//! @param create_subdirs +DirState *DirState::find_path(const std::string &path, int max_depth, bool parse_as_lfn, bool create_subdirs, DirState **last_existing_dir) { - PathTokenizer pt(path, max_depth, parse_as_lfn); + PathTokenizer pt(path, max_depth, parse_as_lfn); - if (last_existing_dir) *last_existing_dir = this; + if (last_existing_dir) + *last_existing_dir = this; - return find_path_tok(pt, 0, create_subdirs, last_existing_dir); + return find_path_tok(pt, 0, create_subdirs, last_existing_dir); } //---------------------------------------------------------------------------- @@ -90,93 +94,155 @@ DirState* DirState::find_path(const std::string &path, int max_depth, bool parse //! @param dir subdir name @param bool create the subdir in this DirsStat //! @param create_subdirs if true and the dir is not found, a new DirState //! child is created -DirState* DirState::find_dir(const std::string &dir, +DirState *DirState::find_dir(const std::string &dir, bool create_subdirs) { - DsMap_i i = m_subdirs.find(dir); + DsMap_i i = m_subdirs.find(dir); - if (i != m_subdirs.end()) return & i->second; + if (i != m_subdirs.end()) + return &i->second; - if (create_subdirs) return create_child(dir); + if (create_subdirs) + return create_child(dir); - return nullptr; + return nullptr; } //---------------------------------------------------------------------------- -//! Reset current transaction statistics. -//! Called from ... to be seen if needed at all XXXX +//! Propagate stat to parents +//! Called from ResourceMonitor::heart_beat() //---------------------------------------------------------------------------- -void DirState::reset_stats() +void DirState::upward_propagate_stats_and_times() { - m_here_stats.Reset(); - m_recursive_subdirs_stats.Reset(); + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.upward_propagate_stats_and_times(); + + m_recursive_subdir_stats.AddUp(i->second.m_recursive_subdir_stats); + m_recursive_subdir_stats.AddUp(i->second.m_here_stats); + // nothing to do for m_here_stats. + + m_recursive_subdir_usage.update_last_times(i->second.m_recursive_subdir_usage); + m_recursive_subdir_usage.update_last_times(i->second.m_here_usage); + } +} - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.reset_stats(); - } +void DirState::apply_stats_to_usages() +{ + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.apply_stats_to_usages(); + } + m_here_usage.update_from_stats(m_here_stats); + m_recursive_subdir_usage.update_from_stats(m_recursive_subdir_stats); } //---------------------------------------------------------------------------- -//! Propagate stat to parents -//! Called from ResourceMonitor::heart_beat() +//! Reset current transaction statistics. +//! Called from ... to be seen if needed at all XXXX //---------------------------------------------------------------------------- -void DirState::upward_propagate_stats() +void DirState::reset_stats() { - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.upward_propagate_stats(); - - m_recursive_subdirs_stats.AddUp(i->second.m_recursive_subdirs_stats); - m_recursive_subdirs_stats.AddUp(i->second.m_here_stats); - // nothing to do for m_here_stats. - } + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + i->second.reset_stats(); + } + m_here_stats.Reset(); + m_recursive_subdir_stats.Reset(); } //---------------------------------------------------------------------------- //! Update statistics. //! Called from ... to be seen XXXX //---------------------------------------------------------------------------- +// attic long long DirState::upward_propagate_usage_purged() { - // XXXXX what's with this? -/* - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - m_usage_purged += i->second.upward_propagate_usage_purged(); - } - m_usage -= m_usage_purged; - - long long ret = m_usage_purged; - m_usage_purged = 0; - return ret; -*/ - return 0; + // XXXXX what's with this? Should be automatic through the queues now .... + /* + for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) + { + m_usage_purged += i->second.upward_propagate_usage_purged(); + } + m_usage -= m_usage_purged; + + long long ret = m_usage_purged; + m_usage_purged = 0; + return ret; + */ + return 0; +} + + +int DirState::count_dirs_to_level(int max_depth) const +{ + int n_dirs = 1; + if (m_depth < max_depth) + { + for (auto & [name, ds] : m_subdirs) + { + n_dirs += ds.count_dirs_to_level(max_depth); + } + } + return n_dirs; } //---------------------------------------------------------------------------- //! Recursive print of statistics. Called if defined in pfc configuration. //! //---------------------------------------------------------------------------- -void DirState::dump_recursively(const char *name, int max_depth) +void DirState::dump_recursively(const char *name, int max_depth) const +{ + printf("%*d %s usage_here=%lld usage_sub=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", + 2 + 2 * m_depth, m_depth, name, + m_here_usage.m_BytesOnDisk, m_recursive_subdir_usage.m_BytesOnDisk, + m_here_usage.m_BytesOnDisk + m_recursive_subdir_usage.m_BytesOnDisk, + // XXXXX here_stats or sum up? or both? + m_here_stats.m_NumIos, m_here_stats.m_Duration, + m_here_stats.m_BytesHit, m_here_stats.m_BytesMissed, m_here_stats.m_BytesBypassed, + m_here_stats.m_BytesWritten); + + if (m_depth < max_depth) + { + for (auto & [name, ds] : m_subdirs) + { + ds.dump_recursively(name.c_str(), max_depth); + } + } +} + + +//============================================================================== +// DataFsState +//============================================================================== + +void DataFsState::upward_propagate_stats_and_times() +{ + m_root.upward_propagate_stats_and_times(); +} + +void DataFsState::apply_stats_to_usages() { - printf("%*d %s usage=%lld usage_extra=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", - 2 + 2*m_depth, m_depth, name, - // XXXXX decide what goes here - // m_usage, m_usage_extra, m_usage + m_usage_extra, - 0ll, 0ll, 0ll, - // XXXXX here_stats or sum up? - m_here_stats.m_NumIos, m_here_stats.m_Duration, - m_here_stats.m_BytesHit, m_here_stats.m_BytesMissed, m_here_stats.m_BytesBypassed, - m_here_stats.m_BytesWritten); - - if (m_depth >= max_depth) - return; - - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - i->second.dump_recursively(i->first.c_str(), max_depth); - } + m_usage_update_time = time(0); + m_root.apply_stats_to_usages(); } -} // end namespace +void DataFsState::reset_stats() +{ + m_root.reset_stats(); + m_stats_reset_time = time(0); +} + +void DataFsState::dump_recursively(int max_depth) const +{ + if (max_depth < 0) + max_depth = 4096; + time_t now = time(0); + + printf("DataFsState::dump_recursively epoch = %lld delta_t = %lld, max_dump_depth = %d\n", + (long long)now, (long long)(now - m_prev_time), max_depth); + + m_prev_time = now; + + m_root.dump_recursively("root", max_depth); +} diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh index 41886d38ff3..632069730ff 100644 --- a/src/XrdPfc/XrdPfcDirState.hh +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -7,51 +7,116 @@ #include #include -namespace XrdPfc -{ - -class PathTokenizer; //============================================================================== // Manifest: -// 1. class DirState -- state of a directory, including current delta-stats -// 2. class DataFSState -- manager of the DirState tree, starting from root (as in "/"). +//------------------------------------------------------------------------------ +// - Data-holding struct DirUsage -- complementary to Stats. +// - Base classes for DirState and DataFsState, shared between in-memory +// tree form and snap-shot vector form. +// - Structs for DirState export in vector form: +// - struct DirStateElement, and +// - struct DataFsSnapshot. +// Those should probably go to another .hh/.cc so the object file can be included +// the dedicated binary for processing of the binary dumps. +// - class DirState -- state of a directory, including current delta-stats. +// - class DataFSState -- manager of the DirState tree, starting from root (as in "/"). +// +// Structs for DirState export in vector form (DirStateElement and DataFsSnapshot) +// are declared in XrdPfcDirStateSnapshot.hh. +//============================================================================== + + +namespace XrdPfc +{ +class PathTokenizer; //============================================================================== -// DirState +// Data-holding struct DirUsage -- complementary to Stats. //============================================================================== struct DirUsage { - time_t m_last_open_time = 0; - time_t m_last_close_time = 0; - long long m_bytes_on_disk = 0; - int m_num_files_open = 0; - int m_num_files = 0; - int m_num_subdirs = 0; + time_t m_LastOpenTime = 0; + time_t m_LastCloseTime = 0; + long long m_BytesOnDisk = 0; + int m_NFilesOpen = 0; + int m_NFiles = 0; + int m_NDirectories = 0; + + void update_from_stats(const DirStats& s) + { + m_BytesOnDisk += s.m_BytesWritten - s.m_BytesRemoved; + m_NFilesOpen += s.m_NFilesOpened - s.m_NFilesClosed; + m_NFiles += s.m_NFilesCreated - s.m_NFilesRemoved; + m_NDirectories += s.m_NDirectoriesCreated - s.m_NDirectoriesRemoved; + } + + void update_last_times(const DirUsage& u) + { + m_LastOpenTime = std::max(m_LastOpenTime, u.m_LastOpenTime); + m_LastCloseTime = std::max(m_LastCloseTime, u.m_LastCloseTime); + } }; -class DirState + +//============================================================================== +// Base classes, shared between in-memory tree form and snap-shot vector form. +//============================================================================== + +struct DirStateBase { -public: - DirState *m_parent = nullptr; std::string m_dir_name; DirStats m_here_stats; - DirStats m_recursive_subdirs_stats; + DirStats m_recursive_subdir_stats; DirUsage m_here_usage; DirUsage m_recursive_subdir_usage; + + DirStateBase() {} + DirStateBase(const std::string &dname) : m_dir_name(dname) {} + + const DirUsage& recursive_subdir_usage() const { return m_recursive_subdir_usage; } +}; + +struct DataFsStateBase +{ + time_t m_usage_update_time = 0; + time_t m_stats_reset_time = 0; + + // FS usage and available space information +}; + + +//============================================================================== +// Structs for DirState export in vector form +//============================================================================== + +struct DirStateElement; +struct DataFsSnapshot; + +//============================================================================== +// DirState +//============================================================================== + +struct DirState : public DirStateBase +{ + typedef std::map DsMap_t; + typedef DsMap_t::iterator DsMap_i; + + DirState *m_parent = nullptr; + DsMap_t m_subdirs; + + int m_depth; + // bool m_stat_report; // not used yet - storing of stats requested; might also need depth + // XXX int m_possible_discrepancy; // num detected possible inconsistencies. here, subdirs? // flag - can potentially be inaccurate -- plus timestamp of it (min or max, if several for subdirs)? - // + the same kind for resursive sums of all subdirs (but not in this dir). - // Thus, to get recursive totals of here+all_subdirs, one need to add them up. - // Have extra members or define an intermediate structure? - // Do we need running averages of these, too, not just traffic? // Snapshot should be fine, no? @@ -64,20 +129,8 @@ public: // inner node upwards. Also, in this case, is it really the best idea to have // map as daughter container? It keeps them sorted for export :) - // m_stats should be separated by "here" and "daughters_recursively", too. - - // begin purge traversal usage \_ so we can have a good estimate of what came in during the traversal - // end purge traversal usage / (should be small, presumably) - // quota info, enabled? - int m_depth; - bool m_stat_report; // not used yet - storing of stats requested - - typedef std::map DsMap_t; - typedef DsMap_t::iterator DsMap_i; - - DsMap_t m_subdirs; void init(); @@ -86,7 +139,6 @@ public: DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs, DirState **last_existing_dir = nullptr); -public: DirState(); @@ -101,13 +153,17 @@ public: DirState* find_dir(const std::string &dir, bool create_subdirs); + + void upward_propagate_stats_and_times(); + void apply_stats_to_usages(); void reset_stats(); - void upward_propagate_stats(); + // attic + long long upward_propagate_usage_purged(); // why would this be any different? isn't it included in stats? - long long upward_propagate_usage_purged(); + int count_dirs_to_level(int max_depth) const; - void dump_recursively(const char *name, int max_depth); + void dump_recursively(const char *name, int max_depth) const; }; @@ -115,44 +171,32 @@ public: // DataFsState //============================================================================== -class DataFsState +struct DataFsState : public DataFsStateBase { - DirState m_root; - time_t m_prev_time; - // XXXX To specify what time-stamps are needed, likely: - // - last Stats update time - // - last Usage update time (and Stats reset) + DirState m_root; + mutable time_t m_prev_time; + -public: DataFsState() : m_root (), m_prev_time (time(0)) {} - DirState* get_root() { return & m_root; } + DirState* get_root() { return & m_root; } DirState* find_dirstate_for_lfn(const std::string& lfn, DirState **last_existing_dir = nullptr) { return m_root.find_path(lfn, -1, true, true, last_existing_dir); } - void reset_stats() { m_root.reset_stats(); } - void upward_propagate_stats() { m_root.upward_propagate_stats(); } - void upward_propagate_usage_purged() { m_root.upward_propagate_usage_purged(); } - - void dump_recursively(int max_depth) - { - if (max_depth < 0) - max_depth = 4096; - time_t now = time(0); - - printf("DataFsState::dump_recursively epoch = %lld delta_t = %lld, max_dump_depth = %d\n", - (long long) now, (long long) (now - m_prev_time), max_depth); + void upward_propagate_stats_and_times(); + void apply_stats_to_usages(); + void reset_stats(); - m_prev_time = now; + // attic + void upward_propagate_usage_purged() { m_root.upward_propagate_usage_purged(); } - m_root.dump_recursively("root", max_depth); - } + void dump_recursively(int max_depth) const; }; } diff --git a/src/XrdPfc/XrdPfcDirStateSnapshot.cc b/src/XrdPfc/XrdPfcDirStateSnapshot.cc new file mode 100644 index 00000000000..2ce0d95ea3f --- /dev/null +++ b/src/XrdPfc/XrdPfcDirStateSnapshot.cc @@ -0,0 +1,84 @@ +#include "XrdPfcDirStateSnapshot.hh" + +#include "XrdOuc/XrdOucJson.hh" + +#include +#include +#include + + +// Redefine to also support ordered_json ... we want to keep variable order in JSON save files. +#define PFC_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + inline void to_json(nlohmann::json &nlohmann_json_j, const Type &nlohmann_json_t) { \ + NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) \ + } \ + inline void from_json(const nlohmann::json &nlohmann_json_j, Type &nlohmann_json_t) { \ + NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) \ + } \ + inline void to_json(nlohmann::ordered_json &nlohmann_json_j, const Type &nlohmann_json_t) { \ + NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) \ + } \ + inline void from_json(const nlohmann::ordered_json &nlohmann_json_j, Type &nlohmann_json_t) { \ + NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) \ + } + +namespace XrdPfc +{ +PFC_DEFINE_TYPE_NON_INTRUSIVE(DirStats, + m_NumIos, m_Duration, m_BytesHit, m_BytesMissed, m_BytesBypassed, m_BytesWritten, m_NCksumErrors, + m_BytesRemoved, m_NFilesOpened, m_NFilesClosed, m_NFilesCreated, m_NFilesRemoved, m_NDirectoriesCreated, m_NDirectoriesRemoved) +PFC_DEFINE_TYPE_NON_INTRUSIVE(DirUsage, + m_LastOpenTime, m_LastCloseTime, m_BytesOnDisk, m_NFilesOpen, m_NFiles, m_NDirectories) +PFC_DEFINE_TYPE_NON_INTRUSIVE(DirStateElement, + m_dir_name, m_here_stats, m_recursive_subdir_stats, m_here_usage, m_recursive_subdir_usage, + m_parent, m_daughters_begin, m_daughters_end) +PFC_DEFINE_TYPE_NON_INTRUSIVE(DataFsSnapshot, + m_usage_update_time, m_stats_reset_time, + m_dir_states) +} + +namespace +{ +// Open file for writing, throw exception on failure. +void open_ofstream(std::ofstream &ofs, const std::string &fname, const char *pfx = nullptr) { + ofs.open(fname, std::ofstream::trunc); + if (!ofs) { + char m[2048]; + snprintf(m, 2048, "%s%sError opening %s for write: %m", pfx ? pfx : "", pfx ? " " : "", fname.c_str()); + throw std::runtime_error(m); + } +} +} + +using namespace XrdPfc; + +void DataFsSnapshot::write_json_file(const std::string &fname, bool include_preamble) +{ + std::ofstream ofs; + open_ofstream(ofs, fname, __func__); + + if (include_preamble) { + ofs << "{ \"dirstate_snapshot\": "; + } + + nlohmann::ordered_json j; + to_json(j, *this); + + ofs << std::setw(1); + ofs << j; + + if (include_preamble) { + ofs << " }"; + } + + ofs << "\n"; + ofs.close(); + return 0; +} + +void DataFsSnapshot::dump() +{ + nlohmann::ordered_json j; // = *this; + to_json(j, *this); + std::cout << j.dump(3) << "\n"; +} diff --git a/src/XrdPfc/XrdPfcDirStateSnapshot.hh b/src/XrdPfc/XrdPfcDirStateSnapshot.hh new file mode 100644 index 00000000000..86155d5e086 --- /dev/null +++ b/src/XrdPfc/XrdPfcDirStateSnapshot.hh @@ -0,0 +1,43 @@ +#ifndef __XRDPFC_DIRSTATESNAPSHOT_HH__ +#define __XRDPFC_DIRSTATESNAPSHOT_HH__ + +#include "XrdPfcDirState.hh" + +#include + +//============================================================================== +// Structs for DirState export in vector form +//============================================================================== + +namespace XrdPfc +{ + +struct DirStateElement : public DirStateBase +{ + int m_parent = -1; + int m_daughters_begin = -1, m_daughters_end = -1; + + DirStateElement() {} + DirStateElement(const DirStateBase &b, int parent) : + DirStateBase(b), + m_parent(parent) + {} +}; + +struct DataFsSnapshot : public DataFsStateBase +{ + std::vector m_dir_states; + + DataFsSnapshot() {} + DataFsSnapshot(const DataFsStateBase &b) : DataFsStateBase(b) {} + + // Import of data into vector form is implemented in ResourceMonitor + // in order to avoid dependence of this struct on DirState. + + void write_json_file(const std::string &fname, bool include_preamble); + void dump(); +}; + +} + +#endif diff --git a/src/XrdPfc/XrdPfcFPurgeState.cc b/src/XrdPfc/XrdPfcFPurgeState.cc index b71da82b811..759032b2c4f 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.cc +++ b/src/XrdPfc/XrdPfcFPurgeState.cc @@ -49,7 +49,7 @@ void FPurgeState::MoveListEntriesToMap() } //---------------------------------------------------------------------------- -//! Open info file. Look at the UV stams and last access time. +//! Open info file. Look at the UV stams and last access time. //! Store the file in sorted map or in a list.s //! @param fname name of cache-info file //! @param Info object diff --git a/src/XrdPfc/XrdPfcFSctl.cc b/src/XrdPfc/XrdPfcFSctl.cc index 2298bfd75b5..bde0b5840de 100644 --- a/src/XrdPfc/XrdPfcFSctl.cc +++ b/src/XrdPfc/XrdPfcFSctl.cc @@ -49,7 +49,7 @@ XrdPfcFSctl::XrdPfcFSctl(XrdPfc::Cache &cInst, XrdSysLogger *logP) : myCache(cInst), hProc(0), Log(logP, "PfcFsctl"), sysTrace(cInst.GetTrace()), m_traceID("PfcFSctl") {} - + /******************************************************************************/ /* C o n f i g u r e */ /******************************************************************************/ @@ -64,11 +64,11 @@ bool XrdPfcFSctl::Configure(const char *CfgFN, hProc = (XrdOfsHandle*)envP->GetPtr("XrdOfsHandle*"); return hProc != 0; } - + /******************************************************************************/ /* F S c t l [ F i l e ] */ /******************************************************************************/ - + int XrdPfcFSctl::FSctl(const int cmd, int alen, const char *args, @@ -76,7 +76,7 @@ int XrdPfcFSctl::FSctl(const int cmd, XrdOucErrInfo &eInfo, const XrdSecEntity *client) { - eInfo.setErrInfo(ENOTSUP, "File based fstcl not supported for a cache."); + eInfo.setErrInfo(ENOTSUP, "File based fstcl not supported for a cache."); return SFS_ERROR; } @@ -95,14 +95,14 @@ int XrdPfcFSctl::FSctl(const int cmd, // Verify command // if (cmd != SFS_FSCTL_PLUGXC) - {eInfo.setErrInfo(EIDRM, "None-cache command issued to a cache."); + {eInfo.setErrInfo(EIDRM, "None-cache command issued to a cache."); return SFS_ERROR; } // Very that we have a command // - if (!xeq || args.Arg1Len < 1) - {eInfo.setErrInfo(EINVAL, "Missing cache command or argument."); + if (!xeq || args.Arg1Len < 1) + {eInfo.setErrInfo(EINVAL, "Missing cache command or argument."); return SFS_ERROR; } diff --git a/src/XrdPfc/XrdPfcFile.hh b/src/XrdPfc/XrdPfcFile.hh index e010182ecf8..e142956de54 100644 --- a/src/XrdPfc/XrdPfcFile.hh +++ b/src/XrdPfc/XrdPfcFile.hh @@ -97,7 +97,7 @@ struct ChunkRequest char *m_buf; // Where to place the data chunk. long long m_off; // Offset *within* the corresponding block. int m_size; // Size of the data chunk. - + ChunkRequest(ReadRequest *rreq, char *buf, long long off, int size) : m_read_req(rreq), m_buf(buf), m_off(off), m_size(size) {} @@ -239,7 +239,7 @@ public: //! Used in XrdPosixXrootd::Close() //---------------------------------------------------------------------- bool ioActive(IO *io); - + //---------------------------------------------------------------------- //! \brief Flags that detach stats should be written out in final sync. //! Called from CacheIO upon Detach. @@ -304,7 +304,7 @@ private: static const char *m_traceID; int m_ref_cnt; //!< number of references from IO or sync - + XrdOssDF *m_data_file; //!< file handle for data file on disk XrdOssDF *m_info_file; //!< file handle for data-info file on disk Info m_cfi; //!< download status of file blocks and access statistics @@ -368,7 +368,7 @@ private: void inc_prefetch_read_cnt(int prc) { if (prc) { m_prefetch_read_cnt += prc; calc_prefetch_score(); } } void inc_prefetch_hit_cnt (int phc) { if (phc) { m_prefetch_hit_cnt += phc; calc_prefetch_score(); } } - void calc_prefetch_score() { m_prefetch_score = float(m_prefetch_hit_cnt) / m_prefetch_read_cnt; } + void calc_prefetch_score() { m_prefetch_score = float(m_prefetch_hit_cnt) / m_prefetch_read_cnt; } // Helpers diff --git a/src/XrdPfc/XrdPfcIOFile.hh b/src/XrdPfc/XrdPfcIOFile.hh index 7e771dbbfb2..46d43a8aa92 100644 --- a/src/XrdPfc/XrdPfcIOFile.hh +++ b/src/XrdPfc/XrdPfcIOFile.hh @@ -72,7 +72,7 @@ public: //! \brief Abstract virtual method of XrdPfc::IO //! Called to destruct the IO object after it is no longer used. void DetachFinalize() override; - + int Fstat(struct stat &sbuff) override; long long FSize() override; diff --git a/src/XrdPfc/XrdPfcIOFileBlock.hh b/src/XrdPfc/XrdPfcIOFileBlock.hh index 31a3c5e77d6..46603bdcce8 100644 --- a/src/XrdPfc/XrdPfcIOFileBlock.hh +++ b/src/XrdPfc/XrdPfcIOFileBlock.hh @@ -56,7 +56,7 @@ public: using XrdOucCacheIO::Read; int Read(char *Buffer, long long Offset, int Length) override; - + int Fstat(struct stat &sbuff) override; long long FSize() override; diff --git a/src/XrdPfc/XrdPfcInfo.cc b/src/XrdPfc/XrdPfcInfo.cc index d063affd33d..218f175ea6d 100644 --- a/src/XrdPfc/XrdPfcInfo.cc +++ b/src/XrdPfc/XrdPfcInfo.cc @@ -66,7 +66,7 @@ struct FpHelper XrdSysTrace* GetTrace() const { return f_trace; } FpHelper(XrdOssDF* fp, off_t off, XrdSysTrace *trace, const char *tid, const TraceHeader &thdr) : - f_fp(fp), f_off(off), f_trace(trace), m_traceID(tid), f_trace_hdr(thdr) + f_fp(fp), f_off(off), f_trace(trace), m_traceID(tid), f_trace_hdr(thdr) {} // Returns true on error diff --git a/src/XrdPfc/XrdPfcInfo.hh b/src/XrdPfc/XrdPfcInfo.hh index b9f965bcb23..ef91a6a2826 100644 --- a/src/XrdPfc/XrdPfcInfo.hh +++ b/src/XrdPfc/XrdPfcInfo.hh @@ -367,7 +367,7 @@ inline void Info::SetBitWritten(int i) inline void Info::SetBitPrefetch(int i) { if (!m_buff_prefetch) return; - + const int cn = i/8; assert(cn < GetBitvecSizeInBytes()); diff --git a/src/XrdPfc/XrdPfcPurgeQuota.cc b/src/XrdPfc/XrdPfcPurgeQuota.cc index 12994ab8d08..0b499b742fa 100644 --- a/src/XrdPfc/XrdPfcPurgeQuota.cc +++ b/src/XrdPfc/XrdPfcPurgeQuota.cc @@ -12,112 +12,112 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin { public: - XrdPfcPurgeQuota () {} - - //---------------------------------------------------------------------------- - //! Set directory statistics - //---------------------------------------------------------------------------- - void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) - { - for (list_i it = m_list.begin(); it != m_list.end(); ++it) - { - it->dirState = rootDS->find_path(it->path, XrdPfc::Cache::Conf().m_dirStatsStoreDepth, false, false); - } - } - - //---------------------------------------------------------------------------- - //! Provide bytes to erase from dir quota listed in a text file - //---------------------------------------------------------------------------- - virtual long long GetBytesToRecover(XrdPfc::DirState *ds) - { - // setup diskusage for each dir path - InitDirStatesForLocalPaths(ds); - - long long totalToRemove = 0; - // get bytes to remove - for (list_i it = m_list.begin(); it != m_list.end(); ++it) - { - // XXXXXX here we should have another mechanism. and probably add up here + subdirs - long long cv = it->dirState->m_recursive_subdir_usage.m_bytes_on_disk - it->nBytesQuota; - if (cv > 0) - it->nBytesToRecover = cv; - else - it->nBytesToRecover = 0; - - totalToRemove += it->nBytesToRecover; - } - - return totalToRemove; - } - - //---------------------------------------------------------------------------- - //! Provide bytes to erase from dir quota listed in a text file - //---------------------------------------------------------------------------- - virtual bool ConfigPurgePin(const char *parms) - { - XrdSysError *log = XrdPfc::Cache::GetInstance().GetLog(); - - // retrive configuration file name - if (!parms || !parms[0] || (strlen(parms) == 0)) - { - log->Emsg("ConfigDecision", "Quota file not specified."); - return false; - } - log->Emsg("ConfigDecision", "Using directory list", parms); - - // parse the file to get directory quotas - const char* config_filename = parms; - const char *theINS = getenv("XRDINSTANCE"); - XrdOucEnv myEnv; - XrdOucStream Config(log, theINS, &myEnv, "=====> PurgeQuota "); - - int fd; - if ((fd = open(config_filename, O_RDONLY, 0)) < 0) - { - log->Emsg("Config() can't open configuration file ", config_filename); - } - - Config.Attach(fd); - static const char *cvec[] = {"*** pfc purge plugin :", 0}; - Config.Capture(cvec); - - char *var; - while ((var = Config.GetMyFirstWord())) - { - std::string dirpath = var; - const char* val; - - if (!(val = Config.GetWord())) + XrdPfcPurgeQuota() {} + + //---------------------------------------------------------------------------- + //! Set directory statistics + //---------------------------------------------------------------------------- + void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) + { + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + it->dirState = rootDS->find_path(it->path, XrdPfc::Cache::Conf().m_dirStatsStoreDepth, false, false); + } + } + + //---------------------------------------------------------------------------- + //! Provide bytes to erase from dir quota listed in a text file + //---------------------------------------------------------------------------- + virtual long long GetBytesToRecover(XrdPfc::DirState *ds) + { + // setup diskusage for each dir path + InitDirStatesForLocalPaths(ds); + + long long totalToRemove = 0; + // get bytes to remove + for (list_i it = m_list.begin(); it != m_list.end(); ++it) + { + // XXXXXX here we should have another mechanism. and probably add up here + subdirs + long long cv = it->dirState->recursive_subdir_usage().m_BytesOnDisk - it->nBytesQuota; + if (cv > 0) + it->nBytesToRecover = cv; + else + it->nBytesToRecover = 0; + + totalToRemove += it->nBytesToRecover; + } + + return totalToRemove; + } + + //---------------------------------------------------------------------------- + //! Provide bytes to erase from dir quota listed in a text file + //---------------------------------------------------------------------------- + virtual bool ConfigPurgePin(const char *parms) + { + XrdSysError *log = XrdPfc::Cache::GetInstance().GetLog(); + + // retrive configuration file name + if (!parms || !parms[0] || (strlen(parms) == 0)) + { + log->Emsg("ConfigDecision", "Quota file not specified."); + return false; + } + log->Emsg("ConfigDecision", "Using directory list", parms); + + // parse the file to get directory quotas + const char *config_filename = parms; + const char *theINS = getenv("XRDINSTANCE"); + XrdOucEnv myEnv; + XrdOucStream Config(log, theINS, &myEnv, "=====> PurgeQuota "); + + int fd; + if ((fd = open(config_filename, O_RDONLY, 0)) < 0) + { + log->Emsg("Config() can't open configuration file ", config_filename); + } + + Config.Attach(fd); + static const char *cvec[] = {"*** pfc purge plugin :", 0}; + Config.Capture(cvec); + + char *var; + while ((var = Config.GetMyFirstWord())) + { + std::string dirpath = var; + const char *val; + + if (!(val = Config.GetWord())) + { + log->Emsg("PurgeQuota plugin", "quota not specified"); + continue; + } + + std::string tmpc = val; + long long quota = 0; + if (::isalpha(*(tmpc.rbegin()))) + { + if (XrdOuca2x::a2sz(*log, "Error getting quota", tmpc.c_str(), "a)) { - log->Emsg("PurgeQuota plugin", "quota not specified"); - continue; + continue; } - - std::string tmpc = val; - long long quota = 0; - if (::isalpha(*(tmpc.rbegin()))) - { - if (XrdOuca2x::a2sz(*log, "Error getting quota", tmpc.c_str(),"a)) - { - continue; - } - } - else + } + else + { + if (XrdOuca2x::a2ll(*log, "Error getting quota", tmpc.c_str(), "a)) { - if (XrdOuca2x::a2ll(*log, "Error getting quota", tmpc.c_str(),"a)) - { - continue; - } + continue; } + } - DirInfo d; - d.path = dirpath; - d.nBytesQuota = quota; - m_list.push_back(d); - } + DirInfo d; + d.path = dirpath; + d.nBytesQuota = quota; + m_list.push_back(d); + } - return true; - } + return true; + } }; /******************************************************************************/ @@ -127,8 +127,8 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin // Return a purge object to use. extern "C" { - XrdPfc::PurgePin *XrdPfcGetPurgePin(XrdSysError &) - { - return new XrdPfcPurgeQuota(); - } + XrdPfc::PurgePin *XrdPfcGetPurgePin(XrdSysError &) + { + return new XrdPfcPurgeQuota(); + } } diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 5f7fe1adab9..89ff01aa4a9 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -3,6 +3,7 @@ #include "XrdPfcPathParseTools.hh" #include "XrdPfcFsTraversal.hh" #include "XrdPfcDirState.hh" +#include "XrdPfcDirStateSnapshot.hh" #include "XrdPfcTrace.hh" using namespace XrdPfc; @@ -44,11 +45,11 @@ void ResourceMonitor::scan_dir_and_recurse(FsTraversal &fst) // Remove files that do not have both cinfo and data? // Remove empty directories before even descending? // Leave this for some consistency pass? - // Note that FsTraversal supports ignored paths ... some details (cofig, N2N to be clarified). + // Note that FsTraversal supports ignored paths ... some details (config, N2N) to be clarified. if (it->second.has_data && it->second.has_cinfo) { - here.m_bytes_on_disk += it->second.stat_data.st_blocks * 512; - here.m_num_files += 1; + here.m_BytesOnDisk += it->second.stat_data.st_blocks * 512; + here.m_NFiles += 1; } } @@ -67,11 +68,11 @@ void ResourceMonitor::scan_dir_and_recurse(FsTraversal &fst) scan_dir_and_recurse(fst); fst.cd_up(); - here.m_num_subdirs += 1; + here.m_NDirectories += 1; - subdirs.m_bytes_on_disk += dhere.m_bytes_on_disk + dsubdirs.m_bytes_on_disk; - subdirs.m_num_files += dhere.m_num_files + dsubdirs.m_num_files; - subdirs.m_num_subdirs += dhere.m_num_subdirs + dsubdirs.m_num_subdirs; + subdirs.m_BytesOnDisk += dhere.m_BytesOnDisk + dsubdirs.m_BytesOnDisk; + subdirs.m_NFiles += dhere.m_NFiles + dsubdirs.m_NFiles; + subdirs.m_NDirectories += dhere.m_NDirectories + dsubdirs.m_NDirectories; } // XXX else try to remove it? } @@ -152,7 +153,7 @@ int ResourceMonitor::process_queues() } } - ds->m_here_usage.m_last_open_time = i.record.m_open_time; + ds->m_here_usage.m_LastOpenTime = i.record.m_open_time; } for (auto &i : m_file_update_stats_q.read_queue()) @@ -179,7 +180,7 @@ int ResourceMonitor::process_queues() DirState *ds = at.m_dir_state; ds->m_here_stats.m_NFilesClosed += 1; - ds->m_here_usage.m_last_close_time = i.record.m_close_time; + ds->m_here_usage.m_LastCloseTime = i.record.m_close_time; // Release the AccessToken! at.clear(); @@ -255,7 +256,7 @@ void ResourceMonitor::heart_beat() if (next_up_prop_time > time(0)) continue; - m_fs_state->upward_propagate_stats(); + m_fs_state->upward_propagate_stats_and_times(); next_up_prop_time += 60; // Here we can learn assumed file-based usage @@ -268,17 +269,36 @@ void ResourceMonitor::heart_beat() // This one should really be rather timely ... as it will be used for calculation // of averages of stuff going on. + m_fs_state->apply_stats_to_usages(); + // Dump statistcs before actual purging so maximum usage values get recorded. - // Should really go to gstream --- and should really go from Heartbeat. + // This should dump out binary snapshot into /pfc-stats/, if so configured. + // Also, optionally, json. + // Could also go to gstream but this could easily be way too large. if (Cache::Conf().is_dir_stat_reporting_on()) { - m_fs_state->dump_recursively(Cache::Conf().m_dirStatsStoreDepth); + const int StoreDepth = Cache::Conf().m_dirStatsStoreDepth; + const DirState &root_ds = *m_fs_state->get_root(); + const int n_sshot_dirs = root_ds.count_dirs_to_level(StoreDepth); + printf("Snapshot n_dirs=%d, total n_dirs=%d\n", n_sshot_dirs, + root_ds.m_here_usage.m_NDirectories + root_ds.m_recursive_subdir_usage.m_NDirectories + 1); + + m_fs_state->dump_recursively(StoreDepth); + + DataFsSnapshot ss(*m_fs_state); + ss.m_dir_states.reserve(n_sshot_dirs); + + ss.m_dir_states.emplace_back( DirStateElement(root_ds, -1) ); + fill_sshot_vec_children(root_ds, 0, ss.m_dir_states, StoreDepth); + + ss.dump(); } - // XXXX - // m_fs_state->apply_stats_to_usages_and_reset_stats(); - // m_fs_state->reset_stats(); // XXXXXX this is for sure after export, otherwise they will be zero + // if needed export to vector form + // - write to file + // - pass to purge pin + m_fs_state->reset_stats(); // check time / diskusage --> purge condition? // run purge as job or thread @@ -286,6 +306,33 @@ void ResourceMonitor::heart_beat() } } +void ResourceMonitor::fill_sshot_vec_children(const DirState &parent_ds, + int parent_idx, + std::vector &vec, + int max_depth) +{ + int pos = vec.size(); + int n_children = parent_ds.m_subdirs.size(); + + for (auto const & [name, child] : parent_ds.m_subdirs) + { + vec.emplace_back( DirStateElement(child, parent_idx) ); + } + + if (parent_ds.m_depth < max_depth) + { + DirStateElement &parent_dse = vec[parent_idx]; + parent_dse.m_daughters_begin = pos; + parent_dse.m_daughters_end = pos + n_children; + + for (auto const & [name, child] : parent_ds.m_subdirs) + { + if (n_children > 0) + fill_sshot_vec_children(child, pos, vec, max_depth); + ++pos; + } + } +} //============================================================================== // Old prototype from Cache / Purge, now to go into heart_beat() here, above. diff --git a/src/XrdPfc/XrdPfcResourceMonitor.hh b/src/XrdPfc/XrdPfcResourceMonitor.hh index e499fe653ad..1f5ee34c7c7 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.hh +++ b/src/XrdPfc/XrdPfcResourceMonitor.hh @@ -14,6 +14,7 @@ namespace XrdPfc { class DataFsState; class DirState; +class DirStateElement; class FsTraversal; //============================================================================== @@ -102,7 +103,7 @@ class ResourceMonitor XrdSysMutex m_queue_mutex; // mutex shared between queues unsigned int m_queue_swap_u1 = 0u; // identifier of current swap - + DataFsState *m_fs_state; XrdOss &m_oss; @@ -193,6 +194,14 @@ public: void heart_beat(); + // --- Helpers for export of DirState vector snapshot. + + void fill_sshot_vec_children(const DirState &parent_ds, + int parent_idx, + std::vector &vec, + int max_depth); + + /* XXX Stuff from Cache, to be revisited. enum ScanAndPurgeThreadState_e { SPTS_Idle, SPTS_Scan, SPTS_Purge, SPTS_Done }; diff --git a/src/XrdPfc/XrdPfcStats.hh b/src/XrdPfc/XrdPfcStats.hh index 7f2e14d4612..2e42aa38adc 100644 --- a/src/XrdPfc/XrdPfcStats.hh +++ b/src/XrdPfc/XrdPfcStats.hh @@ -131,8 +131,8 @@ public: int m_NFilesOpened = 0; int m_NFilesClosed = 0; int m_NFilesCreated = 0; - int m_NFilesRemoved = 0; // purged - int m_NDirectoriesCreated = 0; // this is hard, oss does it for us ... but we MUSt know it for DirState creation. + int m_NFilesRemoved = 0; // purged or otherwise (error, direct requests) + int m_NDirectoriesCreated = 0; int m_NDirectoriesRemoved = 0; //---------------------------------------------------------------------- From fce950df71729bde415b4d05ab62078edde3521c Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Fri, 26 Apr 2024 09:08:59 -0700 Subject: [PATCH 24/43] Fix compilation error (leftover return in void func). --- src/XrdPfc/XrdPfcDirStateSnapshot.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/XrdPfc/XrdPfcDirStateSnapshot.cc b/src/XrdPfc/XrdPfcDirStateSnapshot.cc index 2ce0d95ea3f..0efa835226a 100644 --- a/src/XrdPfc/XrdPfcDirStateSnapshot.cc +++ b/src/XrdPfc/XrdPfcDirStateSnapshot.cc @@ -54,6 +54,8 @@ using namespace XrdPfc; void DataFsSnapshot::write_json_file(const std::string &fname, bool include_preamble) { + // Throws exception on failed file-open. + std::ofstream ofs; open_ofstream(ofs, fname, __func__); @@ -73,7 +75,6 @@ void DataFsSnapshot::write_json_file(const std::string &fname, bool include_prea ofs << "\n"; ofs.close(); - return 0; } void DataFsSnapshot::dump() From e0ddd94c55d61e481625281a3d5a870a2f077f89 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Fri, 26 Apr 2024 15:40:07 -0700 Subject: [PATCH 25/43] Add number of bytes written, b_write, to file_close g-stream record. --- src/XrdPfc/XrdPfc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index 46dd75eed4d..09bf14917fd 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -655,12 +655,12 @@ void Cache::dec_ref_cnt(File* f, bool high_debug) int len = snprintf(buf, 4096, "{\"event\":\"file_close\"," "\"lfn\":\"%s\",\"size\":%lld,\"blk_size\":%d,\"n_blks\":%d,\"n_blks_done\":%d," "\"access_cnt\":%lu,\"attach_t\":%lld,\"detach_t\":%lld,\"remotes\":%s," - "\"b_hit\":%lld,\"b_miss\":%lld,\"b_bypass\":%lld,\"n_cks_errs\":%d}", + "\"b_hit\":%lld,\"b_miss\":%lld,\"b_bypass\":%lld,\"b_write\":%lld,\"n_cks_errs\":%d}", f->GetLocalPath().c_str(), f->GetFileSize(), f->GetBlockSize(), f->GetNBlocks(), f->GetNDownloadedBlocks(), (unsigned long) f->GetAccessCnt(), (long long) as->AttachTime, (long long) as->DetachTime, f->GetRemoteLocations().c_str(), - as->BytesHit, as->BytesMissed, as->BytesBypassed, st.m_NCksumErrors + st.m_BytesHit, st.m_BytesMissed, st.m_BytesBypassed, st.m_BytesWritten, st.m_NCksumErrors ); bool suc = false; if (len < 4096) From d5ccecc6c31ca9db63be5e18cd8b6e4a1330a731 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Mon, 29 Apr 2024 16:05:50 -0700 Subject: [PATCH 26/43] Properly account for the 512-byte round up at the file's end due to file-system block size. --- src/XrdPfc/XrdPfcCommand.cc | 2 +- src/XrdPfc/XrdPfcFile.cc | 2 +- src/XrdPfc/XrdPfcResourceMonitor.cc | 9 +++++++++ src/XrdPfc/XrdPfcResourceMonitor.hh | 5 +++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index 43205f38647..38e27e1f11e 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -271,7 +271,7 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) XrdPfc::Stats stats; stats.m_BytesWritten = file_size; m_res_mon->register_file_update_stats(token, stats); - m_res_mon->register_file_close(token, time(0)); + m_res_mon->register_file_close(token, time(0), stats); } } } diff --git a/src/XrdPfc/XrdPfcFile.cc b/src/XrdPfc/XrdPfcFile.cc index 0229acf2535..8b559560be6 100644 --- a/src/XrdPfc/XrdPfcFile.cc +++ b/src/XrdPfc/XrdPfcFile.cc @@ -98,7 +98,7 @@ File::~File() if (m_resmon_token >= 0) { // Last update of file stats has been sent from the final Sync. - Cache::ResMon().register_file_close(m_resmon_token, time(0)); + Cache::ResMon().register_file_close(m_resmon_token, time(0), m_stats); } TRACEF(Debug, "~File() ended, prefetch score = " << m_prefetch_score); diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 89ff01aa4a9..00b2616e6e3 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -182,6 +182,12 @@ int ResourceMonitor::process_queues() ds->m_here_usage.m_LastCloseTime = i.record.m_close_time; + // If full-stats write is not a multiple of 512-bytes that means we wrote the + // tail block. As usage is measured in 512-byte blocks this gets rounded up here. + long long over_512b = i.record.m_full_stats.m_BytesWritten & 0x1FFll; + if (over_512b) + ds->m_here_stats.m_BytesWritten += 512ll - over_512b; + // Release the AccessToken! at.clear(); m_access_tokens_free_slots.push_back(tid); @@ -191,6 +197,7 @@ int ResourceMonitor::process_queues() { // i.id: DirState*, i.record: PurgeRecord DirState *ds = i.id; + // NOTE -- bytes removed should already be rounded up to 512-bytes for each file. ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; ds->m_here_stats.m_NFilesRemoved += i.record.n_files; } @@ -203,6 +210,7 @@ int ResourceMonitor::process_queues() // find_path can return the last dir found ... but this clearly isn't a valid purge record. continue; } + // NOTE -- bytes removed should already be rounded up to 512-bytes for each file. ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; ds->m_here_stats.m_NFilesRemoved += i.record.n_files; } @@ -214,6 +222,7 @@ int ResourceMonitor::process_queues() TRACE(Error, trc_pfx << "DirState not found for LFN path '" << i.id << "'."); continue; } + // NOTE -- bytes removed should already be rounded up to 512-bytes. ds->m_here_stats.m_BytesRemoved += i.record; ds->m_here_stats.m_NFilesRemoved += 1; } diff --git a/src/XrdPfc/XrdPfcResourceMonitor.hh b/src/XrdPfc/XrdPfcResourceMonitor.hh index 1f5ee34c7c7..a051116e554 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.hh +++ b/src/XrdPfc/XrdPfcResourceMonitor.hh @@ -86,6 +86,7 @@ class ResourceMonitor struct CloseRecord { time_t m_close_time; + Stats m_full_stats; }; struct PurgeRecord { @@ -154,9 +155,9 @@ public: // in File::Open(). } - void register_file_close(int token_id, time_t close_timestamp) { + void register_file_close(int token_id, time_t close_timestamp, const Stats& full_stats) { XrdSysMutexHelper _lock(&m_queue_mutex); - m_file_close_q.push(token_id, {close_timestamp}); + m_file_close_q.push(token_id, {close_timestamp, full_stats}); } // deletions can come from purge and from direct requests (Cache::UnlinkFile), the latter From 8ca0df7e7d31af85cd649d7abbfab779f080bffd Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Thu, 23 May 2024 16:26:15 -0700 Subject: [PATCH 27/43] Major cleanup of stats, usages, snapshots and purge handling. - Separate snapshots for stat reporting and for purge. - ResourceMonitor::heart_beat() now drives the purge process. It still calls the old-style purge function -- but this can now be easily modularized as needed. - Uses stat-blocks as unit of direcotry / file usage. --- src/XrdPfc/XrdPfc.cc | 41 +- src/XrdPfc/XrdPfc.hh | 35 +- src/XrdPfc/XrdPfcCommand.cc | 3 +- src/XrdPfc/XrdPfcConfiguration.cc | 18 +- src/XrdPfc/XrdPfcDirState.cc | 58 ++- src/XrdPfc/XrdPfcDirState.hh | 100 ++--- src/XrdPfc/XrdPfcDirStateSnapshot.cc | 100 +++-- src/XrdPfc/XrdPfcDirStateSnapshot.hh | 55 ++- src/XrdPfc/XrdPfcFPurgeState.cc | 52 +-- src/XrdPfc/XrdPfcFPurgeState.hh | 13 +- src/XrdPfc/XrdPfcFile.cc | 24 +- src/XrdPfc/XrdPfcFile.hh | 3 +- src/XrdPfc/XrdPfcFsTraversal.cc | 73 +--- src/XrdPfc/XrdPfcFsTraversal.hh | 8 +- src/XrdPfc/XrdPfcIO.hh | 1 - src/XrdPfc/XrdPfcIOFile.cc | 1 - src/XrdPfc/XrdPfcIOFile.hh | 1 - src/XrdPfc/XrdPfcIOFileBlock.hh | 1 - src/XrdPfc/XrdPfcInfo.hh | 2 - src/XrdPfc/XrdPfcPurge.cc | 402 +++++------------ src/XrdPfc/XrdPfcPurgePin.hh | 7 +- src/XrdPfc/XrdPfcPurgeQuota.cc | 31 +- src/XrdPfc/XrdPfcResourceMonitor.cc | 625 ++++++++++++++++++++++----- src/XrdPfc/XrdPfcResourceMonitor.hh | 96 ++-- src/XrdPfc/XrdPfcStats.hh | 35 +- 25 files changed, 1069 insertions(+), 716 deletions(-) diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index 09bf14917fd..2a6fe92a30e 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -27,7 +27,6 @@ #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucUtils.hh" -#include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysTimer.hh" #include "XrdSys/XrdSysTrace.hh" @@ -49,16 +48,9 @@ Cache *Cache::m_instance = nullptr; XrdScheduler *Cache::schedP = nullptr; -void *ResourceMonitorHeartBeatThread(void*) +void *ResourceMonitorThread(void*) { - // Cache::GetInstance().ResourceMonitorHeartBeat(); - Cache::ResMon().heart_beat(); - return 0; -} - -void *PurgeThread(void*) -{ - Cache::GetInstance().Purge(); + Cache::ResMon().main_thread_function(); return 0; } @@ -105,6 +97,8 @@ XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, { pthread_t tid; + XrdSysThread::Run(&tid, ResourceMonitorThread, 0, 0, "XrdPfc ResourceMonitor"); + for (int wti = 0; wti < instance.RefConfiguration().m_wqueue_threads; ++wti) { XrdSysThread::Run(&tid, ProcessWriteTaskThread, 0, 0, "XrdPfc WriteTasks "); @@ -114,12 +108,6 @@ XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, { XrdSysThread::Run(&tid, PrefetchThread, 0, 0, "XrdPfc Prefetch "); } - - XrdSysThread::Run(&tid, ResourceMonitorHeartBeatThread, 0, 0, "XrdPfc ResourceMonitorHeartBeat"); - - // XXXX This will be called from ResMon::heart_beat, as and if needed (maybe as XrdJob). - // Now on for testing of the FPurgeState traversal / old-style (default?) purge. - XrdSysThread::Run(&tid, PurgeThread, 0, 0, "XrdPfc Purge"); } XrdPfcFSctl* pfcFSctl = new XrdPfcFSctl(instance, logger); @@ -132,7 +120,7 @@ XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, //============================================================================== void Configuration::calculate_fractional_usages(long long du, long long fu, - double &frac_du, double &frac_fu) + double &frac_du, double &frac_fu) const { // Calculate fractional disk / file usage and clamp them to [0, 1]. @@ -200,7 +188,6 @@ Cache::Cache(XrdSysLogger *logger, XrdOucEnv *env) : m_RAM_write_queue(0), m_RAM_std_size(0), m_isClient(false), - m_in_purge(false), m_active_cond(0) { // Default log level is Warning. @@ -348,6 +335,15 @@ void Cache::ProcessWriteTasks() } } +long long Cache::WritesSinceLastCall() +{ + // Called from ResourceMonitor for an alternative estimation of disk writes. + XrdSysCondVarHelper lock(&m_writeQ.condVar); + long long ret = m_writeQ.writes_between_purges; + m_writeQ.writes_between_purges = 0; + return ret; +} + //============================================================================== char* Cache::RequestRAM(long long size) @@ -677,7 +673,7 @@ void Cache::dec_ref_cnt(File* f, bool high_debug) } } -bool Cache::IsFileActiveOrPurgeProtected(const std::string& path) +bool Cache::IsFileActiveOrPurgeProtected(const std::string& path) const { XrdSysCondVarHelper lock(&m_active_cond); @@ -685,6 +681,11 @@ bool Cache::IsFileActiveOrPurgeProtected(const std::string& path) m_purge_delay_set.find(path) != m_purge_delay_set.end(); } +void Cache::ClearPurgeProtectedSet() +{ + XrdSysCondVarHelper lock(&m_active_cond); + m_purge_delay_set.clear(); +} //============================================================================== //=== PREFETCH @@ -1186,7 +1187,7 @@ int Cache::UnlinkFile(const std::string& f_name, bool fail_if_open) int i_ret = m_oss->Unlink(i_name.c_str()); if (stat_ok) - m_res_mon->register_file_purge(f_name, f_stat.st_blocks * 512); + m_res_mon->register_file_purge(f_name, f_stat.st_blocks); TRACE(Debug, trc_pfx << f_name << ", f_ret=" << f_ret << ", i_ret=" << i_ret); diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index a9d43d3d531..6e02ffd42d6 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -42,8 +42,15 @@ namespace XrdPfc class File; class IO; class PurgePin; -class DataFsState; class ResourceMonitor; + + +template +struct MutexHolder { + MOO &mutex; + MutexHolder(MOO &m) : mutex(m) { mutex.Lock(); } + ~MutexHolder() { mutex.UnLock(); } +}; } @@ -63,7 +70,7 @@ struct Configuration bool is_dir_stat_reporting_on() const { return m_dirStatsMaxDepth >= 0 || ! m_dirStatsDirs.empty() || ! m_dirStatsDirGlobs.empty(); } bool is_purge_plugin_set_up() const { return false; } - void calculate_fractional_usages(long long du, long long fu, double &frac_du, double &frac_fu); + void calculate_fractional_usages(long long du, long long fu, double &frac_du, double &frac_fu) const; CkSumCheck_e get_cs_Chk() const { return (CkSumCheck_e) m_cs_Chk; } @@ -222,11 +229,6 @@ public: //--------------------------------------------------------------------- static bool VCheck(XrdVersionInfo &urVersion) { return true; } - //--------------------------------------------------------------------- - //! Thread function invoked to scan and purge files from disk when needed. - //--------------------------------------------------------------------- - void Purge(); - //--------------------------------------------------------------------- //! Remove cinfo and data files from cache. //--------------------------------------------------------------------- @@ -248,6 +250,8 @@ public: //--------------------------------------------------------------------- void ProcessWriteTasks(); + long long WritesSinceLastCall(); + char* RequestRAM(long long size); void ReleaseRAM(char* buf, long long size); @@ -260,7 +264,9 @@ public: XrdOss* GetOss() const { return m_oss; } - bool IsFileActiveOrPurgeProtected(const std::string&); + bool IsFileActiveOrPurgeProtected(const std::string&) const; + void ClearPurgeProtectedSet(); + PurgePin* GetPurgePin() const { return m_purge_pin; } File* GetFile(const std::string&, IO*, long long off = 0, long long filesize = 0); @@ -290,7 +296,7 @@ private: bool cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name); - static Cache *m_instance; //!< this object + static Cache *m_instance; //!< this object XrdOucEnv *m_env; //!< environment passed in at creation XrdSysError m_log; //!< XrdPfc namespace logger @@ -303,8 +309,8 @@ private: ResourceMonitor *m_res_mon; - std::vector m_decisionpoints; //!< decision plugins - XrdPfc::PurgePin* m_purge_pin; //!< purge plugin + std::vector m_decisionpoints; //!< decision plugins + PurgePin* m_purge_pin; //!< purge plugin Configuration m_configuration; //!< configurable parameters @@ -336,10 +342,9 @@ private: typedef ActiveMap_t::iterator ActiveMap_i; typedef std::set FNameSet_t; - ActiveMap_t m_active; //!< Map of currently active / open files. - FNameSet_t m_purge_delay_set; - bool m_in_purge; - XrdSysCondVar m_active_cond; //!< Cond-var protecting active file data structures. + ActiveMap_t m_active; //!< Map of currently active / open files. + FNameSet_t m_purge_delay_set; //!< Set of files that should not be purged. + mutable XrdSysCondVar m_active_cond; //!< Cond-var protecting active file data structures. void inc_ref_cnt(File*, bool lock, bool high_debug); void dec_ref_cnt(File*, bool high_debug); diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index 38e27e1f11e..77bbcb471c1 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -269,7 +269,8 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) { int token = m_res_mon->register_file_open(file_path, time_now, false); XrdPfc::Stats stats; - stats.m_BytesWritten = file_size; + stats.m_BytesWritten = file_size; + stats.m_StBlocksAdded = (file_size & 0x1ff) ? (file_size >> 9) + 1 : file_size >> 9; m_res_mon->register_file_update_stats(token, stats); m_res_mon->register_file_close(token, time(0), stats); } diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index bc9305aca69..8273f0701fc 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -442,6 +442,18 @@ bool Cache::Config(const char *config_filename, const char *parameters) // sets default value for disk usage XrdOssVSInfo sP; { + if (m_configuration.m_meta_space != m_configuration.m_data_space && + m_oss->StatVS(&sP, m_configuration.m_meta_space.c_str(), 1) < 0) + { + m_log.Emsg("ConfigParameters()", "error obtaining stat info for meta space ", m_configuration.m_meta_space.c_str()); + return false; + } + if (sP.Total < 10ll << 20) + { + m_log.Emsg("ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ", + m_configuration.m_meta_space.c_str()); + return false; + } if (m_oss->StatVS(&sP, m_configuration.m_data_space.c_str(), 1) < 0) { m_log.Emsg("ConfigParameters()", "error obtaining stat info for data space ", m_configuration.m_data_space.c_str()); @@ -614,13 +626,11 @@ bool Cache::Config(const char *config_filename, const char *parameters) m_log.Say(" pfc g-stream has", m_gstream ? "" : " NOT", " been configured via xrootd.monitor directive\n"); - // Create the ResourceMonitor object and perform initial scan. + // Create the ResourceMonitor and get it ready for starting the main thread function. if (aOK) { - m_log.Say("-----> Proxy file cache performing initial directory scan"); m_res_mon = new ResourceMonitor(*m_oss); - aOK = m_res_mon->perform_initial_scan(); - m_log.Say("-----> Proxy File Cache initial directory scan finished"); + m_res_mon->init_before_main(); } m_log.Say("=====> Proxy file cache configuration parsing ", aOK ? "completed" : "failed"); diff --git a/src/XrdPfc/XrdPfcDirState.cc b/src/XrdPfc/XrdPfcDirState.cc index c94a094d6b7..9b777661e82 100644 --- a/src/XrdPfc/XrdPfcDirState.cc +++ b/src/XrdPfc/XrdPfcDirState.cc @@ -108,6 +108,30 @@ DirState *DirState::find_dir(const std::string &dir, return nullptr; } +//---------------------------------------------------------------------------- +//! Propagate usages to parents after initial directory scan. +//! Called from ResourceMonitor::perform_initial_scan() +//---------------------------------------------------------------------------- +void DirState::upward_propagate_initial_scan_usages() +{ + DirUsage &here = m_here_usage; + DirUsage &subdirs = m_recursive_subdir_usage; + + for (auto & [name, daughter] : m_subdirs) + { + daughter.upward_propagate_initial_scan_usages(); + + DirUsage &dhere = daughter.m_here_usage; + DirUsage &dsubdirs = daughter.m_recursive_subdir_usage; + + here.m_NDirectories += 1; + + subdirs.m_StBlocks += dhere.m_StBlocks + dsubdirs.m_StBlocks; + subdirs.m_NFiles += dhere.m_NFiles + dsubdirs.m_NFiles; + subdirs.m_NDirectories += dhere.m_NDirectories + dsubdirs.m_NDirectories; + } +} + //---------------------------------------------------------------------------- //! Propagate stat to parents //! Called from ResourceMonitor::heart_beat() @@ -151,29 +175,6 @@ void DirState::reset_stats() m_recursive_subdir_stats.Reset(); } -//---------------------------------------------------------------------------- -//! Update statistics. -//! Called from ... to be seen XXXX -//---------------------------------------------------------------------------- -// attic -long long DirState::upward_propagate_usage_purged() -{ - // XXXXX what's with this? Should be automatic through the queues now .... - /* - for (DsMap_i i = m_subdirs.begin(); i != m_subdirs.end(); ++i) - { - m_usage_purged += i->second.upward_propagate_usage_purged(); - } - m_usage -= m_usage_purged; - - long long ret = m_usage_purged; - m_usage_purged = 0; - return ret; - */ - return 0; -} - - int DirState::count_dirs_to_level(int max_depth) const { int n_dirs = 1; @@ -195,8 +196,8 @@ void DirState::dump_recursively(const char *name, int max_depth) const { printf("%*d %s usage_here=%lld usage_sub=%lld usage_total=%lld num_ios=%d duration=%d b_hit=%lld b_miss=%lld b_byps=%lld b_wrtn=%lld\n", 2 + 2 * m_depth, m_depth, name, - m_here_usage.m_BytesOnDisk, m_recursive_subdir_usage.m_BytesOnDisk, - m_here_usage.m_BytesOnDisk + m_recursive_subdir_usage.m_BytesOnDisk, + 512 * m_here_usage.m_StBlocks, 512 * m_recursive_subdir_usage.m_StBlocks, + 512 * (m_here_usage.m_StBlocks + m_recursive_subdir_usage.m_StBlocks), // XXXXX here_stats or sum up? or both? m_here_stats.m_NumIos, m_here_stats.m_Duration, m_here_stats.m_BytesHit, m_here_stats.m_BytesMissed, m_here_stats.m_BytesBypassed, @@ -237,12 +238,9 @@ void DataFsState::dump_recursively(int max_depth) const { if (max_depth < 0) max_depth = 4096; - time_t now = time(0); - - printf("DataFsState::dump_recursively epoch = %lld delta_t = %lld, max_dump_depth = %d\n", - (long long)now, (long long)(now - m_prev_time), max_depth); - m_prev_time = now; + printf("DataFsState::dump_recursively delta_t = %lld, max_dump_depth = %d\n", + (long long)(m_usage_update_time - m_stats_reset_time), max_depth); m_root.dump_recursively("root", max_depth); } diff --git a/src/XrdPfc/XrdPfcDirState.hh b/src/XrdPfc/XrdPfcDirState.hh index 632069730ff..39f6952a89e 100644 --- a/src/XrdPfc/XrdPfcDirState.hh +++ b/src/XrdPfc/XrdPfcDirState.hh @@ -14,11 +14,13 @@ // - Data-holding struct DirUsage -- complementary to Stats. // - Base classes for DirState and DataFsState, shared between in-memory // tree form and snap-shot vector form. -// - Structs for DirState export in vector form: -// - struct DirStateElement, and -// - struct DataFsSnapshot. -// Those should probably go to another .hh/.cc so the object file can be included -// the dedicated binary for processing of the binary dumps. +// - Forward declatation of structs for DirState export in vector form: +// - struct DirStateElement \_ for stats and usages snapshot +// - struct DataFsSnapshot / +// - struct DirPurgeElement \_ for purge snapshot +// - struct DataFsPurgeshot / +// Those are in another file so the object file can be included in the +// dedicated binary for processing of the binary dumps. // - class DirState -- state of a directory, including current delta-stats. // - class DataFSState -- manager of the DirState tree, starting from root (as in "/"). // @@ -40,14 +42,29 @@ struct DirUsage { time_t m_LastOpenTime = 0; time_t m_LastCloseTime = 0; - long long m_BytesOnDisk = 0; + long long m_StBlocks = 0; int m_NFilesOpen = 0; int m_NFiles = 0; int m_NDirectories = 0; + DirUsage() = default; + + DirUsage(const DirUsage& s) = default; + + DirUsage& operator=(const DirUsage&) = default; + + DirUsage(const DirUsage &a, const DirUsage &b) : + m_LastOpenTime (std::max(a.m_LastOpenTime, b.m_LastOpenTime)), + m_LastCloseTime (std::max(a.m_LastCloseTime, b.m_LastCloseTime)), + m_StBlocks (a.m_StBlocks + b.m_StBlocks), + m_NFilesOpen (a.m_NFilesOpen + b.m_NFilesOpen), + m_NFiles (a.m_NFiles + b.m_NFiles), + m_NDirectories (a.m_NDirectories + b.m_NDirectories) + {} + void update_from_stats(const DirStats& s) { - m_BytesOnDisk += s.m_BytesWritten - s.m_BytesRemoved; + m_StBlocks += s.m_StBlocksAdded - s.m_StBlocksRemoved; m_NFilesOpen += s.m_NFilesOpened - s.m_NFilesClosed; m_NFiles += s.m_NFilesCreated - s.m_NFilesRemoved; m_NDirectories += s.m_NDirectoriesCreated - s.m_NDirectoriesRemoved; @@ -69,17 +86,8 @@ struct DirStateBase { std::string m_dir_name; - DirStats m_here_stats; - DirStats m_recursive_subdir_stats; - - DirUsage m_here_usage; - DirUsage m_recursive_subdir_usage; - - DirStateBase() {} DirStateBase(const std::string &dname) : m_dir_name(dname) {} - - const DirUsage& recursive_subdir_usage() const { return m_recursive_subdir_usage; } }; struct DataFsStateBase @@ -87,7 +95,11 @@ struct DataFsStateBase time_t m_usage_update_time = 0; time_t m_stats_reset_time = 0; - // FS usage and available space information + long long m_disk_total = 0; // In bytes, from Oss::StatVS() on space data + long long m_disk_used = 0; // "" + long long m_file_usage = 0; // Calculate usage by data files in the cache + long long m_meta_total = 0; // In bytes, from Oss::StatVS() on space meta + long long m_meta_used = 0; // "" }; @@ -98,6 +110,10 @@ struct DataFsStateBase struct DirStateElement; struct DataFsSnapshot; +struct DirPurgeElement; +struct DataFsPurgeshot; + + //============================================================================== // DirState //============================================================================== @@ -107,30 +123,21 @@ struct DirState : public DirStateBase typedef std::map DsMap_t; typedef DsMap_t::iterator DsMap_i; - DirState *m_parent = nullptr; - DsMap_t m_subdirs; - - int m_depth; - // bool m_stat_report; // not used yet - storing of stats requested; might also need depth - - // XXX int m_possible_discrepancy; // num detected possible inconsistencies. here, subdirs? - - // flag - can potentially be inaccurate -- plus timestamp of it (min or max, if several for subdirs)? - - // Do we need running averages of these, too, not just traffic? - // Snapshot should be fine, no? - - // Do we need all-time stats? Files/dirs created, deleted; files opened/closed; - // Well, those would be like Stats-running-average-infinity, just adding stuff in. - - // min/max open (or close ... or both?) time-stamps + DirStats m_here_stats; + DirStats m_recursive_subdir_stats; - // Do we need string name? Probably yes, if we want to construct PFN from a given - // inner node upwards. Also, in this case, is it really the best idea to have - // map as daughter container? It keeps them sorted for export :) + DirUsage m_here_usage; + DirUsage m_recursive_subdir_usage; - // quota info, enabled? + // This should be optional, only if needed and only up to some max level. + // Preferably stored in some extrnal vector (as AccessTokens are) and indexed from here. + // DirStats m_purge_stats; // here + subdir, running avg., as per purge params + // DirStats m_report_stats; // here + subdir, reset after sshot dump + DirState *m_parent = nullptr; + DsMap_t m_subdirs; + int m_depth; + bool m_scanned = false; // set to true after files in this directory are scanned. void init(); @@ -139,6 +146,7 @@ struct DirState : public DirStateBase DirState* find_path_tok(PathTokenizer &pt, int pos, bool create_subdirs, DirState **last_existing_dir = nullptr); + // --- public part --- DirState(); @@ -153,14 +161,14 @@ struct DirState : public DirStateBase DirState* find_dir(const std::string &dir, bool create_subdirs); + // initial scan support + void upward_propagate_initial_scan_usages(); + // stat support void upward_propagate_stats_and_times(); void apply_stats_to_usages(); void reset_stats(); - // attic - long long upward_propagate_usage_purged(); // why would this be any different? isn't it included in stats? - int count_dirs_to_level(int max_depth) const; void dump_recursively(const char *name, int max_depth) const; @@ -174,13 +182,8 @@ struct DirState : public DirStateBase struct DataFsState : public DataFsStateBase { DirState m_root; - mutable time_t m_prev_time; - - DataFsState() : - m_root (), - m_prev_time (time(0)) - {} + DataFsState() : m_root() {} DirState* get_root() { return & m_root; } @@ -193,9 +196,6 @@ struct DataFsState : public DataFsStateBase void apply_stats_to_usages(); void reset_stats(); - // attic - void upward_propagate_usage_purged() { m_root.upward_propagate_usage_purged(); } - void dump_recursively(int max_depth) const; }; diff --git a/src/XrdPfc/XrdPfcDirStateSnapshot.cc b/src/XrdPfc/XrdPfcDirStateSnapshot.cc index 0efa835226a..2bc1422c285 100644 --- a/src/XrdPfc/XrdPfcDirStateSnapshot.cc +++ b/src/XrdPfc/XrdPfcDirStateSnapshot.cc @@ -1,4 +1,5 @@ #include "XrdPfcDirStateSnapshot.hh" +#include "XrdPfcPathParseTools.hh" #include "XrdOuc/XrdOucJson.hh" @@ -25,28 +26,30 @@ namespace XrdPfc { PFC_DEFINE_TYPE_NON_INTRUSIVE(DirStats, - m_NumIos, m_Duration, m_BytesHit, m_BytesMissed, m_BytesBypassed, m_BytesWritten, m_NCksumErrors, - m_BytesRemoved, m_NFilesOpened, m_NFilesClosed, m_NFilesCreated, m_NFilesRemoved, m_NDirectoriesCreated, m_NDirectoriesRemoved) + m_NumIos, m_Duration, m_BytesHit, m_BytesMissed, m_BytesBypassed, m_BytesWritten, m_StBlocksAdded, m_NCksumErrors, + m_StBlocksRemoved, m_NFilesOpened, m_NFilesClosed, m_NFilesCreated, m_NFilesRemoved, m_NDirectoriesCreated, m_NDirectoriesRemoved) PFC_DEFINE_TYPE_NON_INTRUSIVE(DirUsage, - m_LastOpenTime, m_LastCloseTime, m_BytesOnDisk, m_NFilesOpen, m_NFiles, m_NDirectories) + m_LastOpenTime, m_LastCloseTime, m_StBlocks, m_NFilesOpen, m_NFiles, m_NDirectories) PFC_DEFINE_TYPE_NON_INTRUSIVE(DirStateElement, - m_dir_name, m_here_stats, m_recursive_subdir_stats, m_here_usage, m_recursive_subdir_usage, - m_parent, m_daughters_begin, m_daughters_end) + m_dir_name, m_stats, m_usage, + m_parent, m_daughters_begin, m_daughters_end) PFC_DEFINE_TYPE_NON_INTRUSIVE(DataFsSnapshot, - m_usage_update_time, m_stats_reset_time, - m_dir_states) + m_usage_update_time, m_stats_reset_time, m_disk_total, m_disk_used, m_file_usage, m_meta_total, m_meta_used, + m_dir_states) } namespace { // Open file for writing, throw exception on failure. -void open_ofstream(std::ofstream &ofs, const std::string &fname, const char *pfx = nullptr) { - ofs.open(fname, std::ofstream::trunc); - if (!ofs) { - char m[2048]; - snprintf(m, 2048, "%s%sError opening %s for write: %m", pfx ? pfx : "", pfx ? " " : "", fname.c_str()); - throw std::runtime_error(m); - } +void open_ofstream(std::ofstream &ofs, const std::string &fname, const char *pfx = nullptr) +{ + ofs.open(fname, std::ofstream::trunc); + if (!ofs) + { + char m[2048]; + snprintf(m, 2048, "%s%sError opening %s for write: %m", pfx ? pfx : "", pfx ? " " : "", fname.c_str()); + throw std::runtime_error(m); + } } } @@ -54,32 +57,65 @@ using namespace XrdPfc; void DataFsSnapshot::write_json_file(const std::string &fname, bool include_preamble) { - // Throws exception on failed file-open. + // Throws exception on failed file-open. - std::ofstream ofs; - open_ofstream(ofs, fname, __func__); + std::ofstream ofs; + open_ofstream(ofs, fname, __func__); - if (include_preamble) { - ofs << "{ \"dirstate_snapshot\": "; - } + if (include_preamble) + { + ofs << "{ \"dirstate_snapshot\": "; + } - nlohmann::ordered_json j; - to_json(j, *this); + nlohmann::ordered_json j; + to_json(j, *this); - ofs << std::setw(1); - ofs << j; + ofs << std::setw(1); + ofs << j; - if (include_preamble) { - ofs << " }"; - } + if (include_preamble) + { + ofs << " }"; + } - ofs << "\n"; - ofs.close(); + ofs << "\n"; + ofs.close(); } void DataFsSnapshot::dump() { - nlohmann::ordered_json j; // = *this; - to_json(j, *this); - std::cout << j.dump(3) << "\n"; + nlohmann::ordered_json j; // = *this; + to_json(j, *this); + std::cout << j.dump(3) << "\n"; +} + +// DataFsPurgeshot + +int DataFsPurgeshot::find_dir_entry_from_tok(int entry, PathTokenizer &pt, int pos, int *last_existing_entry) const +{ + if (pos == pt.get_n_dirs()) + return entry; + + const DirPurgeElement &dpe = m_dir_vec[entry]; + for (int i = dpe.m_daughters_begin; i != dpe.m_daughters_end; ++i) + { + if (m_dir_vec[i].m_dir_name == pt.get_dir(pos)) { + return find_dir_entry_from_tok(i, pt, pos + 1, last_existing_entry); + } + } + if (last_existing_entry) + *last_existing_entry = entry; + return -1; +} + +int DataFsPurgeshot::find_dir_entry_for_dir_path(const std::string &dir_path) const +{ + PathTokenizer pt(dir_path, -1, false); + return find_dir_entry_from_tok(0, pt, 0, nullptr); +} + +const DirUsage* DataFsPurgeshot::find_dir_usage_for_dir_path(const std::string &dir_path) const +{ + int entry = find_dir_entry_for_dir_path(dir_path); + return entry >= 0 ? &m_dir_vec[entry].m_usage : nullptr; } diff --git a/src/XrdPfc/XrdPfcDirStateSnapshot.hh b/src/XrdPfc/XrdPfcDirStateSnapshot.hh index 86155d5e086..9b2a30a12e0 100644 --- a/src/XrdPfc/XrdPfcDirStateSnapshot.hh +++ b/src/XrdPfc/XrdPfcDirStateSnapshot.hh @@ -12,14 +12,21 @@ namespace XrdPfc { +// For usage / stat reporting + struct DirStateElement : public DirStateBase { + DirStats m_stats; + DirUsage m_usage; + int m_parent = -1; int m_daughters_begin = -1, m_daughters_end = -1; DirStateElement() {} - DirStateElement(const DirStateBase &b, int parent) : + DirStateElement(const DirState &b, int parent) : DirStateBase(b), + m_stats(b.m_here_stats, b.m_recursive_subdir_stats), + m_usage(b.m_here_usage, b.m_recursive_subdir_usage), m_parent(parent) {} }; @@ -29,7 +36,9 @@ struct DataFsSnapshot : public DataFsStateBase std::vector m_dir_states; DataFsSnapshot() {} - DataFsSnapshot(const DataFsStateBase &b) : DataFsStateBase(b) {} + DataFsSnapshot(const DataFsState &b) : + DataFsStateBase(b) + {} // Import of data into vector form is implemented in ResourceMonitor // in order to avoid dependence of this struct on DirState. @@ -38,6 +47,48 @@ struct DataFsSnapshot : public DataFsStateBase void dump(); }; +// For purge planning & execution + +struct DirPurgeElement : public DirStateBase +{ + DirUsage m_usage; + + int m_parent = -1; + int m_daughters_begin = -1, m_daughters_end = -1; + + DirPurgeElement() {} + DirPurgeElement(const DirState &b, int parent) : + DirStateBase(b), + m_usage(b.m_here_usage, b.m_recursive_subdir_usage), + m_parent(parent) + {} +}; + +struct DataFsPurgeshot : public DataFsStateBase +{ + long long m_bytes_to_remove_d = 0, m_bytes_to_remove_f = 0, m_bytes_to_remove = 0; + long long m_estimated_writes_from_writeq = 0; + + bool m_space_based_purge = false; + bool m_age_based_purge = false; + + std::vector m_dir_vec; + // could have parallel vector of DirState* ... or store them in the DirPurgeElement. + // requires some interlock / ref-counting with the source tree. + // or .... just block DirState removal for the duration of the purge :) Yay. + + DataFsPurgeshot() {} + DataFsPurgeshot(const DataFsState &b) : + DataFsStateBase(b) + {} + + int find_dir_entry_from_tok(int entry, PathTokenizer &pt, int pos, int *last_existing_entry) const; + + int find_dir_entry_for_dir_path(const std::string &dir_path) const; + + const DirUsage* find_dir_usage_for_dir_path(const std::string &dir_path) const; +}; + } #endif diff --git a/src/XrdPfc/XrdPfcFPurgeState.cc b/src/XrdPfc/XrdPfcFPurgeState.cc index 759032b2c4f..0105a9dd39b 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.cc +++ b/src/XrdPfc/XrdPfcFPurgeState.cc @@ -28,11 +28,10 @@ const char *FPurgeState::m_traceID = "Purge"; //---------------------------------------------------------------------------- FPurgeState::FPurgeState(long long iNBytesReq, XrdOss &oss) : m_oss(oss), - m_nBytesReq(iNBytesReq), m_nBytesAccum(0), m_nBytesTotal(0), + m_nStBlocksReq((iNBytesReq >> 9) + 1ll), m_nStBlocksAccum(0), m_nStBlocksTotal(0), m_tMinTimeStamp(0), m_tMinUVKeepTimeStamp(0) { - // XXXX init traversal, pass oss? Note, do NOT use DirState (it would have to come from elsewhere) - // well, depends how it's going to be called, eventually + } //---------------------------------------------------------------------------- @@ -60,7 +59,7 @@ void FPurgeState::CheckFile(const FsTraversal &fst, const char *fname, Info &inf { static const char *trc_pfx = "FPurgeState::CheckFile "; - long long nbytes = info.GetNDownloadedBytes(); + long long nblocks = fstat.st_blocks; time_t atime; if (!info.GetLatestDetachTime(atime)) { @@ -70,43 +69,35 @@ void FPurgeState::CheckFile(const FsTraversal &fst, const char *fname, Info &inf } // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime); - m_nBytesTotal += nbytes; - - // XXXX Should remove aged-out files here ... but I have trouble getting - // the DirState and purge report set up consistently. - // Need some serious code reorganization here. - // Biggest problem is maintaining overall state a traversal state consistently. - // Sigh. + m_nStBlocksTotal += nblocks; - // This can be done right with transactional DirState. Also for uvkeep, it seems. + // Could remove aged-out / uv-keep-failed files here ... or in the calling function that + // can aggreagate info for all files in the directory. - // In first two cases we lie about PurgeCandidate time (set to 0) to get them all removed early. - // The age-based purge atime would also be good as there should be nothing - // before that time in the map anyway. - // But we use 0 as a test in purge loop to make sure we continue even if enough + // For now keep using 0 time as this is used in the purge loop to make sure we continue even if enough // disk-space has been freed. if (m_tMinTimeStamp > 0 && atime < m_tMinTimeStamp) { - m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nbytes, 0)); - m_nBytesAccum += nbytes; + m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nblocks, 0)); + m_nStBlocksAccum += nblocks; } else if (m_tMinUVKeepTimeStamp > 0 && Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) && info.GetNoCkSumTimeForUVKeep() < m_tMinUVKeepTimeStamp) { - m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nbytes, 0)); - m_nBytesAccum += nbytes; + m_flist.push_back(PurgeCandidate(fst.m_current_path, fname, nblocks, 0)); + m_nStBlocksAccum += nblocks; } - else if (m_nBytesAccum < m_nBytesReq || (!m_fmap.empty() && atime < m_fmap.rbegin()->first)) + else if (m_nStBlocksAccum < m_nStBlocksReq || (!m_fmap.empty() && atime < m_fmap.rbegin()->first)) { - m_fmap.insert(std::make_pair(atime, PurgeCandidate(fst.m_current_path, fname, nbytes, atime))); - m_nBytesAccum += nbytes; + m_fmap.insert(std::make_pair(atime, PurgeCandidate(fst.m_current_path, fname, nblocks, atime))); + m_nStBlocksAccum += nblocks; // remove newest files from map if necessary - while (!m_fmap.empty() && m_nBytesAccum - m_fmap.rbegin()->second.nBytes >= m_nBytesReq) + while (!m_fmap.empty() && m_nStBlocksAccum - m_fmap.rbegin()->second.nStBlocks >= m_nStBlocksReq) { - m_nBytesAccum -= m_fmap.rbegin()->second.nBytes; + m_nStBlocksAccum -= m_fmap.rbegin()->second.nStBlocks; m_fmap.erase(--(m_fmap.rbegin().base())); } } @@ -141,6 +132,7 @@ void FPurgeState::ProcessDirAndRecurse(FsTraversal &fst) // generate purge event or not? or just flag possible discrepancy? // should this really be done in some other consistency-check traversal? } + fst.close_delete(fh); // XXX ? What do we do with the data-only / cinfo only ? // Protected top-directories are skipped. @@ -180,20 +172,20 @@ bool FPurgeState::TraverseNamespace(const char *root_path) } /* -void FPurgeState::UnlinkInfoAndData(const char *fname, long long nbytes, XrdOssDF *iOssDF) +void FPurgeState::UnlinkInfoAndData(const char *fname, long long nblocks, XrdOssDF *iOssDF) { fname[fname_len - m_info_ext_len] = 0; - if (nbytes > 0) + if (nblocks > 0) { if ( ! Cache.GetInstance().IsFileActiveOrPurgeProtected(dataPath)) { m_n_purged++; - m_bytes_purged += nbytes; + m_bytes_purged += nblocks; } else { m_n_purge_protected++; - m_bytes_purge_protected += nbytes; - m_dir_state->add_usage_purged(nbytes); + m_bytes_purge_protected += nblocks; + m_dir_state->add_usage_purged(nblocks); // XXXX should also tweak other stuff? fname[fname_len - m_info_ext_len] = '.'; return; diff --git a/src/XrdPfc/XrdPfcFPurgeState.hh b/src/XrdPfc/XrdPfcFPurgeState.hh index e2b8a39dd90..a3167a4248a 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.hh +++ b/src/XrdPfc/XrdPfcFPurgeState.hh @@ -25,11 +25,11 @@ public: struct PurgeCandidate // unknown meaning, "file that is candidate for purge", PurgeCandidate would be better. { std::string path; - long long nBytes; + long long nStBlocks; time_t time; PurgeCandidate(const std::string &dname, const char *fname, long long n, time_t t) : - path(dname + fname), nBytes(n), time(t) + path(dname + fname), nStBlocks(n), time(t) {} }; @@ -41,9 +41,9 @@ public: private: XrdOss &m_oss; - long long m_nBytesReq; - long long m_nBytesAccum; - long long m_nBytesTotal; + long long m_nStBlocksReq; + long long m_nStBlocksAccum; + long long m_nStBlocksTotal; time_t m_tMinTimeStamp; time_t m_tMinUVKeepTimeStamp; @@ -61,7 +61,8 @@ public: void setMinTime(time_t min_time) { m_tMinTimeStamp = min_time; } time_t getMinTime() const { return m_tMinTimeStamp; } void setUVKeepMinTime(time_t min_time) { m_tMinUVKeepTimeStamp = min_time; } - long long getNBytesTotal() const { return m_nBytesTotal; } + long long getNStBlocksTotal() const { return m_nStBlocksTotal; } + long long getNBytesTotal() const { return 512ll * m_nStBlocksTotal; } void MoveListEntriesToMap(); diff --git a/src/XrdPfc/XrdPfcFile.cc b/src/XrdPfc/XrdPfcFile.cc index 8b559560be6..51c3f452442 100644 --- a/src/XrdPfc/XrdPfcFile.cc +++ b/src/XrdPfc/XrdPfcFile.cc @@ -26,7 +26,6 @@ #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClFile.hh" -#include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysTimer.hh" #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" @@ -159,6 +158,10 @@ void File::check_delta_stats() void File::report_and_merge_delta_stats() { // Called under m_state_cond lock. + struct stat s; + m_data_file->Fstat(&s); + m_delta_stats.m_StBlocksAdded = s.st_blocks - m_st_blocks; + m_st_blocks = s.st_blocks; Cache::ResMon().register_file_update_stats(m_resmon_token, m_delta_stats); m_stats.AddUp(m_delta_stats); m_delta_stats.Reset(); @@ -387,7 +390,11 @@ bool File::Open() static const char *tpfx = "Open() "; - TRACEF(Dump, tpfx << "open file for disk cache"); + TRACEF(Dump, tpfx << "entered"); + + // Before touching anything, check with ResourceMonitor if a scan is in progress. + // This function will wait internally if needed until it is safe to proceed. + Cache::ResMon().CrossCheckIfScanIsInProgress(m_filename, m_state_cond); const Configuration &conf = Cache::GetInstance().RefConfiguration(); @@ -424,7 +431,7 @@ bool File::Open() return false; } - myEnv.Put("oss.asize", "64k"); // TODO: Calculate? Get it from configuration? Do not know length of access lists ... + myEnv.Put("oss.asize", "64k"); // Advisory, block-map and access list lengths vary. myEnv.Put("oss.cgroup", conf.m_meta_space.c_str()); if ((res = myOss.Create(myUser, ifn.c_str(), 0600, myEnv, XRDOSS_mkpath)) != XrdOssOK) { @@ -460,6 +467,7 @@ bool File::Open() TRACEF(Warning, tpfx << "Basic sanity checks on data file failed, resetting info file, truncating data file."); m_cfi.ResetAllAccessStats(); m_data_file->Ftruncate(0); + Cache::ResMon().register_file_purge(m_filename, data_stat.st_blocks); } } @@ -472,6 +480,7 @@ bool File::Open() initialize_info_file = true; m_cfi.ResetAllAccessStats(); m_data_file->Ftruncate(0); + Cache::ResMon().register_file_purge(m_filename, data_stat.st_blocks); } else { // TODO: If the file is complete, we don't need to reset net cksums. m_cfi.DowngradeCkSumState(conf.get_cs_Chk()); @@ -487,6 +496,12 @@ bool File::Open() m_info_file->Fsync(); TRACEF(Debug, tpfx << "Creating new file info, data size = " << m_file_size << " num blocks = " << m_cfi.GetNBlocks()); } + else + { + if (futimens(m_info_file->getFD(), NULL)) { + TRACEF(Error, tpfx << "failed setting modification time " << ERRNO_AND_ERRSTR(errno)); + } + } m_cfi.WriteIOStatAttach(); m_state_cond.Lock(); @@ -494,6 +509,9 @@ bool File::Open() m_num_blocks = m_cfi.GetNBlocks(); m_prefetch_state = (m_cfi.IsComplete()) ? kComplete : kStopped; // Will engage in AddIO(). + m_data_file->Fstat(&data_stat); + m_st_blocks = data_stat.st_blocks; + m_resmon_token = Cache::ResMon().register_file_open(m_filename, time(0), data_existed); m_resmon_report_threshold = std::min(std::max(200ll * 1024, m_file_size / 50), 500ll * 1024 * 1024); // m_resmon_report_threshold_scaler; // something like 10% of original threshold, to adjust diff --git a/src/XrdPfc/XrdPfcFile.hh b/src/XrdPfc/XrdPfcFile.hh index e142956de54..d280d710bab 100644 --- a/src/XrdPfc/XrdPfcFile.hh +++ b/src/XrdPfc/XrdPfcFile.hh @@ -347,8 +347,9 @@ private: Stats m_stats; //!< cache statistics for this instance Stats m_delta_stats; //!< unreported updates to stats - int m_resmon_token; //!< token used in communication with the ResourceMonitor + long long m_st_blocks; //!< last reported st_blocks long long m_resmon_report_threshold; + int m_resmon_token; //!< token used in communication with the ResourceMonitor void check_delta_stats(); void report_and_merge_delta_stats(); diff --git a/src/XrdPfc/XrdPfcFsTraversal.cc b/src/XrdPfc/XrdPfcFsTraversal.cc index 26ea261fb6f..3bebbc227cc 100644 --- a/src/XrdPfc/XrdPfcFsTraversal.cc +++ b/src/XrdPfc/XrdPfcFsTraversal.cc @@ -27,6 +27,17 @@ FsTraversal::FsTraversal(XrdOss &oss) : FsTraversal::~FsTraversal() {} +int FsTraversal::close_delete(XrdOssDF *&ossDF) +{ + int ret = 0; + if (ossDF) { + ret = ossDF->Close(); + delete ossDF; + } + ossDF = nullptr; + return ret; +} + //---------------------------------------------------------------------------- bool FsTraversal::begin_traversal(DirState *root, const char *root_path) @@ -127,11 +138,19 @@ void FsTraversal::slurp_current_dir() { static const char *trc_pfx = "FsTraversal::slurp_current_dir "; + XrdOssDF &dh = *m_dir_handle_stack.back(); + slurp_dir_ll(dh, m_rel_dir_level, m_current_path.c_str(), trc_pfx); +} + +//---------------------------------------------------------------------------- + +void FsTraversal::slurp_dir_ll(XrdOssDF &dh, int dir_level, const char *path, const char *trc_pfx) +{ + // Low-level implementation of slurp dir. + char fname[256]; struct stat fstat; - XrdOucEnv env; - XrdOssDF &dh = *m_dir_handle_stack.back(); dh.StatRet(&fstat); const char *info_ext = Info::s_infoExtension; @@ -151,7 +170,7 @@ void FsTraversal::slurp_current_dir() } if (rc != XrdOssOK) { - TRACE(Error, trc_pfx << "Readdir error at " << m_current_path << ", err " << XrdSysE2T(-rc) << "."); + TRACE(Error, trc_pfx << "Readdir error at " << path << ", err " << XrdSysE2T(-rc) << "."); break; } @@ -159,7 +178,7 @@ void FsTraversal::slurp_current_dir() if (fname[0] == 0) { - TRACE_PURGE(" Finished reading dir [" << m_current_path << "]. Break loop."); + TRACE_PURGE(" Finished reading dir [" << path << "]. Break loop."); break; } if (fname[0] == '.' && (fname[1] == 0 || (fname[1] == '.' && fname[2] == 0))) @@ -170,7 +189,7 @@ void FsTraversal::slurp_current_dir() if (S_ISDIR(fstat.st_mode)) { - if (m_rel_dir_level == 0 && m_protected_top_dirs.find(fname) != m_protected_top_dirs.end()) + if (dir_level == 0 && m_protected_top_dirs.find(fname) != m_protected_top_dirs.end()) { // Skip protected top-directories. continue; @@ -192,49 +211,5 @@ void FsTraversal::slurp_current_dir() m_current_files[fname].set_data(fstat); } } - - /* - size_t fname_len = strlen(fname); - XrdOssDF *dfh = 0; - - if (S_ISDIR(fstat.st_mode)) - { - if (m_oss_at.Opendir(*iOssDF, fname, env, dfh) == XrdOssOK) - { - cd_down(fname); - TRACE_PURGE(" cd_down -> [" << m_current_path << "]."); - TraverseNamespace(dfh); - cd_up(); - TRACE_PURGE(" cd_up -> [" << m_current_path << "]."); - } - else - TRACE(Warning, trc_pfx << "could not opendir [" << m_current_path << fname << "], " << XrdSysE2T(errno)); - } - else if (fname_len > info_ext_len && strncmp(&fname[fname_len - info_ext_len], info_ext, info_ext_len) == 0) - { - // Check if the file is currently opened / purge-protected is done before unlinking of the file. - - Info cinfo(GetTrace()); - - if (m_oss_at.OpenRO(*iOssDF, fname, env, dfh) == XrdOssOK && cinfo.Read(dfh, m_current_path.c_str(), fname)) - { - CheckFile(fname, cinfo, fstat); - } - else - { - TRACE(Warning, trc_pfx << "can't open or read " << m_current_path << fname << ", err " << XrdSysE2T(errno) << "; purging."); - m_oss_at.Unlink(*iOssDF, fname); - fname[fname_len - info_ext_len] = 0; - m_oss_at.Unlink(*iOssDF, fname); - // generate purge event or not? or just flag possible discrepancy? - } - } - else // XXXX devel debug only, to be removed - { - TRACE_PURGE(" Ignoring [" << fname << "], not a dir or cinfo."); - } - - delete dfh; - */ } } diff --git a/src/XrdPfc/XrdPfcFsTraversal.hh b/src/XrdPfc/XrdPfcFsTraversal.hh index baec350f084..bf65d3169da 100644 --- a/src/XrdPfc/XrdPfcFsTraversal.hh +++ b/src/XrdPfc/XrdPfcFsTraversal.hh @@ -55,6 +55,7 @@ public: static const char *m_traceID; void slurp_current_dir(); + void slurp_dir_ll(XrdOssDF &dh, int dir_level, const char *path, const char *trc_pfx); public: FsTraversal(XrdOss &oss); @@ -67,12 +68,15 @@ public: bool cd_down(const std::string &dir_name); void cd_up(); - int open_at_ro(const char* fname, XrdOssDF *&ossDF) { + int open_at_ro(const char* fname, XrdOssDF *&ossDF) { return m_oss_at.OpenRO(*m_dir_handle_stack.back(), fname, m_env, ossDF); } - int unlink_at(const char* fname) { + int unlink_at(const char* fname) { return m_oss_at.Unlink(*m_dir_handle_stack.back(), fname); } + int close_delete(XrdOssDF *&ossDF); + + XrdOucEnv& default_env() { return m_env; } }; } diff --git a/src/XrdPfc/XrdPfcIO.hh b/src/XrdPfc/XrdPfcIO.hh index 731624ac54c..d4a8ac419b5 100644 --- a/src/XrdPfc/XrdPfcIO.hh +++ b/src/XrdPfc/XrdPfcIO.hh @@ -5,7 +5,6 @@ class XrdSysTrace; #include "XrdPfc.hh" #include "XrdOuc/XrdOucCache.hh" -#include "XrdSys/XrdSysPthread.hh" #include namespace XrdPfc diff --git a/src/XrdPfc/XrdPfcIOFile.cc b/src/XrdPfc/XrdPfcIOFile.cc index bca6bda0f3f..77a1160b7f0 100644 --- a/src/XrdPfc/XrdPfcIOFile.cc +++ b/src/XrdPfc/XrdPfcIOFile.cc @@ -23,7 +23,6 @@ #include "XrdOss/XrdOss.hh" #include "XrdSfs/XrdSfsInterface.hh" #include "XrdSys/XrdSysError.hh" -#include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucPgrwUtils.hh" diff --git a/src/XrdPfc/XrdPfcIOFile.hh b/src/XrdPfc/XrdPfcIOFile.hh index 46d43a8aa92..1aebc2cecdd 100644 --- a/src/XrdPfc/XrdPfcIOFile.hh +++ b/src/XrdPfc/XrdPfcIOFile.hh @@ -20,7 +20,6 @@ #include -#include "XrdSys/XrdSysPthread.hh" #include "XrdPfcIO.hh" #include "XrdPfc.hh" #include "XrdPfcStats.hh" diff --git a/src/XrdPfc/XrdPfcIOFileBlock.hh b/src/XrdPfc/XrdPfcIOFileBlock.hh index 46603bdcce8..8b5effc3eae 100644 --- a/src/XrdPfc/XrdPfcIOFileBlock.hh +++ b/src/XrdPfc/XrdPfcIOFileBlock.hh @@ -21,7 +21,6 @@ #include #include "XrdOuc/XrdOucCache.hh" -#include "XrdSys/XrdSysPthread.hh" #include "XrdPfcIO.hh" diff --git a/src/XrdPfc/XrdPfcInfo.hh b/src/XrdPfc/XrdPfcInfo.hh index ef91a6a2826..9d199dbad78 100644 --- a/src/XrdPfc/XrdPfcInfo.hh +++ b/src/XrdPfc/XrdPfcInfo.hh @@ -20,8 +20,6 @@ #include "XrdPfcTypes.hh" -#include "XrdSys/XrdSysPthread.hh" - #include #include #include diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index 8d3f07d22a2..de042891eaa 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -1,344 +1,156 @@ #include "XrdPfc.hh" -#include "XrdPfcPathParseTools.hh" -#include "XrdPfcDirState.hh" +#include "XrdPfcDirStateSnapshot.hh" +#include "XrdPfcResourceMonitor.hh" #include "XrdPfcFPurgeState.hh" #include "XrdPfcPurgePin.hh" #include "XrdPfcTrace.hh" -#include -#include - -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucUtils.hh" #include "XrdOss/XrdOss.hh" -#include "XrdOss/XrdOssAt.hh" -#include "XrdSys/XrdSysTrace.hh" -using namespace XrdPfc; - -//============================================================================== -// anonymous -//============================================================================== +#include namespace { - -class ScanAndPurgeJob : public XrdJob -{ -public: - ScanAndPurgeJob(const char *desc = "") : XrdJob(desc) {} - - void DoIt() {} // { Cache::GetInstance().ScanAndPurge(); } -}; - + XrdSysTrace* GetTrace() { return XrdPfc::Cache::GetInstance().GetTrace(); } + const char *m_traceID = "ResourceMonitor"; } //============================================================================== -// Cache methods +// OldStylePurgeDriver //============================================================================== - - - -//============================================================================== - -void Cache::Purge() +namespace XrdPfc { - static const char *trc_pfx = "Purge() "; - XrdOucEnv env; - long long disk_usage; - long long estimated_file_usage = m_configuration.m_diskUsageHWM; +void OldStylePurgeDriver(DataFsPurgeshot &ps) +{ + static const char *trc_pfx = "OldStylePurgeDriver "; - // Pause before initial run - sleep(1); + const auto &cache = Cache::TheOne(); + const auto &conf = Cache::Conf(); + auto &resmon = Cache::ResMon(); + auto &oss = *cache.GetOss(); - // { PathTokenizer p("/a/b/c/f.root", 2, true); p.deboog(); } - // { PathTokenizer p("/a/b/f.root", 2, true); p.deboog(); } - // { PathTokenizer p("/a/f.root", 2, true); p.deboog(); } - // { PathTokenizer p("/f.root", 2, true); p.deboog(); } + TRACE(Info, trc_pfx << "Started."); + time_t purge_start = time(0); - int age_based_purge_countdown = 0; // enforce on first purge loop entry. - bool is_first = true; - while (true) - { - time_t purge_start = time(0); + FPurgeState purgeState(2 * ps.m_bytes_to_remove, oss); // prepare twice more volume than required - { - XrdSysCondVarHelper lock(&m_active_cond); + // Make a map of file paths, sorted by access time. - m_in_purge = true; - } + if (ps.m_age_based_purge) + { + purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge); + } + if (conf.is_uvkeep_purge_in_effect()) + { + purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep); + } - TRACE(Info, trc_pfx << "Started."); + bool scan_ok = purgeState.TraverseNamespace("/"); + if ( ! scan_ok) { + TRACE(Error, trc_pfx << "namespace traversal failed at top-directory, this should not happen."); + return; + } - // Bytes to remove based on total disk usage (d) and file usage (f). - long long bytesToRemove_d = 0, bytesToRemove_f = 0; + TRACE(Debug, trc_pfx << "usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes."); - // get amount of space to potentially erase based on total disk usage - XrdOssVSInfo sP; // Make sure we start when a clean slate in each loop - if (m_oss->StatVS(&sP, m_configuration.m_data_space.c_str(), 1) < 0) - { - TRACE(Error, trc_pfx << "can't get StatVS for oss space " << m_configuration.m_data_space); - continue; - } - else - { - disk_usage = sP.Total - sP.Free; - TRACE(Debug, trc_pfx << "used disk space " << disk_usage << " bytes."); + purgeState.MoveListEntriesToMap(); - if (disk_usage > m_configuration.m_diskUsageHWM) - { - bytesToRemove_d = disk_usage - m_configuration.m_diskUsageLWM; - } - } - - // estimate amount of space to erase based on file usage - if (m_configuration.are_file_usage_limits_set()) + ///////////////////////////////////////////////////////////// + /// PurgePin begin + PurgePin *purge_pin = cache.GetPurgePin(); + if (purge_pin) + { + // set dir stat for each path and calculate nBytes to recover for each path + // return total bytes to recover within the plugin + long long clearVal = purge_pin->GetBytesToRecover(ps); + if (clearVal) { - long long estimated_writes_since_last_purge; + TRACE(Debug, "PurgePin remove total " << clearVal << " bytes"); + PurgePin::list_t &dpl = purge_pin->refDirInfos(); + // iterate through the plugin paths + for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) { - XrdSysCondVarHelper lock(&m_writeQ.condVar); - - estimated_writes_since_last_purge = m_writeQ.writes_between_purges; - m_writeQ.writes_between_purges = 0; - } - estimated_file_usage += estimated_writes_since_last_purge; - - TRACE(Debug, trc_pfx << "estimated usage by files " << estimated_file_usage << " bytes."); + TRACE(Debug, trc_pfx << "PurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes"); - bytesToRemove_f = std::max(estimated_file_usage - m_configuration.m_fileUsageNominal, 0ll); - - // Here we estimate fractional usages -- to decide if full scan is necessary before actual purge. - double frac_du = 0, frac_fu = 0; - m_configuration.calculate_fractional_usages(disk_usage, estimated_file_usage, frac_du, frac_fu); + FPurgeState fps(ppit->nBytesToRecover, oss); + bool scan_ok = fps.TraverseNamespace(ppit->path.c_str()); + if ( ! scan_ok) { + TRACE(Warning, trc_pfx << "purge-pin scan of directory failed for " << ppit->path); + continue; + } - if (frac_fu > 1.0 - frac_du) - { - bytesToRemove_f = std::max(bytesToRemove_f, disk_usage - m_configuration.m_diskUsageLWM); + // fill central map from the plugin entry + for (FPurgeState::map_i it = fps.refMap().begin(); it != fps.refMap().end(); ++it) + { + it->second.path = ppit->path + it->second.path; + TRACE(Debug, trc_pfx << "PurgePin found file " << it->second.path.c_str()<< " size " << 512ll * it->second.nStBlocks); + purgeState.refMap().insert(std::make_pair(0, it->second)); // set atime to zero to make sure this is deleted + } } } - - long long bytesToRemove = std::max(bytesToRemove_d, bytesToRemove_f); - - bool enforce_age_based_purge = false; - if (m_configuration.is_age_based_purge_in_effect() || m_configuration.is_uvkeep_purge_in_effect()) + } + /// PurgePin end + ///////////////////////////////////////////////////////////// + + int deleted_file_count = 0; + long long deleted_st_blocks = 0; + + // Loop over map and remove files with oldest values of access time. + struct stat fstat; + int protected_cnt = 0; + long long protected_st_blocks = 0; + long long st_blocks_to_remove = (ps.m_bytes_to_remove << 9) + 1ll; + for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) + { + // Finish when enough space has been freed but not while age-based purging is in progress. + // Those files are marked with time-stamp = 0. + if (st_blocks_to_remove <= 0 && it->first != 0) { - // XXXX ... I could collect those guys in larger vectors (maps?) and do traversal when - // they are empty. - if (--age_based_purge_countdown <= 0) - { - enforce_age_based_purge = true; - age_based_purge_countdown = m_configuration.m_purgeAgeBasedPeriod; - } + break; } - // XXXX this will happen in heart_beat() - bool enforce_traversal_for_usage_collection = is_first; - - // XXXX as above - // copy_out_active_stats_and_update_data_fs_state(); - - TRACE(Debug, trc_pfx << "Precheck:"); - TRACE(Debug, "\tbytes_to_remove_disk = " << bytesToRemove_d << " B"); - TRACE(Debug, "\tbytes_to remove_files = " << bytesToRemove_f << " B (" << (is_first ? "max possible for initial run" : "estimated") << ")"); - TRACE(Debug, "\tbytes_to_remove = " << bytesToRemove << " B"); - TRACE(Debug, "\tenforce_age_based_purge = " << enforce_age_based_purge); - is_first = false; + std::string &infoPath = it->second.path; + std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen); - long long bytesToRemove_at_start = 0; // set after file scan - int deleted_file_count = 0; - - bool purge_required = (bytesToRemove > 0 || enforce_age_based_purge); - - // XXXX-PurgeOpt Need to retain this state between purges so I can avoid doing - // the traversal more often than really needed. - FPurgeState purgeState(2 * bytesToRemove, *m_oss); // prepare twice more volume than required - - if (purge_required || enforce_traversal_for_usage_collection) + if (cache.IsFileActiveOrPurgeProtected(dataPath)) { - // Make a map of file paths, sorted by access time. - - if (m_configuration.is_age_based_purge_in_effect()) - { - purgeState.setMinTime(time(0) - m_configuration.m_purgeColdFilesAge); - } - if (m_configuration.is_uvkeep_purge_in_effect()) - { - purgeState.setUVKeepMinTime(time(0) - m_configuration.m_cs_UVKeep); - } - - bool scan_ok = purgeState.TraverseNamespace("/"); - if ( ! scan_ok) { - TRACE(Error, trc_pfx << "namespace traversal failed at top-directory, this should not happen."); - // XXX once this runs in a job / independent thread, stop processing. - } - - estimated_file_usage = purgeState.getNBytesTotal(); - - TRACE(Debug, trc_pfx << "actual usage by files " << estimated_file_usage << " bytes."); - - // Adjust bytesToRemove_f and then bytesToRemove based on actual file usage, - // possibly retreating below nominal file usage (but not below baseline file usage). - if (m_configuration.are_file_usage_limits_set()) - { - bytesToRemove_f = std::max(estimated_file_usage - m_configuration.m_fileUsageNominal, 0ll); - - double frac_du = 0, frac_fu = 0; - m_configuration.calculate_fractional_usages(disk_usage, estimated_file_usage, frac_du, frac_fu); - - if (frac_fu > 1.0 - frac_du) - { - bytesToRemove = std::max(bytesToRemove_f, disk_usage - m_configuration.m_diskUsageLWM); - bytesToRemove = std::min(bytesToRemove, estimated_file_usage - m_configuration.m_fileUsageBaseline); - } - else - { - bytesToRemove = std::max(bytesToRemove_d, bytesToRemove_f); - } - } - else - { - bytesToRemove = std::max(bytesToRemove_d, bytesToRemove_f); - } - bytesToRemove_at_start = bytesToRemove; - - TRACE(Debug, trc_pfx << "After scan:"); - TRACE(Debug, "\tbytes_to_remove_disk = " << bytesToRemove_d << " B"); - TRACE(Debug, "\tbytes_to remove_files = " << bytesToRemove_f << " B (measured)"); - TRACE(Debug, "\tbytes_to_remove = " << bytesToRemove << " B"); - TRACE(Debug, "\tenforce_age_based_purge = " << enforce_age_based_purge); - TRACE(Debug, "\tmin_time = " << purgeState.getMinTime()); - - if (enforce_age_based_purge) - { - purgeState.MoveListEntriesToMap(); - } + ++protected_cnt; + protected_st_blocks += it->second.nStBlocks; + TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks); + continue; } - ///////////////////////////////////////////////////////////// - /// - /// PurgePin begin - /// - ///////////////////////////////////////////////////////////// - if (m_purge_pin) + // remove info file + if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK) { - // set dir stat for each path and calculate nBytes to recover for each path - // return total bytes to recover within the plugin - long long clearVal = m_purge_pin->GetBytesToRecover(nullptr); // m_fs_state->get_root()); - if (clearVal) - { - TRACE(Debug, "PurgePin remove total " << clearVal << " bytes"); - PurgePin::list_t &dpl = m_purge_pin->refDirInfos(); - // iterate through the plugin paths - for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit) - { - TRACE(Debug, "\tPurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes"); - - FPurgeState fps(ppit->nBytesToRecover, *m_oss); - bool scan_ok = fps.TraverseNamespace(ppit->path.c_str()); - if ( ! scan_ok) { - continue; - } - - // fill central map from the plugin entry - for (FPurgeState::map_i it = fps.refMap().begin(); it != fps.refMap().end(); ++it) - { - it->second.path = ppit->path + it->second.path; - TRACE(Debug, "\t\tPurgePin found file " << it->second.path.c_str()<< " size " << it->second.nBytes); - purgeState.refMap().insert(std::make_pair(0, it->second)); // set atime to zero to make sure this is deleted - } - } - bytesToRemove = std::max(clearVal, bytesToRemove); - purge_required = true; // set the falg! - } + oss.Unlink(infoPath.c_str()); + TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size); } - ///////////////////////////////////////////////////////////// - /// - /// PurgePin end - /// - ///////////////////////////////////////////////////////////// - if (purge_required) + // remove data file + if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK) { - // Loop over map and remove files with oldest values of access time. - struct stat fstat; - int protected_cnt = 0; - long long protected_sum = 0; - for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) - { - // Finish when enough space has been freed but not while age-based purging is in progress. - // Those files are marked with time-stamp = 0. - if (bytesToRemove <= 0 && ! (enforce_age_based_purge && it->first == 0)) - { - break; - } - - std::string &infoPath = it->second.path; - std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen); - - if (IsFileActiveOrPurgeProtected(dataPath)) - { - ++protected_cnt; - protected_sum += it->second.nBytes; - TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << it->second.nBytes); - continue; - } - - // remove info file - if (m_oss->Stat(infoPath.c_str(), &fstat) == XrdOssOK) - { - // cinfo file can be on another oss.space, do not subtract for now. - // Could be relevant for very small block sizes. - // bytesToRemove -= fstat.st_size; - // estimated_file_usage -= fstat.st_size; - // ++deleted_file_count; - - m_oss->Unlink(infoPath.c_str()); - TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << fstat.st_size); - } - - // remove data file - if (m_oss->Stat(dataPath.c_str(), &fstat) == XrdOssOK) - { - bytesToRemove -= it->second.nBytes; - estimated_file_usage -= it->second.nBytes; - ++deleted_file_count; - - m_oss->Unlink(dataPath.c_str()); - TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << it->second.nBytes << ", time: " << it->first); - - // XXXXX Report removal in some other way, aggregate by dirname, get DirState from somewhere else: - /* - if (it->second.dirState != 0) // XXXX This should now always be true. - it->second.dirState->add_usage_purged(it->second.nBytes); - else - TRACE(Error, trc_pfx << "DirState not set for file '" << dataPath << "'."); - */ - } - } - if (protected_cnt > 0) - { - TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << protected_sum); - } - } + st_blocks_to_remove -= it->second.nStBlocks; + deleted_st_blocks += it->second.nStBlocks; + ++deleted_file_count; - { - XrdSysCondVarHelper lock(&m_active_cond); + oss.Unlink(dataPath.c_str()); + TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first); - m_purge_delay_set.clear(); - m_in_purge = false; + resmon.register_file_purge(dataPath, it->second.nStBlocks); } + } + if (protected_cnt > 0) + { + TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks); + } - int purge_duration = time(0) - purge_start; - - TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, total size " << - bytesToRemove_at_start - bytesToRemove << ", bytes to remove at end " << bytesToRemove << ", purge duration " << purge_duration); + int purge_duration = time(0) - purge_start; - int sleep_time = m_configuration.m_purgeInterval - purge_duration; - if (sleep_time > 0) - { - sleep(sleep_time); - } - } + TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks + << ", purge duration " << purge_duration); } + +} // end namespace XrdPfc diff --git a/src/XrdPfc/XrdPfcPurgePin.hh b/src/XrdPfc/XrdPfcPurgePin.hh index 2dfb2dfdba9..9bcb9bb30e0 100644 --- a/src/XrdPfc/XrdPfcPurgePin.hh +++ b/src/XrdPfc/XrdPfcPurgePin.hh @@ -6,7 +6,8 @@ namespace XrdPfc { -class DirState; +class DataFsPurgeshot; +class DirUsage; //---------------------------------------------------------------------------- //! Base class for reguesting directory space to obtain. @@ -21,7 +22,7 @@ public: long long nBytesToRecover{0}; // internal use by the Cache purge thread. to be revisited, maybe an access token is more appropriate. - DirState *dirState{nullptr}; + const DirUsage* dirUsage{nullptr}; }; typedef std::vector list_t; @@ -42,7 +43,7 @@ public: //! //! @return total number of bytes //--------------------------------------------------------------------- - virtual long long GetBytesToRecover(DirState *) = 0; + virtual long long GetBytesToRecover(const DataFsPurgeshot&) = 0; //------------------------------------------------------------------------------ //! Parse configuration arguments. diff --git a/src/XrdPfc/XrdPfcPurgeQuota.cc b/src/XrdPfc/XrdPfcPurgeQuota.cc index 0b499b742fa..cb74d84ccc9 100644 --- a/src/XrdPfc/XrdPfcPurgeQuota.cc +++ b/src/XrdPfc/XrdPfcPurgeQuota.cc @@ -1,6 +1,6 @@ #include "XrdPfc.hh" #include "XrdPfcPurgePin.hh" -#include "XrdPfcDirState.hh" +#include "XrdPfcDirStateSnapshot.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucUtils.hh" @@ -11,34 +11,39 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin { + XrdSysError *log; public: - XrdPfcPurgeQuota() {} + XrdPfcPurgeQuota() : log(XrdPfc::Cache::GetInstance().GetLog()) {} //---------------------------------------------------------------------------- //! Set directory statistics //---------------------------------------------------------------------------- - void InitDirStatesForLocalPaths(XrdPfc::DirState *rootDS) + void InitDirStatesForLocalPaths(const XrdPfc::DataFsPurgeshot &purge_shot) { for (list_i it = m_list.begin(); it != m_list.end(); ++it) { - it->dirState = rootDS->find_path(it->path, XrdPfc::Cache::Conf().m_dirStatsStoreDepth, false, false); + it->dirUsage = purge_shot.find_dir_usage_for_dir_path(it->path); } } //---------------------------------------------------------------------------- //! Provide bytes to erase from dir quota listed in a text file //---------------------------------------------------------------------------- - virtual long long GetBytesToRecover(XrdPfc::DirState *ds) + long long GetBytesToRecover(const XrdPfc::DataFsPurgeshot &purge_shot) override { // setup diskusage for each dir path - InitDirStatesForLocalPaths(ds); + InitDirStatesForLocalPaths(purge_shot); long long totalToRemove = 0; // get bytes to remove for (list_i it = m_list.begin(); it != m_list.end(); ++it) { - // XXXXXX here we should have another mechanism. and probably add up here + subdirs - long long cv = it->dirState->recursive_subdir_usage().m_BytesOnDisk - it->nBytesQuota; + if (it->dirUsage == nullptr) + { + log->Emsg("PurgeQuotaPin--GetBytesToRecover", "directory not found:", it->path.c_str()); + continue; + } + long long cv = 512ll * it->dirUsage->m_StBlocks - it->nBytesQuota; if (cv > 0) it->nBytesToRecover = cv; else @@ -53,17 +58,15 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin //---------------------------------------------------------------------------- //! Provide bytes to erase from dir quota listed in a text file //---------------------------------------------------------------------------- - virtual bool ConfigPurgePin(const char *parms) + bool ConfigPurgePin(const char *parms) override { - XrdSysError *log = XrdPfc::Cache::GetInstance().GetLog(); - // retrive configuration file name if (!parms || !parms[0] || (strlen(parms) == 0)) { - log->Emsg("ConfigDecision", "Quota file not specified."); + log->Emsg("ConfigPurgePin", "Quota file not specified."); return false; } - log->Emsg("ConfigDecision", "Using directory list", parms); + log->Emsg("ConfigPurgePin", "Using directory list", parms); // parse the file to get directory quotas const char *config_filename = parms; @@ -74,7 +77,7 @@ class XrdPfcPurgeQuota : public XrdPfc::PurgePin int fd; if ((fd = open(config_filename, O_RDONLY, 0)) < 0) { - log->Emsg("Config() can't open configuration file ", config_filename); + log->Emsg("ConfigPurgePin() can't open configuration file ", config_filename); } Config.Attach(fd); diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 00b2616e6e3..ce9b054fbb3 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -6,6 +6,15 @@ #include "XrdPfcDirStateSnapshot.hh" #include "XrdPfcTrace.hh" +#include "XrdOss/XrdOss.hh" + +#define RM_DEBUG +#ifdef RM_DEBUG +#define dprintf(...) printf(__VA_ARGS__) +#else +#define dprintf(...) (void(0)) +#endif + using namespace XrdPfc; namespace @@ -14,65 +23,139 @@ namespace const char *m_traceID = "ResourceMonitor"; } +//------------------------------------------------------------------------------ + ResourceMonitor::ResourceMonitor(XrdOss& oss) : - m_fs_state(new DataFsState), + m_fs_state(* new DataFsState), m_oss(oss) {} ResourceMonitor::~ResourceMonitor() { - delete m_fs_state; + delete &m_fs_state; } //------------------------------------------------------------------------------ // Initial scan //------------------------------------------------------------------------------ +void ResourceMonitor::CrossCheckIfScanIsInProgress(const std::string &lfn, XrdSysCondVar &cond) +{ + m_dir_scan_mutex.Lock(); + if (m_dir_scan_in_progress) { + m_dir_scan_open_requests.push_back({lfn, cond}); + LfnCondRecord &lcr = m_dir_scan_open_requests.back(); + cond.Lock(); + m_dir_scan_mutex.UnLock(); + while ( ! lcr.f_checked) + cond.Wait(); + cond.UnLock(); + } else { + m_dir_scan_mutex.UnLock(); + } +} + +void ResourceMonitor::process_inter_dir_scan_open_requests(FsTraversal &fst) +{ + m_dir_scan_mutex.Lock(); + while ( ! m_dir_scan_open_requests.empty()) + { + LfnCondRecord &lcr = m_dir_scan_open_requests.front(); + m_dir_scan_mutex.UnLock(); + + cross_check_or_process_oob_lfn(lcr.f_lfn, fst); + lcr.f_cond.Lock(); + lcr.f_checked = true; + lcr.f_cond.Signal(); + lcr.f_cond.UnLock(); + + m_dir_scan_mutex.Lock(); + m_dir_scan_open_requests.pop_front(); + } + m_dir_scan_mutex.UnLock(); +} + +void ResourceMonitor::cross_check_or_process_oob_lfn(const std::string &lfn, FsTraversal &fst) +{ + // Check if lfn has already been processed ... or process it now and mark + // the DirState accordingly (partially processed oob). + static const char *trc_pfx = "cross_check_or_process_oob_lfn() "; + + DirState *last_existing_ds = nullptr; + DirState *ds = m_fs_state.find_dirstate_for_lfn(lfn, &last_existing_ds); + if (ds->m_scanned) + return; + + size_t pos = lfn.find_last_of("/"); + std::string dir = (pos == std::string::npos) ? "" : lfn.substr(0, pos); + + XrdOssDF *dhp = m_oss.newDir(trc_pfx); + if (dhp->Opendir(dir.c_str(), fst.default_env()) == XrdOssOK) + { + fst.slurp_dir_ll(*dhp, ds->m_depth, dir.c_str(), trc_pfx); + + // XXXX clone of function below .... move somewhere? Esp. removal of non-paired files? + DirUsage &here = ds->m_here_usage; + for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it) + { + if (it->second.has_data && it->second.has_cinfo) { + here.m_StBlocks += it->second.stat_data.st_blocks; + here.m_NFiles += 1; + } + } + } + delete dhp; + ds->m_scanned = true; +} + void ResourceMonitor::scan_dir_and_recurse(FsTraversal &fst) { - printf("In scan_dir_and_recurse for '%s', size of dir_vec = %d, file_stat_map = %d\n", + dprintf("In scan_dir_and_recurse for '%s', size of dir_vec = %d, file_stat_map = %d\n", fst.m_current_path.c_str(), (int)fst.m_current_dirs.size(), (int)fst.m_current_files.size()); - // Breadth first, accumulate into "here" - DirUsage &here = fst.m_dir_state->m_here_usage; - for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it) + // Breadth first, accumulate into "here", unless it was already scanned via an + // OOB open file request. + if ( ! fst.m_dir_state->m_scanned) { - printf("would be doing something with %s ... has_data=%d, has_cinfo=%d\n", - it->first.c_str(), it->second.has_data, it->second.has_cinfo); - - // XXX Make some of these optional? - // Remove files that do not have both cinfo and data? - // Remove empty directories before even descending? - // Leave this for some consistency pass? - // Note that FsTraversal supports ignored paths ... some details (config, N2N) to be clarified. - - if (it->second.has_data && it->second.has_cinfo) { - here.m_BytesOnDisk += it->second.stat_data.st_blocks * 512; - here.m_NFiles += 1; + DirUsage &here = fst.m_dir_state->m_here_usage; + for (auto it = fst.m_current_files.begin(); it != fst.m_current_files.end(); ++it) + { + dprintf("would be doing something with %s ... has_data=%d, has_cinfo=%d\n", + it->first.c_str(), it->second.has_data, it->second.has_cinfo); + + // XXX Make some of these optional? + // Remove files that do not have both cinfo and data? + // Remove empty directories before even descending? + // Leave this for some consistency pass? + // Note that FsTraversal supports ignored paths ... some details (config, N2N) to be clarified. + + if (it->second.has_data && it->second.has_cinfo) { + here.m_StBlocks += it->second.stat_data.st_blocks; + here.m_NFiles += 1; + } } + fst.m_dir_state->m_scanned = true; } - // Sub-dirs second, accumulate into "subdirs". - DirUsage &subdirs = fst.m_dir_state->m_recursive_subdir_usage; + // Swap-out directories as inter_dir_scan can use the FsTraversal. std::vector dirs; dirs.swap(fst.m_current_dirs); + + if (++m_dir_scan_check_counter >= 100) + { + process_inter_dir_scan_open_requests(fst); + m_dir_scan_check_counter = 0; + } + + // Descend into sub-dirs, do not accumulate into recursive_subdir_usage yet. This is done + // in a separate pass to allow for proper accounting of files being opened during the initial scan. for (auto &dname : dirs) { if (fst.cd_down(dname)) { - DirState *daughter = fst.m_dir_state; - DirUsage &dhere = daughter->m_here_usage; - DirUsage &dsubdirs = daughter->m_recursive_subdir_usage; - scan_dir_and_recurse(fst); fst.cd_up(); - - here.m_NDirectories += 1; - - subdirs.m_BytesOnDisk += dhere.m_BytesOnDisk + dsubdirs.m_BytesOnDisk; - subdirs.m_NFiles += dhere.m_NFiles + dsubdirs.m_NFiles; - subdirs.m_NDirectories += dhere.m_NDirectories + dsubdirs.m_NDirectories; } // XXX else try to remove it? } @@ -83,23 +166,50 @@ bool ResourceMonitor::perform_initial_scan() // Called after PFC configuration is complete, but before full startup of the daemon. // Base line usages are accumulated as part of the file-system, traversal. - bool success_p = true; + update_vs_and_file_usage_info(); + DirState *root_ds = m_fs_state.get_root(); FsTraversal fst(m_oss); fst.m_protected_top_dirs.insert("pfc-stats"); // XXXX This should come from config. Also: N2N? - if (fst.begin_traversal(m_fs_state->get_root(), "/")) + if ( ! fst.begin_traversal(root_ds, "/")) + return false; + { - scan_dir_and_recurse(fst); + XrdSysMutexHelper _lock(m_dir_scan_mutex); + m_dir_scan_in_progress = true; + m_dir_scan_check_counter = 0; // recheck oob file-open requests periodically. } - else + + scan_dir_and_recurse(fst); + + fst.end_traversal(); + + // We have all directories scanned, available in DirState tree, let all remaining files go + // and then we shall do the upward propagation of usages. { - // Fail startup, can't open /. - success_p = false; + XrdSysMutexHelper _lock(m_dir_scan_mutex); + m_dir_scan_in_progress = false; + m_dir_scan_check_counter = 0; + + while ( ! m_dir_scan_open_requests.empty()) + { + LfnCondRecord &lcr = m_dir_scan_open_requests.front(); + lcr.f_cond.Lock(); + lcr.f_checked = true; + lcr.f_cond.Signal(); + lcr.f_cond.UnLock(); + + m_dir_scan_open_requests.pop_front(); + } } - fst.end_traversal(); - return success_p; + // Do upward propagation of usages. + root_ds->upward_propagate_initial_scan_usages(); + + update_vs_and_file_usage_info(); + + return true; } //------------------------------------------------------------------------------ @@ -114,7 +224,7 @@ int ResourceMonitor::process_queues() // We really want all open records to be processed before file-stats updates // and all those before the close records. // Purges are sort of tangential as they really just modify bytes / number - // of files in a direcotry and do not deal with any persistent file id tokens. + // of files in a directory and do not deal with any persistent file id tokens. int n_records = 0; { @@ -131,15 +241,15 @@ int ResourceMonitor::process_queues() for (auto &i : m_file_open_q.read_queue()) { // i.id: LFN, i.record: OpenRecord - int tid = i.id; - AccessToken &at = token(tid); - printf("process file open for token %d, time %ld -- %s\n", tid, i.record.m_open_time, at.m_filename.c_str()); + AccessToken &at = token(i.id); + dprintf("process file open for token %d, time %ld -- %s\n", + i.id, i.record.m_open_time, at.m_filename.c_str()); // Resolve fname into DirState. // We could clear the filename after this ... or keep it, should we need it later on. // For now it is just used for debug printouts. DirState *last_existing_ds = nullptr; - DirState *ds = m_fs_state->find_dirstate_for_lfn(at.m_filename, &last_existing_ds); + DirState *ds = m_fs_state.find_dirstate_for_lfn(at.m_filename, &last_existing_ds); at.m_dir_state = ds; ds->m_here_stats.m_NFilesOpened += 1; @@ -159,72 +269,67 @@ int ResourceMonitor::process_queues() for (auto &i : m_file_update_stats_q.read_queue()) { // i.id: token, i.record: Stats - int tid = i.id; - AccessToken &at = token(tid); + AccessToken &at = token(i.id); // Stats DirState *ds = at.m_dir_state; - printf("process file update for token %d, %p -- %s\n", - tid, ds, at.m_filename.c_str()); + dprintf("process file update for token %d, %p -- %s\n", + i.id, ds, at.m_filename.c_str()); ds->m_here_stats.AddUp(i.record); + m_current_usage_in_st_blocks += i.record.m_StBlocksAdded; } for (auto &i : m_file_close_q.read_queue()) { // i.id: token, i.record: CloseRecord - int tid = i.id; - AccessToken &at = token(tid); - printf("process file close for token %d, time %ld -- %s\n", - tid, i.record.m_close_time, at.m_filename.c_str()); + AccessToken &at = token(i.id); + dprintf("process file close for token %d, time %ld -- %s\n", + i.id, i.record.m_close_time, at.m_filename.c_str()); DirState *ds = at.m_dir_state; ds->m_here_stats.m_NFilesClosed += 1; - ds->m_here_usage.m_LastCloseTime = i.record.m_close_time; - // If full-stats write is not a multiple of 512-bytes that means we wrote the - // tail block. As usage is measured in 512-byte blocks this gets rounded up here. - long long over_512b = i.record.m_full_stats.m_BytesWritten & 0x1FFll; - if (over_512b) - ds->m_here_stats.m_BytesWritten += 512ll - over_512b; - - // Release the AccessToken! at.clear(); - m_access_tokens_free_slots.push_back(tid); + } + { // Release the AccessToken slots under lock. + XrdSysMutexHelper _lock(&m_queue_mutex); + for (auto &i : m_file_close_q.read_queue()) + m_access_tokens_free_slots.push_back(i.id); } for (auto &i : m_file_purge_q1.read_queue()) { // i.id: DirState*, i.record: PurgeRecord DirState *ds = i.id; - // NOTE -- bytes removed should already be rounded up to 512-bytes for each file. - ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; - ds->m_here_stats.m_NFilesRemoved += i.record.n_files; + ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks; + ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files; + m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks; } for (auto &i : m_file_purge_q2.read_queue()) { // i.id: directory-path, i.record: PurgeRecord - DirState *ds = m_fs_state->get_root()->find_path(i.id, -1, false, false); + DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, false, false); if ( ! ds) { TRACE(Error, trc_pfx << "DirState not found for directory path '" << i.id << "'."); // find_path can return the last dir found ... but this clearly isn't a valid purge record. continue; } - // NOTE -- bytes removed should already be rounded up to 512-bytes for each file. - ds->m_here_stats.m_BytesRemoved += i.record.m_total_size; - ds->m_here_stats.m_NFilesRemoved += i.record.n_files; + ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks; + ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files; + m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks; } for (auto &i : m_file_purge_q3.read_queue()) { - // i.id: LFN, i.record: size of file - DirState *ds = m_fs_state->get_root()->find_path(i.id, -1, true, false); + // i.id: LFN, i.record: size of file in st_blocks + DirState *ds = m_fs_state.get_root()->find_path(i.id, -1, true, false); if ( ! ds) { TRACE(Error, trc_pfx << "DirState not found for LFN path '" << i.id << "'."); continue; } - // NOTE -- bytes removed should already be rounded up to 512-bytes. - ds->m_here_stats.m_BytesRemoved += i.record; - ds->m_here_stats.m_NFilesRemoved += 1; + ds->m_here_stats.m_StBlocksRemoved += i.record; + ds->m_here_stats.m_NFilesRemoved += 1; + m_current_usage_in_st_blocks -= i.record; } // Read queues / vectors are cleared at swap time. @@ -239,82 +344,120 @@ int ResourceMonitor::process_queues() void ResourceMonitor::heart_beat() { - printf("RMon entering heart_beat!\n"); + static const char *tpfx = "heart_beat() "; + + const Configuration &conf = Cache::Conf(); + const DirState &root_ds = *m_fs_state.get_root(); + + const int s_queue_proc_interval = 10; + // const s_stats_up_prop_interval = 60; -- for when we have dedicated purge / stat report structs + const int s_sshot_report_interval = 60; // to be bumped (300s?) or made configurable. + const int s_purge_check_interval = 60; + const int s_purge_report_interval = conf.m_purgeInterval; + const int s_purge_cold_files_interval = conf.m_purgeInterval * conf.m_purgeAgeBasedPeriod; // initial scan performed as part of config time_t now = time(0); - time_t next_queue_proc_time = now + 10; - time_t next_up_prop_time = now + 60; + time_t next_queue_proc_time = now + s_queue_proc_interval; + time_t next_sshot_report_time = now + s_sshot_report_interval; + time_t next_purge_check_time = now + s_purge_check_interval; + time_t next_purge_report_time = now + s_purge_report_interval; + time_t next_purge_cold_files_time = now + s_purge_cold_files_interval; + + // XXXXX On initial entry should reclaim space from queues as they might have grown + // very large during the initial scan. while (true) { time_t start = time(0); - time_t next_event = std::min(next_queue_proc_time, next_up_prop_time); + time_t next_event = std::min({ next_queue_proc_time, next_sshot_report_time, + next_purge_check_time, next_purge_report_time, next_purge_cold_files_time }); + if (next_event > start) { unsigned int t_sleep = next_event - start; - printf("sleeping for %u seconds, to be improved ...\n", t_sleep); + TRACE(Debug, tpfx << "sleeping for " << t_sleep << " seconds until the next beat."); sleep(t_sleep); } + // Check if purge has been running and has completed yet. + // For now this is only used to prevent removal of empty leaf directories + // during stat propagation so we do not need to wait for the condition in + // the above sleep. + if (m_purge_task_active) { + MutexHolder _lck(m_purge_task_cond); + if (m_purge_task_complete) { + m_purge_task_active = m_purge_task_complete = false; + } + } + + // Always process the queues. int n_processed = process_queues(); - next_queue_proc_time += 10; - printf("processed %d records\n", n_processed); + next_queue_proc_time += s_queue_proc_interval; + TRACE(Debug, tpfx << "process_queues -- n_records=" << n_processed); - if (next_up_prop_time > time(0)) - continue; + // Always update basic info on m_fs_state (space, usage, file_usage). + update_vs_and_file_usage_info(); - m_fs_state->upward_propagate_stats_and_times(); - next_up_prop_time += 60; + now = time(0); + if (next_sshot_report_time <= now) + { + next_sshot_report_time += s_sshot_report_interval; - // Here we can learn assumed file-based usage - // run the "disk-usage" - // decide if age-based purge needs to be run (or uvkeep one) - // decide if the standard / plugin purge needs to be called + // XXXX pass in m_purge_task_active as control over "should empty dirs be purged"; + // Or should this be separate pass or variant in purge? + m_fs_state.upward_propagate_stats_and_times(); + m_fs_state.apply_stats_to_usages(); - // Check time, is it time to export into vector format, to disk, into /pfc-stats. - // This one should really be rather timely ... as it will be used for calculation - // of averages of stuff going on. + // Dump statistics before actual purging so maximum usage values get recorded. + // This should dump out binary snapshot into /pfc-stats/, if so configured. + // Also, optionally, json. + // Could also go to gstream but this easily gets too large. + if (conf.is_dir_stat_reporting_on()) + { + const int store_depth = conf.m_dirStatsStoreDepth; + const int n_sshot_dirs = root_ds.count_dirs_to_level(store_depth); + dprintf("Snapshot n_dirs=%d, total n_dirs=%d\n", n_sshot_dirs, + root_ds.m_here_usage.m_NDirectories + root_ds.m_recursive_subdir_usage.m_NDirectories + 1); - m_fs_state->apply_stats_to_usages(); + m_fs_state.dump_recursively(store_depth); - // Dump statistcs before actual purging so maximum usage values get recorded. - // This should dump out binary snapshot into /pfc-stats/, if so configured. - // Also, optionally, json. - // Could also go to gstream but this could easily be way too large. - if (Cache::Conf().is_dir_stat_reporting_on()) - { - const int StoreDepth = Cache::Conf().m_dirStatsStoreDepth; - const DirState &root_ds = *m_fs_state->get_root(); - const int n_sshot_dirs = root_ds.count_dirs_to_level(StoreDepth); - printf("Snapshot n_dirs=%d, total n_dirs=%d\n", n_sshot_dirs, - root_ds.m_here_usage.m_NDirectories + root_ds.m_recursive_subdir_usage.m_NDirectories + 1); + DataFsSnapshot ss(m_fs_state); + ss.m_dir_states.reserve(n_sshot_dirs); - m_fs_state->dump_recursively(StoreDepth); + ss.m_dir_states.emplace_back( DirStateElement(root_ds, -1) ); + fill_sshot_vec_children(root_ds, 0, ss.m_dir_states, store_depth); - DataFsSnapshot ss(*m_fs_state); - ss.m_dir_states.reserve(n_sshot_dirs); + // This should really be export to a file (preferably binary, but then bin->json command is needed, too). + ss.dump(); + } - ss.m_dir_states.emplace_back( DirStateElement(root_ds, -1) ); - fill_sshot_vec_children(root_ds, 0, ss.m_dir_states, StoreDepth); + m_fs_state.reset_stats(); - ss.dump(); + now = time(0); } - // if needed export to vector form - // - write to file - // - pass to purge pin + bool do_purge_check = next_purge_check_time <= now; + bool do_purge_report = next_purge_report_time <= now; + bool do_purge_cold_files = next_purge_cold_files_time <= now; + if (do_purge_check || do_purge_report || do_purge_cold_files) + { + perform_purge_check(do_purge_cold_files, do_purge_report ? TRACE_Info : TRACE_Debug); - m_fs_state->reset_stats(); + next_purge_check_time = now + s_purge_check_interval; + if (do_purge_report) next_purge_report_time = now + s_purge_report_interval; + if (do_purge_cold_files) next_purge_cold_files_time = now + s_purge_cold_files_interval; + } - // check time / diskusage --> purge condition? - // run purge as job or thread - // m_fs_state->upward_propagate_usage_purged(); // XXXX this is the old way - } + } // end while forever } +//------------------------------------------------------------------------------ +// DirState export helpers +//------------------------------------------------------------------------------ + void ResourceMonitor::fill_sshot_vec_children(const DirState &parent_ds, int parent_idx, std::vector &vec, @@ -343,6 +486,254 @@ void ResourceMonitor::fill_sshot_vec_children(const DirState &parent_ds, } } +void ResourceMonitor::fill_pshot_vec_children(const DirState &parent_ds, + int parent_idx, + std::vector &vec, + int max_depth) +{ + int pos = vec.size(); + int n_children = parent_ds.m_subdirs.size(); + + for (auto const & [name, child] : parent_ds.m_subdirs) + { + vec.emplace_back( DirPurgeElement(child, parent_idx) ); + } + + if (parent_ds.m_depth < max_depth) + { + DirPurgeElement &parent_dpe = vec[parent_idx]; + parent_dpe.m_daughters_begin = pos; + parent_dpe.m_daughters_end = pos + n_children; + + for (auto const & [name, child] : parent_ds.m_subdirs) + { + if (n_children > 0) + fill_pshot_vec_children(child, pos, vec, max_depth); + ++pos; + } + } +} + +//------------------------------------------------------------------------------ +// Purge helpers, drivers, etc. +//------------------------------------------------------------------------------ + +void ResourceMonitor::update_vs_and_file_usage_info() +{ + static const char *trc_pfx = "update_vs_and_file_usage_info() "; + + const auto &conf = Cache::Conf(); + XrdOssVSInfo vsi; + + // StatVS error (after it succeeded in config) implies a memory corruption (according to Mr. H). + if (m_oss.StatVS(&vsi, conf.m_data_space.c_str(), 1) < 0) { + TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_data_space << "'. This is a fatal error."); + _exit(1); + } + m_fs_state.m_disk_total = vsi.Total; + m_fs_state.m_disk_used = vsi.Total - vsi.Free; + m_fs_state.m_file_usage = 512ll * m_current_usage_in_st_blocks; + if (m_oss.StatVS(&vsi, conf.m_meta_space.c_str(), 1) < 0) { + TRACE(Error, trc_pfx << "can't get StatVS for oss space '" << conf.m_meta_space << "'. This is a fatal error."); + _exit(1); + } + m_fs_state.m_meta_total = vsi.Total; + m_fs_state.m_meta_used = vsi.Total - vsi.Free; +} + +void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) +{ + static const char *trc_pfx = "perform_purge_check() "; + const Configuration &conf = Cache::Conf(); + + std::unique_ptr psp( new DataFsPurgeshot(m_fs_state) ); + DataFsPurgeshot &ps = *psp; + + // Purge precheck I. -- available disk space + + if (ps.m_disk_used > conf.m_diskUsageHWM) { + ps.m_bytes_to_remove_d = ps.m_disk_used - conf.m_diskUsageLWM; + ps.m_space_based_purge = true; + } + + // Purge precheck II. -- usage by files. + // Keep it updated, but only act on it if so configured. + + ps.m_file_usage = 512ll * m_current_usage_in_st_blocks; + // These are potentially wrong as cache might be writing over preallocated byte ranges. + ps.m_estimated_writes_from_writeq = Cache::GetInstance().WritesSinceLastCall(); + // Can have another estimate based on eiter writes or st-blocks from purge-stats, once we have them. + + if (conf.are_file_usage_limits_set()) + { + ps.m_bytes_to_remove_f = std::max(ps.m_file_usage - conf.m_fileUsageNominal, 0ll); + + // Here we estimate fractional usages -- to decide if full scan is necessary before actual purge. + double frac_du = 0, frac_fu = 0; + conf.calculate_fractional_usages(ps.m_disk_used, ps.m_file_usage, frac_du, frac_fu); + + if (frac_fu > 1.0 - frac_du) + { + ps.m_bytes_to_remove_f = std::max(ps.m_bytes_to_remove_f, ps.m_disk_used - conf.m_diskUsageLWM); + ps.m_space_based_purge = true; + } + } + + ps.m_bytes_to_remove = std::max(ps.m_bytes_to_remove_d, ps.m_bytes_to_remove_f); + + // Purge precheck III. -- check if age-based purge is required + // We ignore uvkeep time, it requires reading of cinfo files and it is enforced in File::Open() anyway. + + if (purge_cold_files && conf.is_age_based_purge_in_effect()) // || conf.is_uvkeep_purge_in_effect()) + { + ps.m_age_based_purge = true; + } + + TRACE_INT(tl, trc_pfx << "Purge check:"); + TRACE_INT(tl, "\tbytes_to_remove_disk = " << ps.m_bytes_to_remove_d << " B"); + TRACE_INT(tl, "\tbytes_to remove_files = " << ps.m_bytes_to_remove_f << " B"); + TRACE_INT(tl, "\tbytes_to_remove = " << ps.m_bytes_to_remove << " B"); + TRACE_INT(tl, "\tspace_based_purge = " << ps.m_space_based_purge); + TRACE_INT(tl, "\tage_based_purge = " << ps.m_age_based_purge); + + if ( ! ps.m_space_based_purge && ! ps.m_age_based_purge) { + TRACE(Info, trc_pfx << "purge not required."); + Cache::GetInstance().ClearPurgeProtectedSet(); + return; + } + if (m_purge_task_active) { + TRACE(Warning, trc_pfx << "purge required but previous purge task is still active!"); + return; + } + + TRACE(Info, trc_pfx << "purge required ... scheduling purge task."); + + // At this point we have all the information: report, decide on action. + // There is still some missing infrastructure, especially as regards to purge-plugin: + // - at what point do we start bugging the pu-pin to start coughing up purge lists? + // - have a new parameter or just do it "one cycle before full"? + // - what if it doesn't -- when do we do the old-stlye scan & purge? + // - how do we do age-based purge and uvkeep purge? + // - they are really quite different -- and could run separately, registering + // files into a purge-candidate list. This has to be rechecked before the actual + // deletion -- eg, by comparing stat time of cinfo + doing the is-active / is-purge-protected. + + const DirState &root_ds = *m_fs_state.get_root(); + const int n_pshot_dirs = root_ds.count_dirs_to_level(9999); + const int n_calc_dirs = 1 + root_ds.m_here_usage.m_NDirectories + root_ds.m_recursive_subdir_usage.m_NDirectories; + dprintf("purge dir count recursive=%d vs from_usage=%d\n", n_pshot_dirs, n_calc_dirs); + + ps.m_dir_vec.reserve(n_calc_dirs); + ps.m_dir_vec.emplace_back( DirPurgeElement(root_ds, -1) ); + fill_pshot_vec_children(root_ds, 0, ps.m_dir_vec, 9999); + + m_purge_task_active = true; + + struct PurgeDriverJob : public XrdJob + { + DataFsPurgeshot *m_purge_shot_ptr; + + PurgeDriverJob(DataFsPurgeshot *psp) : + XrdJob("XrdPfc::ResourceMonitor::PurgeDriver"), + m_purge_shot_ptr(psp) + {} + + void DoIt() override + { + Cache::ResMon().perform_purge_task(*m_purge_shot_ptr); + Cache::ResMon().perform_purge_task_cleanup(); + + delete m_purge_shot_ptr; + delete this; + } + }; + + Cache::schedP->Schedule( new PurgeDriverJob(psp.release()) ); +} + +namespace XrdPfc +{ + void OldStylePurgeDriver(DataFsPurgeshot &ps); +} + +void ResourceMonitor::perform_purge_task(DataFsPurgeshot &ps) +{ + // BEWARE: Runs in a dedicated thread - is only to communicate back to the + // hear_beat() / data structs via the purge queues and condition variable. + + // const char *tpfx = "perform_purge_task "; + + { + MutexHolder _lck(m_purge_task_cond); + m_purge_task_start = time(0); + } + + // For now, fall back to the old purge ... to be improved with: + // - new scan, following the DataFsPurgeshot; + // - usage of cinfo stat mtime for time of last access (touch already done at output); + // - use DirState* to report back purged files. + // Already changed to report back purged files --- but using the string / path variant. + OldStylePurgeDriver(ps); // In XrdPfcPurge.cc +} + +void ResourceMonitor::perform_purge_task_cleanup() +{ + // Separated out so the purge_task can exit without post-checks. + + { + MutexHolder _lck(m_purge_task_cond); + m_purge_task_end = time(0); + m_purge_task_complete = true; + m_purge_task_cond.Signal(); + } + Cache::GetInstance().ClearPurgeProtectedSet(); +} + +//============================================================================== +// Main thread function, do initial test, then enter heart_beat(). +//============================================================================== + +void ResourceMonitor::init_before_main() +{ + // setup for in-scan -- this is called from initial setup. + MutexHolder _lck(m_dir_scan_mutex); + m_dir_scan_in_progress = true; +} + +void ResourceMonitor::main_thread_function() +{ + const char *tpfx = "main_thread_function "; + { + time_t is_start = time(0); + TRACE(Info, tpfx << "Stating initial directory scan."); + + if ( ! perform_initial_scan()) { + TRACE(Error, tpfx << "Initial directory scan has failed. This is a terminal error, aborting.") + _exit(1); + } + // Reset of m_dir_scan_in_progress is done in perform_initial_scan() + + time_t is_duration = time(0) - is_start; + TRACE(Info, tpfx << "Initial directory scan complete, duration=" << is_duration <<"s"); + + // run first process queues + int n_proc_is = process_queues(); + TRACE(Info, tpfx << "First process_queues finished, n_records=" << n_proc_is); + + // shrink queues if scan time was longer than 30s. + if (is_duration > 30 || n_proc_is > 3000) + { + m_file_open_q.shrink_read_queue(); + m_file_update_stats_q.shrink_read_queue(); + m_file_close_q.shrink_read_queue(); + m_file_purge_q1.shrink_read_queue(); + m_file_purge_q2.shrink_read_queue(); + m_file_purge_q3.shrink_read_queue(); + } + } + heart_beat(); +} + //============================================================================== // Old prototype from Cache / Purge, now to go into heart_beat() here, above. //============================================================================== @@ -431,7 +822,7 @@ void Proto_ResourceMonitorHeartBeat() // TRACE(Info, trc_pfx << "HeartBeat finished, heartbeat_duration " << heartbeat_duration); - // int sleep_time = m_configuration.m_purgeInterval - heartbeat_duration; + // int sleep_time = m_fs_state..m_purgeInterval - heartbeat_duration; int sleep_time = 60 - heartbeat_duration; if (sleep_time > 0) { diff --git a/src/XrdPfc/XrdPfcResourceMonitor.hh b/src/XrdPfc/XrdPfcResourceMonitor.hh index a051116e554..ed5724b07bf 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.hh +++ b/src/XrdPfc/XrdPfcResourceMonitor.hh @@ -7,6 +7,7 @@ #include #include +#include class XrdOss; @@ -15,6 +16,9 @@ namespace XrdPfc { class DataFsState; class DirState; class DirStateElement; +class DataFsSnapshot; +class DirPurgeElement; +class DataFsPurgeshot; class FsTraversal; //============================================================================== @@ -59,6 +63,9 @@ class ResourceMonitor iterator begin() const { return m_read_queue.begin(); } iterator end() const { return m_read_queue.end(); } + // Shrinkage of overgrown queues + void shrink_read_queue() { m_read_queue.clear(); m_read_queue.shrink_to_fit(); } + private: queue_type m_write_queue, m_read_queue; }; @@ -90,8 +97,8 @@ class ResourceMonitor }; struct PurgeRecord { - long long m_total_size; - int n_files; + long long m_size_in_st_blocks; + int m_n_files; }; Queue m_file_open_q; @@ -102,12 +109,31 @@ class ResourceMonitor Queue m_file_purge_q3; // DirPurge queue -- not needed? But we do need last-change timestamp in DirState. + long long m_current_usage_in_st_blocks = 0; // aggregate disk usage by files + XrdSysMutex m_queue_mutex; // mutex shared between queues - unsigned int m_queue_swap_u1 = 0u; // identifier of current swap + unsigned int m_queue_swap_u1 = 0u; // identifier of current swap cycle - DataFsState *m_fs_state; + DataFsState &m_fs_state; XrdOss &m_oss; + // Requests for File opens during name-space scans. Such LFNs are processed + // with some priority + struct LfnCondRecord + { + const std::string &f_lfn; + XrdSysCondVar &f_cond; + bool f_checked = false; + }; + + XrdSysMutex m_dir_scan_mutex; + std::list m_dir_scan_open_requests; + int m_dir_scan_check_counter; + bool m_dir_scan_in_progress = false; + + void process_inter_dir_scan_open_requests(FsTraversal &fst); + void cross_check_or_process_oob_lfn(const std::string &lfn, FsTraversal &fst); + public: ResourceMonitor(XrdOss& oss); ~ResourceMonitor(); @@ -127,7 +153,7 @@ public: token_id = m_access_tokens_free_slots.back(); m_access_tokens_free_slots.pop_back(); m_access_tokens[token_id].m_filename = filename; - m_access_tokens[token_id].m_last_write_queue_pos = m_queue_swap_u1 - 1; + m_access_tokens[token_id].m_last_queue_swap_u1 = m_queue_swap_u1 - 1; } else { token_id = (int) m_access_tokens.size(); m_access_tokens.push_back({filename, m_queue_swap_u1 - 1}); @@ -162,27 +188,27 @@ public: // deletions can come from purge and from direct requests (Cache::UnlinkFile), the latter // also covering the emergency shutdown of a file. - void register_file_purge(DirState* target, long long file_size) { + void register_file_purge(DirState* target, long long size_in_st_blocks) { XrdSysMutexHelper _lock(&m_queue_mutex); - m_file_purge_q1.push(target, {file_size, 1}); + m_file_purge_q1.push(target, {size_in_st_blocks, 1}); } - void register_multi_file_purge(DirState* target, long long file_size, int n_files) { + void register_multi_file_purge(DirState* target, long long size_in_st_blocks, int n_files) { XrdSysMutexHelper _lock(&m_queue_mutex); - m_file_purge_q1.push(target, {file_size, n_files}); + m_file_purge_q1.push(target, {size_in_st_blocks, n_files}); } - void register_multi_file_purge(const std::string& target, long long file_size, int n_files) { + void register_multi_file_purge(const std::string& target, long long size_in_st_blocks, int n_files) { XrdSysMutexHelper _lock(&m_queue_mutex); - m_file_purge_q2.push(target, {file_size, n_files}); + m_file_purge_q2.push(target, {size_in_st_blocks, n_files}); } - void register_file_purge(const std::string& filename, long long file_size) { + void register_file_purge(const std::string& filename, long long size_in_st_blocks) { XrdSysMutexHelper _lock(&m_queue_mutex); - m_file_purge_q3.push(filename, file_size); + m_file_purge_q3.push(filename, size_in_st_blocks); } // void register_dir_purge(DirState* target); // target assumed to be empty at this point, triggered by a file_purge removing the last file in it. // hmmh, this is actually tricky ... who will purge the dirs? we should now at export-to-vector time - // and can prune leaf directories. Tthis might fail if a file has been created in there in the meantime, which is ok. + // and can prune leaf directories. This might fail if a file has been created in there in the meantime, which is ok. // However, is there a race condition between rmdir and creation of a new file in that dir? Ask Andy. // --- Helpers for event processing and actions @@ -202,22 +228,34 @@ public: std::vector &vec, int max_depth); + void fill_pshot_vec_children(const DirState &parent_ds, + int parent_idx, + std::vector &vec, + int max_depth); - /* XXX Stuff from Cache, to be revisited. - enum ScanAndPurgeThreadState_e { SPTS_Idle, SPTS_Scan, SPTS_Purge, SPTS_Done }; - - XrdSysCondVar m_stats_n_purge_cond; //!< communication between heart-beat and scan-purge threads - - int m_last_scan_duration; - int m_last_purge_duration; - ScanAndPurgeThreadState_e m_spt_state; - // --- - m_stats_n_purge_cond(0), - m_last_scan_duration(0), - m_last_purge_duration(0), - m_spt_state(SPTS_Idle) - - */ + // Interface to other part of XCache -- note the CamelCase() notation. + void CrossCheckIfScanIsInProgress(const std::string &lfn, XrdSysCondVar &cond); + + // main function, steers startup then enters heart_beat. does not die. + void init_before_main(); // called from startup thread / configuration processing + void main_thread_function(); // run in dedicated thread + + XrdSysCondVar m_purge_task_cond {0}; + // The following variables are set under the above lock, purge task signals to heart_beat. + time_t m_purge_task_start {0}; + time_t m_purge_task_end {0}; + bool m_purge_task_active {false}; // from the perspective of heart-beat, set only in heartbeat + bool m_purge_task_complete {false}; // from the perspective of the task, reset in heartbeat, set in task + // When m_purge_task_active == true, DirState entries are not removed from the tree to + // allow purge thread to report cleared files directly via DirState ptr. + // Note, DirState removal happens during stat propagation traversal. + + // Purge helpers etc. + void update_vs_and_file_usage_info(); + void perform_purge_check(bool purge_cold_files, int tl); + + void perform_purge_task(DataFsPurgeshot &ps); + void perform_purge_task_cleanup(); }; } diff --git a/src/XrdPfc/XrdPfcStats.hh b/src/XrdPfc/XrdPfcStats.hh index 2e42aa38adc..a0e182f763c 100644 --- a/src/XrdPfc/XrdPfcStats.hh +++ b/src/XrdPfc/XrdPfcStats.hh @@ -40,6 +40,7 @@ public: long long m_BytesMissed = 0; //!< number of bytes served from remote and cached long long m_BytesBypassed = 0; //!< number of bytes served directly through XrdCl long long m_BytesWritten = 0; //!< number of bytes written to disk + long long m_StBlocksAdded = 0; //!< number of 512-byte blocks the file has grown by int m_NCksumErrors = 0; //!< number of checksum errors while getting data from remote //---------------------------------------------------------------------- @@ -50,6 +51,17 @@ public: Stats& operator=(const Stats&) = default; + Stats(const Stats& a, const Stats& b) : + m_NumIos (a.m_NumIos + b.m_NumIos), + m_Duration (a.m_Duration + b.m_Duration), + m_BytesHit (a.m_BytesHit + b.m_BytesHit), + m_BytesMissed (a.m_BytesMissed + b.m_BytesMissed), + m_BytesBypassed (a.m_BytesBypassed + b.m_BytesBypassed), + m_BytesWritten (a.m_BytesWritten + b.m_BytesWritten), + m_StBlocksAdded (a.m_StBlocksAdded + b.m_StBlocksAdded), + m_NCksumErrors (a.m_NCksumErrors + b.m_NCksumErrors) + {} + //---------------------------------------------------------------------- void AddReadStats(const Stats &s) @@ -80,7 +92,6 @@ public: m_Duration += duration; } - //---------------------------------------------------------------------- long long BytesRead() const @@ -96,6 +107,7 @@ public: m_BytesMissed = ref.m_BytesMissed - m_BytesMissed; m_BytesBypassed = ref.m_BytesBypassed - m_BytesBypassed; m_BytesWritten = ref.m_BytesWritten - m_BytesWritten; + m_StBlocksAdded = ref.m_StBlocksAdded - m_StBlocksAdded; m_NCksumErrors = ref.m_NCksumErrors - m_NCksumErrors; } @@ -107,6 +119,7 @@ public: m_BytesMissed += s.m_BytesMissed; m_BytesBypassed += s.m_BytesBypassed; m_BytesWritten += s.m_BytesWritten; + m_StBlocksAdded += s.m_StBlocksAdded; m_NCksumErrors += s.m_NCksumErrors; } @@ -118,6 +131,7 @@ public: m_BytesMissed = 0; m_BytesBypassed = 0; m_BytesWritten = 0; + m_StBlocksAdded = 0; m_NCksumErrors = 0; } }; @@ -127,7 +141,7 @@ public: class DirStats : public Stats { public: - long long m_BytesRemoved = 0; + long long m_StBlocksRemoved = 0; // number of 512-byte blocks removed from the directory int m_NFilesOpened = 0; int m_NFilesClosed = 0; int m_NFilesCreated = 0; @@ -143,9 +157,16 @@ public: DirStats& operator=(const DirStats&) = default; - //---------------------------------------------------------------------- - - // maybe some missing AddSomething functions, like for read/write + DirStats(const DirStats& a, const DirStats& b) : + Stats(a, b), + m_StBlocksRemoved (a.m_StBlocksRemoved + b.m_StBlocksRemoved), + m_NFilesOpened (a.m_NFilesOpened + b.m_NFilesOpened), + m_NFilesClosed (a.m_NFilesClosed + b.m_NFilesClosed), + m_NFilesCreated (a.m_NFilesCreated + b.m_NFilesCreated), + m_NFilesRemoved (a.m_NFilesRemoved + b.m_NFilesRemoved), + m_NDirectoriesCreated (a.m_NDirectoriesCreated + b.m_NDirectoriesCreated), + m_NDirectoriesRemoved (a.m_NDirectoriesRemoved + b.m_NDirectoriesRemoved) + {} //---------------------------------------------------------------------- @@ -153,7 +174,7 @@ public: void DeltaToReference(const DirStats& ref) { Stats::DeltaToReference(ref); - m_BytesRemoved = ref.m_BytesRemoved - m_BytesRemoved; + m_StBlocksRemoved = ref.m_StBlocksRemoved - m_StBlocksRemoved; m_NFilesOpened = ref.m_NFilesOpened - m_NFilesOpened; m_NFilesClosed = ref.m_NFilesClosed - m_NFilesClosed; m_NFilesCreated = ref.m_NFilesCreated - m_NFilesCreated; @@ -166,7 +187,7 @@ public: void AddUp(const DirStats& s) { Stats::AddUp(s); - m_BytesRemoved += s.m_BytesRemoved; + m_StBlocksRemoved += s.m_StBlocksRemoved; m_NFilesOpened += s.m_NFilesOpened; m_NFilesClosed += s.m_NFilesClosed; m_NFilesCreated += s.m_NFilesCreated; From baf81a24612b7f743c18b4a1fcdaaa922d622570 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Thu, 23 May 2024 21:22:58 -0700 Subject: [PATCH 28/43] Add missing include for gcc-14. --- src/XrdPfc/XrdPfcResourceMonitor.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index ce9b054fbb3..06654369213 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -8,6 +8,8 @@ #include "XrdOss/XrdOss.hh" +#include + #define RM_DEBUG #ifdef RM_DEBUG #define dprintf(...) printf(__VA_ARGS__) From 806ea8218caea7dc8bf134f3444ad1ef122663b1 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Fri, 24 May 2024 15:53:27 -0700 Subject: [PATCH 29/43] Fix missing initialization of m_current_usage_in_st_blocks after ResourceMonitor::initial_scan. --- src/XrdPfc/XrdPfcResourceMonitor.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 06654369213..3deaa751d6e 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -208,7 +208,8 @@ bool ResourceMonitor::perform_initial_scan() // Do upward propagation of usages. root_ds->upward_propagate_initial_scan_usages(); - + m_current_usage_in_st_blocks = root_ds->m_here_usage.m_StBlocks + + root_ds->m_recursive_subdir_usage.m_StBlocks; update_vs_and_file_usage_info(); return true; @@ -306,7 +307,7 @@ int ResourceMonitor::process_queues() DirState *ds = i.id; ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks; ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files; - m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks; + m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks; } for (auto &i : m_file_purge_q2.read_queue()) { @@ -319,7 +320,7 @@ int ResourceMonitor::process_queues() } ds->m_here_stats.m_StBlocksRemoved += i.record.m_size_in_st_blocks; ds->m_here_stats.m_NFilesRemoved += i.record.m_n_files; - m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks; + m_current_usage_in_st_blocks -= i.record.m_size_in_st_blocks; } for (auto &i : m_file_purge_q3.read_queue()) { @@ -331,7 +332,7 @@ int ResourceMonitor::process_queues() } ds->m_here_stats.m_StBlocksRemoved += i.record; ds->m_here_stats.m_NFilesRemoved += 1; - m_current_usage_in_st_blocks -= i.record; + m_current_usage_in_st_blocks -= i.record; } // Read queues / vectors are cleared at swap time. From 5ef374fce30b448854b0c7ec6f7b8fd483a436bc Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Tue, 28 May 2024 11:56:28 -0700 Subject: [PATCH 30/43] Correction in checking meta-data minimum required space --- src/XrdPfc/XrdPfcConfiguration.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index 8273f0701fc..f541ebaf065 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -448,7 +448,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) m_log.Emsg("ConfigParameters()", "error obtaining stat info for meta space ", m_configuration.m_meta_space.c_str()); return false; } - if (sP.Total < 10ll << 20) + if (m_configuration.m_meta_space != m_configuration.m_data_space && sP.Total < 10ll << 20) { m_log.Emsg("ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ", m_configuration.m_meta_space.c_str()); From 90931f9f0841d64cf67fbf0f49e5d0124cb92f35 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 29 May 2024 15:07:53 -0700 Subject: [PATCH 31/43] Pass initialzied fstat to FPurgeState::CheckFile --- src/XrdPfc/XrdPfcFPurgeState.cc | 10 +++++++--- src/XrdPfc/XrdPfcFsTraversal.hh | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/XrdPfc/XrdPfcFPurgeState.cc b/src/XrdPfc/XrdPfcFPurgeState.cc index 0105a9dd39b..3aa3f50773a 100644 --- a/src/XrdPfc/XrdPfcFPurgeState.cc +++ b/src/XrdPfc/XrdPfcFPurgeState.cc @@ -114,15 +114,20 @@ void FPurgeState::ProcessDirAndRecurse(FsTraversal &fst) const std::string i_name = f_name + Info::s_infoExtension; XrdOssDF *fh = nullptr; - struct stat fstat; Info cinfo(GetTrace()); // XXX Note, the initial scan now uses stat information only! + if (! it->second.has_both()) { + // cinfo or data file is missing. What do we do? Erase? + // Should really be checked in some other "consistency" traversal. + continue; + } + if (fst.open_at_ro(i_name.c_str(), fh) == XrdOssOK && cinfo.Read(fh, fst.m_current_path.c_str(), i_name.c_str())) { - CheckFile(fst, i_name.c_str(), cinfo, fstat); + CheckFile(fst, i_name.c_str(), cinfo, it->second.stat_data); } else { @@ -134,7 +139,6 @@ void FPurgeState::ProcessDirAndRecurse(FsTraversal &fst) } fst.close_delete(fh); - // XXX ? What do we do with the data-only / cinfo only ? // Protected top-directories are skipped. } diff --git a/src/XrdPfc/XrdPfcFsTraversal.hh b/src/XrdPfc/XrdPfcFsTraversal.hh index bf65d3169da..183a4de376a 100644 --- a/src/XrdPfc/XrdPfcFsTraversal.hh +++ b/src/XrdPfc/XrdPfcFsTraversal.hh @@ -27,6 +27,7 @@ public: void set_data (const struct stat &s) { stat_data = s; has_data = true; } void set_cinfo(const struct stat &s) { stat_cinfo = s; has_cinfo = true; } + bool has_both() const { return has_data && has_cinfo; } }; protected: From 285d0390cc2e5133ee944bdacbf972e7c242fcc7 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 29 May 2024 16:11:55 -0700 Subject: [PATCH 32/43] Correction in number of blocks to remove. --- src/XrdPfc/XrdPfcPurge.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index de042891eaa..19b9d662b5d 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -101,7 +101,7 @@ void OldStylePurgeDriver(DataFsPurgeshot &ps) struct stat fstat; int protected_cnt = 0; long long protected_st_blocks = 0; - long long st_blocks_to_remove = (ps.m_bytes_to_remove << 9) + 1ll; + long long st_blocks_to_remove = (ps.m_bytes_to_remove >> 9) + 1ll; for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) { // Finish when enough space has been freed but not while age-based purging is in progress. From bdaa6e386c6c4a2ce10ee3c8f0aca45dca5a5869 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Tue, 11 Jun 2024 11:31:00 -0700 Subject: [PATCH 33/43] Fix xrd_pfc_command/remove_file to properly pass leading slash to oss. Remove non-supported command options from option parser (cut-n-pasted from create_file, apparently). --- src/XrdPfc/XrdPfcCommand.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index 77bbcb471c1..15c527a75a5 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -299,7 +299,7 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) SplitParser ap(token, " "); int argc = ap.fill_argv(argv); - XrdOucArgs Spec(&m_log, err_prefix, "hvs:b:t:d:", + XrdOucArgs Spec(&m_log, err_prefix, "h", "help", 1, "h", (const char *) 0); @@ -326,7 +326,7 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) return; } - std::string f_name(cp.get_reminder()); + std::string f_name(cp.get_reminder_with_delim()); TRACE(Debug, err_prefix << "file argument '" << f_name << "'."); From ae37dd2398281f13dccc1a3d78c5f425a7563677 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Fri, 14 Jun 2024 10:21:23 -0700 Subject: [PATCH 34/43] Correction in interpreting diskusage configuration parameters: Integrate fileusage limits in purge cycle when diskusage high water mark is not reached. --- src/XrdPfc/XrdPfc.cc | 21 --------------------- src/XrdPfc/XrdPfc.hh | 2 -- src/XrdPfc/XrdPfcResourceMonitor.cc | 14 ++++++-------- 3 files changed, 6 insertions(+), 31 deletions(-) diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index 2a6fe92a30e..661ca57b399 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -119,27 +119,6 @@ XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, //============================================================================== -void Configuration::calculate_fractional_usages(long long du, long long fu, - double &frac_du, double &frac_fu) const -{ - // Calculate fractional disk / file usage and clamp them to [0, 1]. - - // Fractional total usage above LWM: - // - can be > 1 if usage is above HWM; - // - can be < 0 if triggered via age-based-purging. - frac_du = (double) (du - m_diskUsageLWM) / (m_diskUsageHWM - m_diskUsageLWM); - - // Fractional file usage above baseline. - // - can be > 1 if file usage is above max; - // - can be < 0 if file usage is below baseline. - frac_fu = (double) (fu - m_fileUsageBaseline) / (m_fileUsageMax - m_fileUsageBaseline); - - frac_du = std::min( std::max( frac_du, 0.0), 1.0 ); - frac_fu = std::min( std::max( frac_fu, 0.0), 1.0 ); -} - -//============================================================================== - Cache &Cache::CreateInstance(XrdSysLogger *logger, XrdOucEnv *env) { assert (m_instance == 0); diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index 6e02ffd42d6..4c3a06b265b 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -70,8 +70,6 @@ struct Configuration bool is_dir_stat_reporting_on() const { return m_dirStatsMaxDepth >= 0 || ! m_dirStatsDirs.empty() || ! m_dirStatsDirGlobs.empty(); } bool is_purge_plugin_set_up() const { return false; } - void calculate_fractional_usages(long long du, long long fu, double &frac_du, double &frac_fu) const; - CkSumCheck_e get_cs_Chk() const { return (CkSumCheck_e) m_cs_Chk; } bool is_cschk_cache() const { return m_cs_Chk & CSChk_Cache; } diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 3deaa751d6e..cc2baba2866 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -569,15 +569,13 @@ void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) if (conf.are_file_usage_limits_set()) { - ps.m_bytes_to_remove_f = std::max(ps.m_file_usage - conf.m_fileUsageNominal, 0ll); - - // Here we estimate fractional usages -- to decide if full scan is necessary before actual purge. - double frac_du = 0, frac_fu = 0; - conf.calculate_fractional_usages(ps.m_disk_used, ps.m_file_usage, frac_du, frac_fu); - - if (frac_fu > 1.0 - frac_du) + if (ps.m_space_based_purge ) + { + ps.m_bytes_to_remove_f = std::max(ps.m_file_usage - conf.m_fileUsageBaseline, 0ll); + } + else if (ps.m_file_usage > conf.m_fileUsageMax) { - ps.m_bytes_to_remove_f = std::max(ps.m_bytes_to_remove_f, ps.m_disk_used - conf.m_diskUsageLWM); + ps.m_bytes_to_remove_f = std::max(ps.m_file_usage - conf.m_fileUsageNominal, 0ll); ps.m_space_based_purge = true; } } From 9c174a1646c6c473a0fe1e7aa3caf698033d8a1b Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Mon, 29 Jul 2024 17:35:12 +0000 Subject: [PATCH 35/43] Small updates encountered when working on Purge Plugin These are mostly things I determined I'd need as I was working on the LotMan plugin. The biggest thing was the addition of various extra public headers that I found I needed to import XrdPfc/XrdPfc.hh --- src/XrdHeaders.cmake | 1 + src/XrdPfc.cmake | 13 +++++++++++++ src/XrdPfc/XrdPfcConfiguration.cc | 4 ++-- src/XrdVersionPlugin.hh | 3 ++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/XrdHeaders.cmake b/src/XrdHeaders.cmake index c723a6a7a81..b7a92f1fd99 100644 --- a/src/XrdHeaders.cmake +++ b/src/XrdHeaders.cmake @@ -26,6 +26,7 @@ set( XROOTD_PUBLIC_HEADERS XrdNet/XrdNetSocket.hh XrdOuc/XrdOucBuffer.hh XrdOuc/XrdOucCRC.hh + XrdOuc/XrdOucCache.hh XrdOuc/XrdOucCacheCM.hh XrdOuc/XrdOucCacheStats.hh XrdOuc/XrdOucCallBack.hh diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index 7c48ff9c822..9e8e59f06b3 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -104,6 +104,19 @@ install( COMMAND ln -sf lib${LIB_XRD_FILECACHE}.so lib${LIB_XRD_FILECACHE_LEGACY}.so WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR} )" ) +install( + FILES + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfcPurgePin.hh + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfcDirStateSnapshot.hh + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfcDirState.hh + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfcStats.hh + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfc.hh + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfcFile.hh + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfcTypes.hh + ${CMAKE_CURRENT_SOURCE_DIR}/XrdPfc/XrdPfcInfo.hh + DESTINATION ${CMAKE_INSTALL_PREFIX}/include/xrootd/XrdPfc +) + install( TARGETS ${LIB_XRD_BLACKLIST} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index f541ebaf065..7c6ea2c1eaa 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -248,7 +248,7 @@ bool Cache::xplib(XrdOucStream &Config) std::string libp; if (! (val = Config.GetWord()) || ! val[0]) { - TRACE(Info," Cache::Config() decisionlib not specified; always caching files"); + TRACE(Info," Cache::Config() purgelib not specified; will use LRU for purging files"); return true; } else @@ -272,7 +272,7 @@ bool Cache::xplib(XrdOucStream &Config) PurgePin * dp = ep(m_log); if (! dp) { - TRACE(Error, "Config() decisionlib was not able to create a directory purge object"); + TRACE(Error, "Config() purgelib was not able to create a Purge Plugin object?"); return false; } m_purge_pin = dp; diff --git a/src/XrdVersionPlugin.hh b/src/XrdVersionPlugin.hh index 121949e6826..1643cf0dbfe 100644 --- a/src/XrdVersionPlugin.hh +++ b/src/XrdVersionPlugin.hh @@ -223,7 +223,8 @@ XrdVERSIONPLUGIN_Mapd(ofs.cmslib, XrdCmsGetClient )\ XrdVERSIONPLUGIN_Mapd(cms.vnid, XrdCmsgetVnId )\ XrdVERSIONPLUGIN_Mapd(cms.perf, XrdCmsPerfMonitor )\ - XrdVERSIONPLUGIN_Mapd(pfc.decisionlib, XrdPfcGetDecision )\ + XrdVERSIONPLUGIN_Mapd(pfc.decisionlib, XrdPfcGetDecision )\ + XrdVERSIONPLUGIN_Mapd(pfc.purgelib, XrdPfcGetPurgePin )\ XrdVERSIONPLUGIN_Mapd(xrd.protocol, XrdgetProtocol )\ XrdVERSIONPLUGIN_Mapd(http.secxtractor, XrdHttpGetSecXtractor )\ XrdVERSIONPLUGIN_Mapd(http.exthandler, XrdHttpGetExtHandler )\ From 002b5e4b46552355ed71410be02b12d620f1d5af Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Mon, 29 Jul 2024 14:28:11 -0700 Subject: [PATCH 36/43] Implement DirStats::Reset() --- src/XrdPfc/XrdPfcStats.hh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/XrdPfc/XrdPfcStats.hh b/src/XrdPfc/XrdPfcStats.hh index a0e182f763c..1a94426bf63 100644 --- a/src/XrdPfc/XrdPfcStats.hh +++ b/src/XrdPfc/XrdPfcStats.hh @@ -195,6 +195,19 @@ public: m_NDirectoriesCreated += s.m_NDirectoriesCreated; m_NDirectoriesRemoved += s.m_NDirectoriesRemoved; } + + using Stats::Reset; // activate overload based on arg + void Reset() + { + Stats::Reset(); + m_StBlocksRemoved = 0; + m_NFilesOpened = 0; + m_NFilesClosed = 0; + m_NFilesCreated = 0; + m_NFilesRemoved = 0; + m_NDirectoriesCreated = 0; + m_NDirectoriesRemoved = 0; + } }; } From e41c579411d3f6c592db11eaef76a431bab5db44 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 31 Jul 2024 16:00:19 -0700 Subject: [PATCH 37/43] Check all file usage values are less that highWatermark in the diskusage section --- src/XrdPfc/XrdPfcConfiguration.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index 7c6ea2c1eaa..387786fc132 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -491,6 +491,13 @@ bool Cache::Config(const char *config_filename, const char *parameters) m_log.Emsg("ConfigParameters()", "pfc.diskusage files should have baseline < nominal < max."); aOK = false; } + + + if (aOK && m_configuration.m_fileUsageMax >= m_configuration.m_diskUsageLWM) + { + m_log.Emsg("ConfigParameters()", "pfc.diskusage files values must be below lowWatermark"); + aOK = false; + } } else aOK = false; } From aa398e605637207276783bb4e467a5c0621e792a Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 31 Jul 2024 16:01:55 -0700 Subject: [PATCH 38/43] Modify assert in the FsTraversal::begin_traversal: file paths must begin with /, but do not have to end with it. --- src/XrdPfc/XrdPfcFsTraversal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPfc/XrdPfcFsTraversal.cc b/src/XrdPfc/XrdPfcFsTraversal.cc index 3bebbc227cc..552341cea4e 100644 --- a/src/XrdPfc/XrdPfcFsTraversal.cc +++ b/src/XrdPfc/XrdPfcFsTraversal.cc @@ -54,7 +54,7 @@ bool FsTraversal::begin_traversal(const char *root_path) { static const char *trc_pfx = "FsTraversal::begin_traversal "; - assert(root_path && strlen(root_path) > 0 && root_path[strlen(root_path) - 1] == '/'); + assert(root_path && strlen(root_path) > 0 && root_path[0] == '/'); m_rel_dir_level = 0; m_current_path = root_path; From ccad0d5d80a12268b5c0d2f7087346fee8ea5e71 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Wed, 31 Jul 2024 20:30:09 -0700 Subject: [PATCH 39/43] Correct calculation of bytes to remove in perform_purge_check() when file usage limits are sets: base, nominal, and max --- src/XrdPfc/XrdPfcDirStateSnapshot.hh | 2 +- src/XrdPfc/XrdPfcResourceMonitor.cc | 148 ++++++++++++++++++++++----- src/XrdPfc/XrdPfcResourceMonitor.hh | 1 + 3 files changed, 127 insertions(+), 24 deletions(-) diff --git a/src/XrdPfc/XrdPfcDirStateSnapshot.hh b/src/XrdPfc/XrdPfcDirStateSnapshot.hh index 9b2a30a12e0..045f5854fba 100644 --- a/src/XrdPfc/XrdPfcDirStateSnapshot.hh +++ b/src/XrdPfc/XrdPfcDirStateSnapshot.hh @@ -66,7 +66,7 @@ struct DirPurgeElement : public DirStateBase struct DataFsPurgeshot : public DataFsStateBase { - long long m_bytes_to_remove_d = 0, m_bytes_to_remove_f = 0, m_bytes_to_remove = 0; + long long m_bytes_to_remove = 0; long long m_estimated_writes_from_writeq = 0; bool m_space_based_purge = false; diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index cc2baba2866..91a3e976032 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -427,6 +427,8 @@ void ResourceMonitor::heart_beat() m_fs_state.dump_recursively(store_depth); + /* + // json dump to std::out for debug purpose DataFsSnapshot ss(m_fs_state); ss.m_dir_states.reserve(n_sshot_dirs); @@ -435,6 +437,7 @@ void ResourceMonitor::heart_beat() // This should really be export to a file (preferably binary, but then bin->json command is needed, too). ss.dump(); + */ } m_fs_state.reset_stats(); @@ -544,45 +547,147 @@ void ResourceMonitor::update_vs_and_file_usage_info() m_fs_state.m_meta_used = vsi.Total - vsi.Free; } -void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) +long long ResourceMonitor::get_file_usage_bytes_to_remove(const DataFsPurgeshot &ps, long long write_estimate, int tl) { - static const char *trc_pfx = "perform_purge_check() "; + // short names from config values const Configuration &conf = Cache::Conf(); + long long f0 = conf.m_fileUsageBaseline; + long long f1 = conf.m_fileUsageNominal; + long long f2 = conf.m_fileUsageMax; + long long w1 = conf.m_diskUsageLWM; + long long w2 = conf.m_diskUsageHWM; - std::unique_ptr psp( new DataFsPurgeshot(m_fs_state) ); - DataFsPurgeshot &ps = *psp; + // get usage from purge snapshot + long long T = ps.m_disk_total; + long long x = ps.m_file_usage; + long long u = ps.m_disk_used; - // Purge precheck I. -- available disk space + // get file usage increase from the previous time interval check + long long delta = write_estimate; + TRACE_INT(tl, "File usage increased since revious purge interval " << delta ); + + long long bytes_to_remove = 0; + + // helper lambda function + auto clamp = [&x, &bytes_to_remove](long long lowval, long long highval) + { + long long val = x; + long long newval = val - bytes_to_remove; - if (ps.m_disk_used > conf.m_diskUsageHWM) { - ps.m_bytes_to_remove_d = ps.m_disk_used - conf.m_diskUsageLWM; - ps.m_space_based_purge = true; + // removed too much + if (newval < lowval) + { + return lowval - val; + } + + // removed too little + if (newval > highval) + { + return highval - val; + } + // keep the original value + return bytes_to_remove; + }; + + // under file quota, nothing to do + if (x < f0) + return 0; + + // total disk usage exceeds highWatermark + if (u >= w2) + { + TRACE_INT(tl, "Disk usage: " << ps.m_disk_used << " exceed highWatermark " << conf.m_diskUsageHWM); + float frac_u = static_cast(u - w2) / (T - w2); + float frac_x = static_cast(x - f0) / (f1 - f0); + + if (w2 == T) + { + bytes_to_remove = u -w1; + } + else + { + if (frac_x > frac_u) + { + // the cache is the reason for going out of w2 range + bytes_to_remove = (frac_x - frac_u) * (f1 - f0); + bytes_to_remove += delta; + bytes_to_remove = clamp(f0, f1); + } + else + { + // someone else is filling disk space, go to f1 + bytes_to_remove = clamp(f0, f2); + } + return bytes_to_remove; + } } - // Purge precheck II. -- usage by files. - // Keep it updated, but only act on it if so configured. + // file quota and total disk usage is within normal range, check if this space usage is + // proportinal to disk usage and correct it + if (u > w1 && x > f1) + { + float frac_u = static_cast(u - w1) / (w2 - w1); + float frac_x = static_cast(x - f1) / (f2 - f1); + if (frac_x > frac_u) + { + TRACE_INT(tl, "Disproportional file quota usage comapared to disc usage (frac_x/frac_u) = " << frac_x << "/"<< frac_u); + bytes_to_remove = (frac_x - frac_u) * (f2 - f1); + bytes_to_remove += delta; + } + + // check the new x val will not be below f0 + bytes_to_remove = clamp(f0, f2); + return bytes_to_remove; + } + + // final check: disk useage is lower that w1, check if exceed the max file usage f2 + if (x > f2) + { + // drop usage to f2 + // compare with global disk usage in the previous purge cycle (default 300s) + // check delta is not overflowing f2, else set numver of bytes to remove according remove to f0 + + TRACE_INT(tl, "File usage exceeds maxim file usage. Total disk usage is under lowWatermark. Clearing to low file usage."); + long long f2delta = std::max(f2 - delta, f0); + bytes_to_remove = clamp(f0, f2delta); + return bytes_to_remove; + } + + return bytes_to_remove; +} + +void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) +{ + static const char *trc_pfx = "perform_purge_check() "; + const Configuration &conf = Cache::Conf(); + + std::unique_ptr psp( new DataFsPurgeshot(m_fs_state) ); + DataFsPurgeshot &ps = *psp; ps.m_file_usage = 512ll * m_current_usage_in_st_blocks; // These are potentially wrong as cache might be writing over preallocated byte ranges. ps.m_estimated_writes_from_writeq = Cache::GetInstance().WritesSinceLastCall(); // Can have another estimate based on eiter writes or st-blocks from purge-stats, once we have them. + + TRACE_INT(tl, trc_pfx << "Purge check:"); + ps.m_bytes_to_remove = 0; if (conf.are_file_usage_limits_set()) { - if (ps.m_space_based_purge ) - { - ps.m_bytes_to_remove_f = std::max(ps.m_file_usage - conf.m_fileUsageBaseline, 0ll); - } - else if (ps.m_file_usage > conf.m_fileUsageMax) + ps.m_bytes_to_remove = get_file_usage_bytes_to_remove(ps, ps.m_estimated_writes_from_writeq, tl); + } + else + { + if (ps.m_disk_used > conf.m_diskUsageHWM) { - ps.m_bytes_to_remove_f = std::max(ps.m_file_usage - conf.m_fileUsageNominal, 0ll); - ps.m_space_based_purge = true; + TRACE_INT(tl, "Disk usage: " << ps.m_disk_used << " exceed highWatermark."); + ps.m_bytes_to_remove = ps.m_disk_used - conf.m_diskUsageLWM; } } - ps.m_bytes_to_remove = std::max(ps.m_bytes_to_remove_d, ps.m_bytes_to_remove_f); + ps.m_space_based_purge = ps.m_bytes_to_remove ? 1 : 0; - // Purge precheck III. -- check if age-based purge is required + // Purge precheck -- check if age-based purge is required // We ignore uvkeep time, it requires reading of cinfo files and it is enforced in File::Open() anyway. if (purge_cold_files && conf.is_age_based_purge_in_effect()) // || conf.is_uvkeep_purge_in_effect()) @@ -590,10 +695,7 @@ void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) ps.m_age_based_purge = true; } - TRACE_INT(tl, trc_pfx << "Purge check:"); - TRACE_INT(tl, "\tbytes_to_remove_disk = " << ps.m_bytes_to_remove_d << " B"); - TRACE_INT(tl, "\tbytes_to remove_files = " << ps.m_bytes_to_remove_f << " B"); - TRACE_INT(tl, "\tbytes_to_remove = " << ps.m_bytes_to_remove << " B"); + TRACE_INT(tl, "\tbytes_to_remove = " << ps.m_bytes_to_remove << " B"); TRACE_INT(tl, "\tspace_based_purge = " << ps.m_space_based_purge); TRACE_INT(tl, "\tage_based_purge = " << ps.m_age_based_purge); diff --git a/src/XrdPfc/XrdPfcResourceMonitor.hh b/src/XrdPfc/XrdPfcResourceMonitor.hh index ed5724b07bf..770c7ed7b15 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.hh +++ b/src/XrdPfc/XrdPfcResourceMonitor.hh @@ -133,6 +133,7 @@ class ResourceMonitor void process_inter_dir_scan_open_requests(FsTraversal &fst); void cross_check_or_process_oob_lfn(const std::string &lfn, FsTraversal &fst); + long long get_file_usage_bytes_to_remove(const DataFsPurgeshot &ps, long long previous_file_usage, int logLeve); public: ResourceMonitor(XrdOss& oss); From d7e9485f380fc11bbf7f99ff690ecf2ab425b83b Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Thu, 1 Aug 2024 14:06:54 -0700 Subject: [PATCH 40/43] Correct type in log message --- src/XrdPfc/XrdPfcResourceMonitor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 91a3e976032..3ef479cd678 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -564,7 +564,7 @@ long long ResourceMonitor::get_file_usage_bytes_to_remove(const DataFsPurgeshot // get file usage increase from the previous time interval check long long delta = write_estimate; - TRACE_INT(tl, "File usage increased since revious purge interval " << delta ); + TRACE_INT(tl, "file usage increased since the previous purge interval in bytes: " << delta ); long long bytes_to_remove = 0; From 06864a20a2d9b7161d4cf171b60d168f305197b7 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Thu, 1 Aug 2024 14:12:10 -0700 Subject: [PATCH 41/43] Separate plugin's and the default purge FPurgeState's map. Reduce the number of bytes to remove by the default purge according to purge plugin. --- src/XrdPfc/XrdPfcPurge.cc | 177 ++++++++++++++++++++++---------------- 1 file changed, 102 insertions(+), 75 deletions(-) diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index 19b9d662b5d..cec0d64ab50 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -21,47 +21,97 @@ namespace namespace XrdPfc { -void OldStylePurgeDriver(DataFsPurgeshot &ps) +long long UnlinkPurgeStateFilesInMap(FPurgeState& purgeState, long long bytes_to_remove, const std::string& root_path) { - static const char *trc_pfx = "OldStylePurgeDriver "; + static const char *trc_pfx = "UnlinkPurgeStateFilesInMap "; + + struct stat fstat; + int protected_cnt = 0; + int deleted_file_count = 0; + long long deleted_st_blocks = 0; + long long protected_st_blocks = 0; + long long st_blocks_to_remove = (bytes_to_remove >> 9) + 1ll; + const auto &cache = Cache::TheOne(); - const auto &conf = Cache::Conf(); auto &resmon = Cache::ResMon(); auto &oss = *cache.GetOss(); - TRACE(Info, trc_pfx << "Started."); - time_t purge_start = time(0); + TRACE(Info, trc_pfx << "Started, root_path = " << root_path << ", bytes_to_remove = " << bytes_to_remove); + + // Loop over map and remove files with oldest values of access time. + for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) + { + // Finish when enough space has been freed but not while age-based purging is in progress. + // Those files are marked with time-stamp = 0. + if (st_blocks_to_remove <= 0 && it->first != 0) + { + break; + } + + std::string &infoPath = it->second.path; + std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen); + + if (cache.IsFileActiveOrPurgeProtected(dataPath)) + { + ++protected_cnt; + protected_st_blocks += it->second.nStBlocks; + TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks); + continue; + } + // remove info file + if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK) + { + oss.Unlink(infoPath.c_str()); + TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size); + } + else + { + TRACE(Error, trc_pfx << "Can't locate file " << dataPath); + } - FPurgeState purgeState(2 * ps.m_bytes_to_remove, oss); // prepare twice more volume than required + // remove data file + if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK) + { + st_blocks_to_remove -= it->second.nStBlocks; + deleted_st_blocks += it->second.nStBlocks; + ++deleted_file_count; - // Make a map of file paths, sorted by access time. + oss.Unlink(dataPath.c_str()); + TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first); - if (ps.m_age_based_purge) - { - purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge); + resmon.register_file_purge(dataPath, it->second.nStBlocks); + } } - if (conf.is_uvkeep_purge_in_effect()) + if (protected_cnt > 0) { - purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep); + TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks); } - bool scan_ok = purgeState.TraverseNamespace("/"); - if ( ! scan_ok) { - TRACE(Error, trc_pfx << "namespace traversal failed at top-directory, this should not happen."); - return; - } + TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks) + + return deleted_st_blocks; +} - TRACE(Debug, trc_pfx << "usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes."); +// ------------------------------------------------------------------------------------- - purgeState.MoveListEntriesToMap(); +void OldStylePurgeDriver(DataFsPurgeshot &ps) +{ + static const char *trc_pfx = "OldStylePurgeDriver "; + const auto &cache = Cache::TheOne(); + const auto &conf = Cache::Conf(); + auto &oss = *cache.GetOss(); + time_t purge_start = time(0); + + ///////////////////////////////////////////////////////////// + /// PurgePin ///////////////////////////////////////////////////////////// - /// PurgePin begin PurgePin *purge_pin = cache.GetPurgePin(); + long long std_blocks_removed_by_pin = 0; if (purge_pin) - { + { // set dir stat for each path and calculate nBytes to recover for each path // return total bytes to recover within the plugin long long clearVal = purge_pin->GetBytesToRecover(ps); @@ -79,78 +129,55 @@ void OldStylePurgeDriver(DataFsPurgeshot &ps) if ( ! scan_ok) { TRACE(Warning, trc_pfx << "purge-pin scan of directory failed for " << ppit->path); continue; - } - - // fill central map from the plugin entry - for (FPurgeState::map_i it = fps.refMap().begin(); it != fps.refMap().end(); ++it) - { - it->second.path = ppit->path + it->second.path; - TRACE(Debug, trc_pfx << "PurgePin found file " << it->second.path.c_str()<< " size " << 512ll * it->second.nStBlocks); - purgeState.refMap().insert(std::make_pair(0, it->second)); // set atime to zero to make sure this is deleted - } + } + + fps.MoveListEntriesToMap(); + std_blocks_removed_by_pin += UnlinkPurgeStateFilesInMap(fps, ppit->nBytesToRecover, ppit->path); } } } - /// PurgePin end - ///////////////////////////////////////////////////////////// - int deleted_file_count = 0; - long long deleted_st_blocks = 0; + ///////////////////////////////////////////////////////////// + /// Default purge + ///////////////////////////////////////////////////////////// - // Loop over map and remove files with oldest values of access time. - struct stat fstat; - int protected_cnt = 0; - long long protected_st_blocks = 0; - long long st_blocks_to_remove = (ps.m_bytes_to_remove >> 9) + 1ll; - for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it) + // check if the default pargue is still needed after purge pin + long long pin_removed_bytes = std_blocks_removed_by_pin * 512ll; + long long default_purge_blocks_removed = 0; + if (ps.m_bytes_to_remove > pin_removed_bytes) { - // Finish when enough space has been freed but not while age-based purging is in progress. - // Those files are marked with time-stamp = 0. - if (st_blocks_to_remove <= 0 && it->first != 0) - { - break; - } - - std::string &infoPath = it->second.path; - std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen); + // init default purge + long long bytes_to_remove = ps.m_bytes_to_remove - pin_removed_bytes; + FPurgeState purgeState(2 * bytes_to_remove, oss); // prepare twice more volume than required - if (cache.IsFileActiveOrPurgeProtected(dataPath)) + if (ps.m_age_based_purge) { - ++protected_cnt; - protected_st_blocks += it->second.nStBlocks; - TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks); - continue; + purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge); } - - // remove info file - if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK) + if (conf.is_uvkeep_purge_in_effect()) { - oss.Unlink(infoPath.c_str()); - TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size); + purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep); } - // remove data file - if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK) + // Make a map of file paths, sorted by access time. + bool scan_ok = purgeState.TraverseNamespace("/"); + if (!scan_ok) { - st_blocks_to_remove -= it->second.nStBlocks; - deleted_st_blocks += it->second.nStBlocks; - ++deleted_file_count; + TRACE(Error, trc_pfx << "default purge namespace traversal failed at top-directory, this should not happen."); + return; + } - oss.Unlink(dataPath.c_str()); - TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first); + TRACE(Debug, trc_pfx << "default purge usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes."); - resmon.register_file_purge(dataPath, it->second.nStBlocks); - } - } - if (protected_cnt > 0) - { - TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks); + purgeState.MoveListEntriesToMap(); + default_purge_blocks_removed = UnlinkPurgeStateFilesInMap(purgeState, bytes_to_remove, "/"); } + // print the total summary + ///////////////////////////////////////////////// int purge_duration = time(0) - purge_start; - - TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks - << ", purge duration " << purge_duration); + long long total_bytes_removed = (default_purge_blocks_removed + std_blocks_removed_by_pin) * 512ll; + TRACE(Info, trc_pfx << "Finished, removed total size " << total_bytes_removed << ", purge duration " << purge_duration); } } // end namespace XrdPfc From 15d8e6d9e0a42b3927470d22e075bd8bccbb3772 Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Thu, 22 Aug 2024 10:52:49 -0700 Subject: [PATCH 42/43] Schedule purge task periodically if purge plugin is instantiated and has a periodic call flag set. Pruge plugin has the period call set by the default. It can be overridden by the virtual XrdPfc::PurgePin::CallPeriodically() interface. --- src/XrdPfc/XrdPfcPurgePin.hh | 9 +++++++++ src/XrdPfc/XrdPfcResourceMonitor.cc | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/XrdPfc/XrdPfcPurgePin.hh b/src/XrdPfc/XrdPfcPurgePin.hh index 9bcb9bb30e0..63e440b5197 100644 --- a/src/XrdPfc/XrdPfcPurgePin.hh +++ b/src/XrdPfc/XrdPfcPurgePin.hh @@ -34,6 +34,15 @@ protected: public: virtual ~PurgePin() {} + + //--------------------------------------------------------------------- + //! + //! + //! @return total number of bytes + //--------------------------------------------------------------------- + virtual bool CallPeriodically() { return true; }; + + //--------------------------------------------------------------------- //! Provide erase information from directory statistics //! diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index 3ef479cd678..d37c56fe6e4 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -5,6 +5,7 @@ #include "XrdPfcDirState.hh" #include "XrdPfcDirStateSnapshot.hh" #include "XrdPfcTrace.hh" +#include "XrdPfcPurgePin.hh" #include "XrdOss/XrdOss.hh" @@ -699,7 +700,10 @@ void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) TRACE_INT(tl, "\tspace_based_purge = " << ps.m_space_based_purge); TRACE_INT(tl, "\tage_based_purge = " << ps.m_age_based_purge); - if ( ! ps.m_space_based_purge && ! ps.m_age_based_purge) { + bool periodic = Cache::GetInstance().GetPurgePin() ? + Cache::GetInstance().GetPurgePin()->CallPeriodically() : false; + + if ( ! ps.m_space_based_purge && ! ps.m_age_based_purge && !periodic ) { TRACE(Info, trc_pfx << "purge not required."); Cache::GetInstance().ClearPurgeProtectedSet(); return; @@ -709,7 +713,7 @@ void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) return; } - TRACE(Info, trc_pfx << "purge required ... scheduling purge task."); + TRACE(Info, trc_pfx << "scheduling purge task."); // At this point we have all the information: report, decide on action. // There is still some missing infrastructure, especially as regards to purge-plugin: From 8438f57874f3cfb685468ed8f2250a4dcad965db Mon Sep 17 00:00:00 2001 From: Alja Mrak Tadel Date: Thu, 22 Aug 2024 11:14:04 -0700 Subject: [PATCH 43/43] Undefine RM_DEBUG to disable resource monitor debug prints --- src/XrdPfc/XrdPfcResourceMonitor.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/XrdPfc/XrdPfcResourceMonitor.cc b/src/XrdPfc/XrdPfcResourceMonitor.cc index d37c56fe6e4..69e0b223adf 100644 --- a/src/XrdPfc/XrdPfcResourceMonitor.cc +++ b/src/XrdPfc/XrdPfcResourceMonitor.cc @@ -11,7 +11,7 @@ #include -#define RM_DEBUG +// #define RM_DEBUG #ifdef RM_DEBUG #define dprintf(...) printf(__VA_ARGS__) #else @@ -351,7 +351,6 @@ void ResourceMonitor::heart_beat() static const char *tpfx = "heart_beat() "; const Configuration &conf = Cache::Conf(); - const DirState &root_ds = *m_fs_state.get_root(); const int s_queue_proc_interval = 10; // const s_stats_up_prop_interval = 60; -- for when we have dedicated purge / stat report structs @@ -422,10 +421,11 @@ void ResourceMonitor::heart_beat() if (conf.is_dir_stat_reporting_on()) { const int store_depth = conf.m_dirStatsStoreDepth; - const int n_sshot_dirs = root_ds.count_dirs_to_level(store_depth); - dprintf("Snapshot n_dirs=%d, total n_dirs=%d\n", n_sshot_dirs, + #ifdef RM_DEBUG + const DirState &root_ds = *m_fs_state.get_root(); + dprintf("Snapshot n_dirs=%d, total n_dirs=%d\n", root_ds.count_dirs_to_level(store_depth), root_ds.m_here_usage.m_NDirectories + root_ds.m_recursive_subdir_usage.m_NDirectories + 1); - + #endif m_fs_state.dump_recursively(store_depth); /* @@ -726,10 +726,11 @@ void ResourceMonitor::perform_purge_check(bool purge_cold_files, int tl) // deletion -- eg, by comparing stat time of cinfo + doing the is-active / is-purge-protected. const DirState &root_ds = *m_fs_state.get_root(); - const int n_pshot_dirs = root_ds.count_dirs_to_level(9999); const int n_calc_dirs = 1 + root_ds.m_here_usage.m_NDirectories + root_ds.m_recursive_subdir_usage.m_NDirectories; +#ifdef RM_DEBUG + const int n_pshot_dirs = root_ds.count_dirs_to_level(9999); dprintf("purge dir count recursive=%d vs from_usage=%d\n", n_pshot_dirs, n_calc_dirs); - +#endif ps.m_dir_vec.reserve(n_calc_dirs); ps.m_dir_vec.emplace_back( DirPurgeElement(root_ds, -1) ); fill_pshot_vec_children(root_ds, 0, ps.m_dir_vec, 9999);