From 640f64d504d155687e72508630e2141876d1061f Mon Sep 17 00:00:00 2001 From: guj Date: Tue, 3 Sep 2024 10:26:10 -0700 Subject: [PATCH 01/13] added new test 8c for particles todo: BTD support --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c17631c6cb..114627f38f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -767,6 +767,7 @@ set(openPMD_EXAMPLE_NAMES 8_benchmark_parallel 8a_benchmark_write_parallel 8b_benchmark_read_parallel + 8c_benchmark_ptl_parallel 10_streaming_write 10_streaming_read 12_span_write From fc47ffdf6f8d9df6197b29c49c2faf989ceca377 Mon Sep 17 00:00:00 2001 From: guj Date: Tue, 3 Sep 2024 11:50:24 -0700 Subject: [PATCH 02/13] added test 8c --- examples/8c_benchmark_ptl_parallel.cpp | 799 +++++++++++++++++++++++++ 1 file changed, 799 insertions(+) create mode 100644 examples/8c_benchmark_ptl_parallel.cpp diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp new file mode 100644 index 0000000000..b3de4b2968 --- /dev/null +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -0,0 +1,799 @@ +/* Copyright 2020-2021 Junmin Gu, Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#if openPMD_HAVE_ADIOS2 +#include +#endif + +#if openPMD_HAVE_CUDA_EXAMPLES +#include +#include +#endif + +using std::cout; +using namespace openPMD; + +using MaxResSteadyClock = std::conditional_t; + +/** The Memory profiler class for profiling purpose + * + * Simple Memory usage report that works on linux system + */ + +static std::chrono::time_point< MaxResSteadyClock > m_ProgStart = MaxResSteadyClock::now(); + +class MemoryProfiler +{ +public: + /** Simple Memory profiler for linux + * + * @param[in] rank MPI rank + * @param[in] tag item name to measure + */ + MemoryProfiler(int rank, const std::string &tag) + { + m_Rank = rank; +#if defined(__linux) + // m_Name = "/proc/meminfo"; + m_Name = "/proc/self/status"; + Display(tag); +#else + (void)tag; + m_Name = ""; +#endif + } + + /** + * + * Read from /proc/self/status and display the Virtual Memory info at rank 0 + * on console + * + * @param tag item name to measure + * @param rank MPI rank + */ + + void Display(const std::string &tag) + { + if (0 == m_Name.size()) + return; + + if (m_Rank > 0) + return; + + std::cout << " memory at: " << tag; + std::ifstream input(m_Name.c_str()); + + if (input.is_open()) + { + for (std::string line; getline(input, line);) + { + if (line.find("VmRSS") == 0) + std::cout << line << " "; + if (line.find("VmSize") == 0) + std::cout << line << " "; + if (line.find("VmSwap") == 0) + std::cout << line; + } + std::cout << std::endl; + input.close(); + } + } + +private: + int m_Rank; + std::string m_Name; +}; + + +/** The Timer class for profiling purpose + * + * Simple Timer that measures time consumption btw constucture and destructor + * Reports at rank 0 at the console, for immediate convenience + */ +class Timer +{ +public: + enum VERBOSE_LEVEL {NONE, MIN, FULL}; + /** + * + * Simple Timer + * + * @param tag item name to measure + * @param rank MPI rank + */ + Timer(const std::string &tag, int rank, VERBOSE_LEVEL vl = FULL) + + //Timer(const std::string &tag, int rank, bool silent = false) + { + m_Tag = tag; + m_Rank = rank; + m_Start = MaxResSteadyClock::now(); + + /* + m_Silent = silent; + if (!m_Silent) + MemoryProfiler(rank, tag); + */ + m_Silent = vl; + if (m_Silent == FULL) + MemoryProfiler(rank, tag); + } + + double getDuration() + { + auto curr = MaxResSteadyClock::now(); + + double secs = std::chrono::duration_cast > (curr - m_Start).count(); + return secs; + } + + ~Timer() + { + if (m_Silent == NONE) + return; + + if (m_Silent == FULL) + { + std::string tt = "~" + m_Tag; + MemoryProfiler mp(m_Rank, tt.c_str()); + } + + double secs = getDuration(); + + if (m_Rank > 0) + return; + + std::cout << " [" << m_Tag << "] took:" << secs << " seconds"; + std::cout << " Time Elapsed:" + << secs + std::chrono::duration_cast< std::chrono::duration > (m_Start - m_ProgStart).count() + << std::endl; + + std::cout << std::endl; + } + +private: + std::chrono::time_point< MaxResSteadyClock > m_Start; + + std::string m_Tag; + int m_Rank = 0; + //bool m_Silent = false; + VERBOSE_LEVEL m_Silent = Timer::NONE; +}; + +class LocalProfiler +{ +public: + LocalProfiler() = default; + ~LocalProfiler () = default; + + void setRank(int r) {m_Rank = r;} + void update(Timer& timer) { m_Counter ++; m_Total += timer.getDuration(); } + + int m_Rank = 0; // info only + int m_Counter=0; + double m_Total = 0; +}; + +static std::map m_GlobalProfilers; + +class Checkpoint +{ +public: + Checkpoint(std::string name, int rank) + :m_name(name) + { + auto fp = m_GlobalProfilers.find(name); + if ( fp == m_GlobalProfilers.end()) { + LocalProfiler p; + p.setRank(rank); + m_GlobalProfilers[name] = p; + } + m_Timer = new Timer(name, rank, Timer::NONE); + } + ~Checkpoint() + { + m_GlobalProfilers[m_name].update(*m_Timer); + + if (m_Timer != NULL) + delete m_Timer; + } + +private: + Timer* m_Timer = NULL; + std::string m_name; +}; + + + +/** createDataCPU + * generate a shared ptr of given size with given type & default value on + * CPU + * + * @param T data type + * @param size data size + * @param val data value by default + * @param increment data increment by linear with index + * + */ + +template +std::shared_ptr +createDataCPU(const unsigned long &size, const T &val, const T &increment) +{ + auto E = std::shared_ptr{new T[size], [](T *d) { delete[] d; }}; + + for (unsigned long i = 0ul; i < size; i++) + { + if (increment != 0) + E.get()[i] = val + i * increment; + else + E.get()[i] = val; + } + return E; +} + +#if openPMD_HAVE_CUDA_EXAMPLES +template +std::shared_ptr +createDataGPU(const unsigned long &size, const T &val, const T &increment) +{ + auto myCudaMalloc = [](size_t mySize) { + void *ptr; + cudaMalloc((void **)&ptr, mySize); + return ptr; + }; + auto deleter = [](T *ptr) { cudaFree(ptr); }; + auto E = std::shared_ptr{(T *)myCudaMalloc(size * sizeof(T)), deleter}; + + T *data = new T[size]; + for (unsigned long i = 0ul; i < size; i++) + { + if (increment != 0) + data[i] = val + i * increment; + else + data[i] = val; + } + cudaMemcpy(E.get(), data, size * sizeof(T), cudaMemcpyHostToDevice); + return E; +} +#endif + +template +std::shared_ptr +createData(const unsigned long &size, const T &val, const T &increment) +{ + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + Checkpoint c(" CreateData", rank); + +#if openPMD_HAVE_CUDA_EXAMPLES + return createDataGPU(size, val, increment); +#else + return createDataCPU(size, val, increment); +#endif +} + +/** Find supported backends + * (looking for ADIOS2 or H5) + * + */ +std::vector getBackends(bool bpOnly) +{ + std::vector res; +#if openPMD_HAVE_ADIOS2 + res.emplace_back(".bp"); +#endif + +#if openPMD_HAVE_HDF5 + if (!bpOnly) + res.emplace_back(".h5"); +#endif + return res; +} + +// Forward declaration +class TestInput; + +/* Class BasicParticlePattern + * defines grid layout from user inputs + * subclasses detail the layout of mesh/particle at each rank + */ +class BasicParticlePattern +{ +public: + BasicParticlePattern(const TestInput &input); + + void getParticleLayout(unsigned long& offset, unsigned long &count, unsigned long &total); + + void run(); + void store(Series &series, int step); + void storeParticles(ParticleSpecies &currSpecies, int &step); + + unsigned long countMe(const Extent &count); + unsigned long indexMe(const Offset &count); + + const std::string getBaseFileName() const; + const TestInput &m_Input; + + void printMe(); + openPMD::Extent ProperExtent (unsigned long long n, bool init) const; +}; // class BasicParticlePattern + + +/** Class TestInput + * + */ +class TestInput +{ +public: + TestInput() = default; + + int m_MPISize = 1; //!< MPI size + int m_MPIRank = 0; //!< MPI rank + + + // default distribution is between 1 - 2 million ptls per rank + unsigned long m_PtlMin = 1000000; + unsigned long m_PtlMax = 2000000; + + int m_Steps = 1; //!< num of iterations + std::string m_Backend = ""; //!< I/O backend by file ending + + bool m_UseJoinedDim=false; + openPMD::IterationEncoding m_Encoding = + openPMD::IterationEncoding::variableBased; + + //! prefix for the output directory + std::string m_Prefix = "../samples"; +}; // class TestInput + +void parse(TestInput &input, std::string line) +{ + // no valid input a=b + if (line.size() <= 3) + return; + if (line[0] == '#') + return; + + std::istringstream iline(line); + + std::string s; + std::vector vec; + while (std::getline(iline, s, '=')) + vec.push_back(s); + + if (vec.size() != 2) + return; + + if (vec.at(0).compare("encoding") == 0) + { + if (vec.at(1).compare("f") == 0) + input.m_Encoding = openPMD::IterationEncoding::fileBased; + else if (vec.at(1).compare("g") == 0) + input.m_Encoding = openPMD::IterationEncoding::groupBased; +#if openPMD_HAVE_ADIOS2 + // BP5 must be matched with a stream engine. + if (auxiliary::getEnvString("OPENPMD_ADIOS2_ENGINE", "BP4") == "BP5") + input.m_Encoding = openPMD::IterationEncoding::variableBased; +#endif + + return; + } + // Apply a specific backend instead of trying all available ones + if (vec.at(0).compare("backend") == 0) + { + if (vec[1][0] != '.') + input.m_Backend += '.'; + + input.m_Backend += vec[1]; + } + + if (vec[0].compare("joinedArray") == 0) + { + if (vec[1].size() > 0) { + if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) + input.m_UseJoinedDim = true; + } + return; + } + + if (vec[0].compare("maxMil") == 0) + { + input.m_PtlMax = ( (unsigned long) atoi(vec[1].c_str()) ) * (unsigned long) 1000000; + return; + } + + if (vec[0].compare("minMil") == 0) + { + input.m_PtlMin = ( (unsigned long) atoi(vec[1].c_str()) ) * (unsigned long) 1000000; + if (input.m_PtlMin > input.m_PtlMax) + input.m_PtlMin = input.m_PtlMax; + return; + } + + if (vec[0].compare("steps") == 0) + { + input.m_Steps = atoi(vec[1].c_str()); + return; + } + + if (vec[0].compare("fileLocation") == 0) + { + input.m_Prefix = vec[1]; + return; + } +} + +int parseArgs(int argc, char *argv[], TestInput &input) +{ + if (argc == 2) + { + std::fstream infile; + infile.open(argv[1], std::ios::in); + if (!infile.is_open()) + { + if (input.m_MPIRank == 0) + std::cout << "No such file: " << argv[1] << std::endl; + return -1; + } + + std::string tp; + while (getline(infile, tp)) + { + parse(input, tp); + } + + infile.close(); + return 1; + } + std::cout<<" Expecting: "< "< result(input.m_MPISize, 0); + //unsigned long buffer[m_Input.m_MPISize]; + MPI_Allgather (&p.m_Total, 1, MPI_DOUBLE, result.data(), 1, MPI_DOUBLE, MPI_COMM_WORLD); + + auto [min, max] = std::minmax_element(result.begin(),result.end()); + + if ( 0 == input.m_MPIRank ) + std::cout << name << "\t\t "<(), ProperExtent(np, true)); + auto const realDataSet = + openPMD::Dataset(openPMD::determineDatatype(), ProperExtent(np, true)); + currSpecies["id"].resetDataset(intDataSet); + currSpecies["charge"].resetDataset(realDataSet); + + currSpecies["position"]["x"].resetDataset(realDataSet); + + //currSpecies["positionOffset"]["x"].resetDataset(realDataSet); + //currSpecies["positionOffset"]["x"].makeConstant(0.); + + + { + Checkpoint remove2(" SP_Barrier_2", m_Input.m_MPIRank); + MPI_Barrier(MPI_COMM_WORLD); + } + + + Checkpoint remove3(" SP_cs", m_Input.m_MPIRank); + if (count > 0) + { + auto ids = createData(count, offset, 1); + currSpecies["id"].storeChunk(ids, ProperExtent(offset, false), {count}); + + auto charges = createData(count, 0.1 * step, 0.0001); + currSpecies["charge"].storeChunk(charges, ProperExtent(offset, false), {count}); + + auto mx = createData(count, 1.0 * step, 0.0002); + currSpecies["position"]["x"].storeChunk(mx, ProperExtent(offset, false), {count}); + } +} // storeParticles + +/* + * Return total number of particles + * set to be a multiple of mesh size + * + */ +void BasicParticlePattern::getParticleLayout(unsigned long& offset, unsigned long &count, unsigned long &total) +{ + { + Checkpoint x1(" ComputeLayout", m_Input.m_MPIRank); + if (m_Input.m_PtlMin >= m_Input.m_PtlMax) + { + count = m_Input.m_PtlMax; + } + else + { + std::random_device rd; // a seed source for the random number engine + std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() + std::uniform_int_distribution<> distrib(m_Input.m_PtlMin, m_Input.m_PtlMax); + + //for (int n = 0; n != 10; ++n) + // std::cout << distrib(gen) << ' '; + count = distrib(gen); + } + + // gather from all ranks to get offset/total + + if (m_Input.m_UseJoinedDim) + return; + } + //Timer g("Gather Particle logistics ", m_Input.m_MPIRank); + Checkpoint x(" GetPTLOffset", m_Input.m_MPIRank); + + std::vector result(m_Input.m_MPISize, 0); + //unsigned long buffer[m_Input.m_MPISize]; + MPI_Allgather (&count, 1, MPI_UNSIGNED_LONG, result.data(), 1, MPI_UNSIGNED_LONG, MPI_COMM_WORLD); + + total = 0; + auto const num_results = static_cast(result.size()); + for (int i=0; i This is a Particle Only test, With Joined Dimension applied to ADIOS " < This is a Particle Only test. " < Date: Tue, 3 Sep 2024 16:23:25 -0700 Subject: [PATCH 03/13] added option to force PDW flush --- examples/8c_benchmark_ptl_parallel.cpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index b3de4b2968..d66d0bd830 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -373,6 +373,7 @@ class TestInput std::string m_Backend = ""; //!< I/O backend by file ending bool m_UseJoinedDim=false; + bool m_CallPDW=false; openPMD::IterationEncoding m_Encoding = openPMD::IterationEncoding::variableBased; @@ -430,6 +431,15 @@ void parse(TestInput &input, std::string line) return; } + if (vec[0].compare("usePDW") == 0) + { + if (vec[1].size() > 0) { + if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) + input.m_CallPDW = true; + } + return; + } + if (vec[0].compare("maxMil") == 0) { input.m_PtlMax = ( (unsigned long) atoi(vec[1].c_str()) ) * (unsigned long) 1000000; @@ -648,6 +658,9 @@ void BasicParticlePattern::store(Series &series, int step) storeParticles(currSpecies, step); + if (m_Input.m_CallPDW) + series.flush(); + { Checkpoint remove2("Barrier_3", m_Input.m_MPIRank); MPI_Barrier(MPI_COMM_WORLD); @@ -778,7 +791,6 @@ openPMD::Extent BasicParticlePattern::ProperExtent (unsigned long long n, bool i else return {}; } - /* * Print pattern layout */ @@ -787,10 +799,14 @@ void BasicParticlePattern::printMe() if ( 0 < m_Input.m_MPIRank ) return; + std::string pdw_status=" just EndStep"; + if (m_Input.m_CallPDW) + pdw_status=" PDW + EndStep"; + if (m_Input.m_UseJoinedDim) - std::cout << " ====> This is a Particle Only test, With Joined Dimension applied to ADIOS " < This is a Particle Only test, With Joined Dimension applied to ADIOS."< This is a Particle Only test. " < This is a Particle Only test. " < Date: Tue, 3 Sep 2024 16:31:49 -0700 Subject: [PATCH 04/13] warning fix --- examples/8c_benchmark_ptl_parallel.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index d66d0bd830..5ce6dad730 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -227,12 +227,12 @@ class Checkpoint { m_GlobalProfilers[m_name].update(*m_Timer); - if (m_Timer != NULL) + if (m_Timer != nullptr) delete m_Timer; } private: - Timer* m_Timer = NULL; + Timer* m_Timer = nullptr; std::string m_name; }; @@ -345,11 +345,11 @@ class BasicParticlePattern unsigned long countMe(const Extent &count); unsigned long indexMe(const Offset &count); - const std::string getBaseFileName() const; + [[nodiscard]] const std::string getBaseFileName() const; const TestInput &m_Input; void printMe(); - openPMD::Extent ProperExtent (unsigned long long n, bool init) const; + [[nodiscard]] openPMD::Extent ProperExtent (unsigned long long n, bool init) const; }; // class BasicParticlePattern From b29dd099090d49354cf45f379495a0e1b9e62c43 Mon Sep 17 00:00:00 2001 From: guj Date: Tue, 3 Sep 2024 16:51:11 -0700 Subject: [PATCH 05/13] tabs --- examples/8c_benchmark_ptl_parallel.cpp | 118 ++++++++++++------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index 5ce6dad730..aab4e6ce45 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -140,16 +140,16 @@ class Timer { m_Tag = tag; m_Rank = rank; - m_Start = MaxResSteadyClock::now(); + m_Start = MaxResSteadyClock::now(); - /* - m_Silent = silent; - if (!m_Silent) + /* + m_Silent = silent; + if (!m_Silent) MemoryProfiler(rank, tag); - */ - m_Silent = vl; - if (m_Silent == FULL) - MemoryProfiler(rank, tag); + */ + m_Silent = vl; + if (m_Silent == FULL) + MemoryProfiler(rank, tag); } double getDuration() @@ -157,7 +157,7 @@ class Timer auto curr = MaxResSteadyClock::now(); double secs = std::chrono::duration_cast > (curr - m_Start).count(); - return secs; + return secs; } ~Timer() @@ -171,14 +171,14 @@ class Timer MemoryProfiler mp(m_Rank, tt.c_str()); } - double secs = getDuration(); - + double secs = getDuration(); + if (m_Rank > 0) return; - std::cout << " [" << m_Tag << "] took:" << secs << " seconds"; + std::cout << " [" << m_Tag << "] took:" << secs << " seconds"; std::cout << " Time Elapsed:" - << secs + std::chrono::duration_cast< std::chrono::duration > (m_Start - m_ProgStart).count() + << secs + std::chrono::duration_cast< std::chrono::duration > (m_Start - m_ProgStart).count() << std::endl; std::cout << std::endl; @@ -416,18 +416,18 @@ void parse(TestInput &input, std::string line) // Apply a specific backend instead of trying all available ones if (vec.at(0).compare("backend") == 0) { - if (vec[1][0] != '.') - input.m_Backend += '.'; - - input.m_Backend += vec[1]; + if (vec[1][0] != '.') + input.m_Backend += '.'; + + input.m_Backend += vec[1]; } if (vec[0].compare("joinedArray") == 0) - { + { if (vec[1].size() > 0) { - if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) + if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) input.m_UseJoinedDim = true; - } + } return; } @@ -449,11 +449,11 @@ void parse(TestInput &input, std::string line) if (vec[0].compare("minMil") == 0) { input.m_PtlMin = ( (unsigned long) atoi(vec[1].c_str()) ) * (unsigned long) 1000000; - if (input.m_PtlMin > input.m_PtlMax) - input.m_PtlMin = input.m_PtlMax; + if (input.m_PtlMin > input.m_PtlMax) + input.m_PtlMin = input.m_PtlMax; return; } - + if (vec[0].compare("steps") == 0) { input.m_Steps = atoi(vec[1].c_str()); @@ -487,7 +487,7 @@ int parseArgs(int argc, char *argv[], TestInput &input) } infile.close(); - return 1; + return 1; } std::cout<<" Expecting: "< "< result(input.m_MPISize, 0); - //unsigned long buffer[m_Input.m_MPISize]; - MPI_Allgather (&p.m_Total, 1, MPI_DOUBLE, result.data(), 1, MPI_DOUBLE, MPI_COMM_WORLD); - - auto [min, max] = std::minmax_element(result.begin(),result.end()); - - if ( 0 == input.m_MPIRank ) - std::cout << name << "\t\t "< result(input.m_MPISize, 0); + //unsigned long buffer[m_Input.m_MPISize]; + MPI_Allgather (&p.m_Total, 1, MPI_DOUBLE, result.data(), 1, MPI_DOUBLE, MPI_COMM_WORLD); + + auto [min, max] = std::minmax_element(result.begin(),result.end()); + + if ( 0 == input.m_MPIRank ) + std::cout << name << "\t\t "< 0) { - auto ids = createData(count, offset, 1); - currSpecies["id"].storeChunk(ids, ProperExtent(offset, false), {count}); - - auto charges = createData(count, 0.1 * step, 0.0001); - currSpecies["charge"].storeChunk(charges, ProperExtent(offset, false), {count}); - - auto mx = createData(count, 1.0 * step, 0.0002); - currSpecies["position"]["x"].storeChunk(mx, ProperExtent(offset, false), {count}); + auto ids = createData(count, offset, 1); + currSpecies["id"].storeChunk(ids, ProperExtent(offset, false), {count}); + + auto charges = createData(count, 0.1 * step, 0.0001); + currSpecies["charge"].storeChunk(charges, ProperExtent(offset, false), {count}); + + auto mx = createData(count, 1.0 * step, 0.0002); + currSpecies["position"]["x"].storeChunk(mx, ProperExtent(offset, false), {count}); } } // storeParticles @@ -760,14 +760,14 @@ void BasicParticlePattern::getParticleLayout(unsigned long& offset, unsigned lon std::vector result(m_Input.m_MPISize, 0); //unsigned long buffer[m_Input.m_MPISize]; MPI_Allgather (&count, 1, MPI_UNSIGNED_LONG, result.data(), 1, MPI_UNSIGNED_LONG, MPI_COMM_WORLD); - + total = 0; auto const num_results = static_cast(result.size()); for (int i=0; i This is a Particle Only test, With Joined Dimension applied to ADIOS."< This is a Particle Only test. " < Date: Tue, 3 Sep 2024 16:55:35 -0700 Subject: [PATCH 06/13] eol --- examples/8c_benchmark_ptl_parallel.cpp | 56 +++++++++++++------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index aab4e6ce45..005d418d76 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -159,7 +159,7 @@ class Timer double secs = std::chrono::duration_cast > (curr - m_Start).count(); return secs; } - + ~Timer() { if (m_Silent == NONE) @@ -199,10 +199,10 @@ class LocalProfiler LocalProfiler() = default; ~LocalProfiler () = default; - void setRank(int r) {m_Rank = r;} + void setRank(int r) {m_Rank = r;} void update(Timer& timer) { m_Counter ++; m_Total += timer.getDuration(); } - - int m_Rank = 0; // info only + + int m_Rank = 0; // info only int m_Counter=0; double m_Total = 0; }; @@ -226,7 +226,7 @@ class Checkpoint ~Checkpoint() { m_GlobalProfilers[m_name].update(*m_Timer); - + if (m_Timer != nullptr) delete m_Timer; } @@ -298,7 +298,7 @@ createData(const unsigned long &size, const T &val, const T &increment) int rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); Checkpoint c(" CreateData", rank); - + #if openPMD_HAVE_CUDA_EXAMPLES return createDataGPU(size, val, increment); #else @@ -312,7 +312,7 @@ createData(const unsigned long &size, const T &val, const T &increment) */ std::vector getBackends(bool bpOnly) { - std::vector res; + std::vector res; #if openPMD_HAVE_ADIOS2 res.emplace_back(".bp"); #endif @@ -335,9 +335,9 @@ class BasicParticlePattern { public: BasicParticlePattern(const TestInput &input); - + void getParticleLayout(unsigned long& offset, unsigned long &count, unsigned long &total); - + void run(); void store(Series &series, int step); void storeParticles(ParticleSpecies &currSpecies, int &step); @@ -367,8 +367,8 @@ class TestInput // default distribution is between 1 - 2 million ptls per rank unsigned long m_PtlMin = 1000000; - unsigned long m_PtlMax = 2000000; - + unsigned long m_PtlMax = 2000000; + int m_Steps = 1; //!< num of iterations std::string m_Backend = ""; //!< I/O backend by file ending @@ -543,14 +543,14 @@ int main(int argc, char *argv[]) MPI_Comm_rank(MPI_COMM_WORLD, &input.m_MPIRank); int succ = parseArgs(argc, argv, input); - if (succ <= 0) + if (succ <= 0) { return -1; } - - doWork(input); - + + doWork(input); + { MPI_Barrier(MPI_COMM_WORLD); if ( 0 == input.m_MPIRank ) { @@ -570,9 +570,9 @@ int main(int argc, char *argv[]) std::cout << name << "\t\t "<(), ProperExtent(np, true)); auto const realDataSet = @@ -711,7 +711,7 @@ void BasicParticlePattern::storeParticles(ParticleSpecies &currSpecies, int &ste } - Checkpoint remove3(" SP_cs", m_Input.m_MPIRank); + Checkpoint remove3(" SP_cs", m_Input.m_MPIRank); if (count > 0) { auto ids = createData(count, offset, 1); @@ -722,7 +722,7 @@ void BasicParticlePattern::storeParticles(ParticleSpecies &currSpecies, int &ste auto mx = createData(count, 1.0 * step, 0.0002); currSpecies["position"]["x"].storeChunk(mx, ProperExtent(offset, false), {count}); - } + } } // storeParticles /* @@ -732,14 +732,14 @@ void BasicParticlePattern::storeParticles(ParticleSpecies &currSpecies, int &ste */ void BasicParticlePattern::getParticleLayout(unsigned long& offset, unsigned long &count, unsigned long &total) { - { + { Checkpoint x1(" ComputeLayout", m_Input.m_MPIRank); if (m_Input.m_PtlMin >= m_Input.m_PtlMax) { count = m_Input.m_PtlMax; } else - { + { std::random_device rd; // a seed source for the random number engine std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() std::uniform_int_distribution<> distrib(m_Input.m_PtlMin, m_Input.m_PtlMax); @@ -756,7 +756,7 @@ void BasicParticlePattern::getParticleLayout(unsigned long& offset, unsigned lon } //Timer g("Gather Particle logistics ", m_Input.m_MPIRank); Checkpoint x(" GetPTLOffset", m_Input.m_MPIRank); - + std::vector result(m_Input.m_MPISize, 0); //unsigned long buffer[m_Input.m_MPISize]; MPI_Allgather (&count, 1, MPI_UNSIGNED_LONG, result.data(), 1, MPI_UNSIGNED_LONG, MPI_COMM_WORLD); From 309ad554132e94ba72e309c74ce3af933c3633cf Mon Sep 17 00:00:00 2001 From: guj Date: Wed, 4 Sep 2024 11:41:00 -0700 Subject: [PATCH 07/13] use defaults if no input file is supplied --- examples/8c_benchmark_ptl_parallel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index 005d418d76..5c3892f59c 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -487,10 +487,10 @@ int parseArgs(int argc, char *argv[], TestInput &input) } infile.close(); - return 1; + return 1; } - std::cout<<" Expecting: "< "< "< Date: Mon, 9 Sep 2024 17:57:22 -0400 Subject: [PATCH 08/13] minor fix with tabs --- examples/8c_benchmark_ptl_parallel.cpp | 30 +++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index 5c3892f59c..10e15259b7 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -317,6 +317,12 @@ std::vector getBackends(bool bpOnly) res.emplace_back(".bp"); #endif + if (bpOnly) { + if (res.size() == 0) + std::cerr<<" BP is not supported "< 0) { - if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) - input.m_UseJoinedDim = true; - } + if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) + input.m_UseJoinedDim = true; + } return; } @@ -509,17 +514,17 @@ void doWork(TestInput & input) if ( 0 < input.m_Backend.size() ) { BasicParticlePattern p(input); - p.printMe(); - p.run(); + p.printMe(); + p.run(); } else { for (auto const &which : backends) { input.m_Backend = which; - BasicParticlePattern p(input); - p.printMe(); - p.run(); + BasicParticlePattern p(input); + p.printMe(); + p.run(); } } } @@ -812,4 +817,3 @@ void BasicParticlePattern::printMe() << "\n\t NumPtls (millions) per rank/step: "< Date: Tue, 10 Sep 2024 14:04:56 -0700 Subject: [PATCH 09/13] added PDW string --- examples/8c_benchmark_ptl_parallel.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index 10e15259b7..779339069a 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -664,8 +664,11 @@ void BasicParticlePattern::store(Series &series, int step) storeParticles(currSpecies, step); if (m_Input.m_CallPDW) - series.flush(); - + { + std::string pdwStr = "PDW-"+std::to_string(step); + Timer pdwTimer(pdwStr, m_Input.m_MPIRank, Timer::FULL); + series.flush(); + } { Checkpoint remove2("Barrier_3", m_Input.m_MPIRank); MPI_Barrier(MPI_COMM_WORLD); From b175dcc6403e9d8c033e1329c44171cc7dc4c90c Mon Sep 17 00:00:00 2001 From: guj Date: Fri, 20 Sep 2024 13:22:11 -0700 Subject: [PATCH 10/13] parameter fix --- examples/8c_benchmark_ptl_parallel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index 779339069a..91eb0c3382 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -212,7 +212,7 @@ static std::map m_GlobalProfilers; class Checkpoint { public: - Checkpoint(std::string name, int rank) + Checkpoint(std::string const& name, int rank) :m_name(name) { auto fp = m_GlobalProfilers.find(name); From 348a1c31db80cc8d961f3dd3732b5e5048227da1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 20:29:08 +0000 Subject: [PATCH 11/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/8c_benchmark_ptl_parallel.cpp | 484 ++++++++++++++----------- 1 file changed, 264 insertions(+), 220 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index 91eb0c3382..20aa9c6303 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -44,16 +44,18 @@ using std::cout; using namespace openPMD; -using MaxResSteadyClock = std::conditional_t; +using MaxResSteadyClock = std::conditional_t< + std::chrono::high_resolution_clock::is_steady, + std::chrono::high_resolution_clock, + std::chrono::steady_clock>; /** The Memory profiler class for profiling purpose * * Simple Memory usage report that works on linux system */ -static std::chrono::time_point< MaxResSteadyClock > m_ProgStart = MaxResSteadyClock::now(); +static std::chrono::time_point m_ProgStart = + MaxResSteadyClock::now(); class MemoryProfiler { @@ -117,7 +119,6 @@ class MemoryProfiler std::string m_Name; }; - /** The Timer class for profiling purpose * * Simple Timer that measures time consumption btw constucture and destructor @@ -126,7 +127,12 @@ class MemoryProfiler class Timer { public: - enum VERBOSE_LEVEL {NONE, MIN, FULL}; + enum VERBOSE_LEVEL + { + NONE, + MIN, + FULL + }; /** * * Simple Timer @@ -136,108 +142,119 @@ class Timer */ Timer(const std::string &tag, int rank, VERBOSE_LEVEL vl = FULL) - //Timer(const std::string &tag, int rank, bool silent = false) + // Timer(const std::string &tag, int rank, bool silent = false) { m_Tag = tag; m_Rank = rank; - m_Start = MaxResSteadyClock::now(); - - /* - m_Silent = silent; - if (!m_Silent) - MemoryProfiler(rank, tag); - */ - m_Silent = vl; - if (m_Silent == FULL) - MemoryProfiler(rank, tag); + m_Start = MaxResSteadyClock::now(); + + /* + m_Silent = silent; + if (!m_Silent) + MemoryProfiler(rank, tag); + */ + m_Silent = vl; + if (m_Silent == FULL) + MemoryProfiler(rank, tag); } double getDuration() { auto curr = MaxResSteadyClock::now(); - double secs = std::chrono::duration_cast > (curr - m_Start).count(); - return secs; + double secs = + std::chrono::duration_cast >( + curr - m_Start) + .count(); + return secs; } ~Timer() { - if (m_Silent == NONE) + if (m_Silent == NONE) return; - if (m_Silent == FULL) - { - std::string tt = "~" + m_Tag; - MemoryProfiler mp(m_Rank, tt.c_str()); - } + if (m_Silent == FULL) + { + std::string tt = "~" + m_Tag; + MemoryProfiler mp(m_Rank, tt.c_str()); + } - double secs = getDuration(); + double secs = getDuration(); if (m_Rank > 0) return; std::cout << " [" << m_Tag << "] took:" << secs << " seconds"; std::cout << " Time Elapsed:" - << secs + std::chrono::duration_cast< std::chrono::duration > (m_Start - m_ProgStart).count() + << secs + + std::chrono::duration_cast >( + m_Start - m_ProgStart) + .count() << std::endl; std::cout << std::endl; } private: - std::chrono::time_point< MaxResSteadyClock > m_Start; + std::chrono::time_point m_Start; std::string m_Tag; int m_Rank = 0; - //bool m_Silent = false; + // bool m_Silent = false; VERBOSE_LEVEL m_Silent = Timer::NONE; }; class LocalProfiler { public: - LocalProfiler() = default; - ~LocalProfiler () = default; + LocalProfiler() = default; + ~LocalProfiler() = default; - void setRank(int r) {m_Rank = r;} - void update(Timer& timer) { m_Counter ++; m_Total += timer.getDuration(); } + void setRank(int r) + { + m_Rank = r; + } + void update(Timer &timer) + { + m_Counter++; + m_Total += timer.getDuration(); + } - int m_Rank = 0; // info only - int m_Counter=0; - double m_Total = 0; + int m_Rank = 0; // info only + int m_Counter = 0; + double m_Total = 0; }; -static std::map m_GlobalProfilers; +static std::map m_GlobalProfilers; class Checkpoint { public: - Checkpoint(std::string const& name, int rank) - :m_name(name) - { - auto fp = m_GlobalProfilers.find(name); - if ( fp == m_GlobalProfilers.end()) { - LocalProfiler p; - p.setRank(rank); - m_GlobalProfilers[name] = p; - } - m_Timer = new Timer(name, rank, Timer::NONE); - } - ~Checkpoint() - { - m_GlobalProfilers[m_name].update(*m_Timer); - - if (m_Timer != nullptr) - delete m_Timer; - } + Checkpoint(std::string const &name, int rank) : m_name(name) + { + auto fp = m_GlobalProfilers.find(name); + if (fp == m_GlobalProfilers.end()) + { + LocalProfiler p; + p.setRank(rank); + m_GlobalProfilers[name] = p; + } + m_Timer = new Timer(name, rank, Timer::NONE); + } + ~Checkpoint() + { + m_GlobalProfilers[m_name].update(*m_Timer); + + if (m_Timer != nullptr) + delete m_Timer; + } private: - Timer* m_Timer = nullptr; - std::string m_name; + Timer *m_Timer = nullptr; + std::string m_name; }; - - /** createDataCPU * generate a shared ptr of given size with given type & default value on * CPU @@ -295,9 +312,9 @@ template std::shared_ptr createData(const unsigned long &size, const T &val, const T &increment) { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - Checkpoint c(" CreateData", rank); + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + Checkpoint c(" CreateData", rank); #if openPMD_HAVE_CUDA_EXAMPLES return createDataGPU(size, val, increment); @@ -317,15 +334,16 @@ std::vector getBackends(bool bpOnly) res.emplace_back(".bp"); #endif - if (bpOnly) { - if (res.size() == 0) - std::cerr<<" BP is not supported "< 0) { - if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) + if (vec[1].size() > 0) + { + if ((vec[1][0] == 't') or (vec[1][0] == 'T')) input.m_UseJoinedDim = true; } return; @@ -438,24 +457,27 @@ void parse(TestInput &input, std::string line) if (vec[0].compare("usePDW") == 0) { - if (vec[1].size() > 0) { - if ( (vec[1][0] == 't') or (vec[1][0] == 'T') ) - input.m_CallPDW = true; + if (vec[1].size() > 0) + { + if ((vec[1][0] == 't') or (vec[1][0] == 'T')) + input.m_CallPDW = true; } return; } if (vec[0].compare("maxMil") == 0) { - input.m_PtlMax = ( (unsigned long) atoi(vec[1].c_str()) ) * (unsigned long) 1000000; + input.m_PtlMax = + ((unsigned long)atoi(vec[1].c_str())) * (unsigned long)1000000; return; } if (vec[0].compare("minMil") == 0) { - input.m_PtlMin = ( (unsigned long) atoi(vec[1].c_str()) ) * (unsigned long) 1000000; - if (input.m_PtlMin > input.m_PtlMax) - input.m_PtlMin = input.m_PtlMax; + input.m_PtlMin = + ((unsigned long)atoi(vec[1].c_str())) * (unsigned long)1000000; + if (input.m_PtlMin > input.m_PtlMax) + input.m_PtlMin = input.m_PtlMax; return; } @@ -494,46 +516,45 @@ int parseArgs(int argc, char *argv[], TestInput &input) infile.close(); return 1; } - std::cout<<" No input file. Using defaults. Otherwise, try: "< "< " << std::endl; return 1; } - /** TEST doWork * * run the actual test scenarios using the input */ -void doWork(TestInput & input) +void doWork(TestInput &input) { Checkpoint g("Total: ", input.m_MPIRank); auto const backends = getBackends(input.m_UseJoinedDim); try { - if ( 0 < input.m_Backend.size() ) - { - BasicParticlePattern p(input); - p.printMe(); - p.run(); - } - else - { - for (auto const &which : backends) - { - input.m_Backend = which; + if (0 < input.m_Backend.size()) + { BasicParticlePattern p(input); p.printMe(); p.run(); - } - } + } + else + { + for (auto const &which : backends) + { + input.m_Backend = which; + BasicParticlePattern p(input); + p.printMe(); + p.run(); + } + } } catch (std::exception const &ex) { if (0 == input.m_MPIRank) std::cout << "Error: " << ex.what() << std::endl; } - } /** TEST MAIN * @@ -553,32 +574,41 @@ int main(int argc, char *argv[]) return -1; } - doWork(input); { - MPI_Barrier(MPI_COMM_WORLD); - if ( 0 == input.m_MPIRank ) { - std::cout<<" ============= GLOBAL PROFILER SUMMARY =========="< result(input.m_MPISize, 0); - //unsigned long buffer[m_Input.m_MPISize]; - MPI_Allgather (&p.m_Total, 1, MPI_DOUBLE, result.data(), 1, MPI_DOUBLE, MPI_COMM_WORLD); - - auto [min, max] = std::minmax_element(result.begin(),result.end()); + MPI_Barrier(MPI_COMM_WORLD); + if (0 == input.m_MPIRank) + { + std::cout << " ============= GLOBAL PROFILER SUMMARY ==========" + << std::endl; + std::cout << "NAME: \t\t NumCalls: \t Min(sec): \t Max (secs): \n"; + } - if ( 0 == input.m_MPIRank ) - std::cout << name << "\t\t "< result(input.m_MPISize, 0); + // unsigned long buffer[m_Input.m_MPISize]; + MPI_Allgather( + &p.m_Total, + 1, + MPI_DOUBLE, + result.data(), + 1, + MPI_DOUBLE, + MPI_COMM_WORLD); + + auto [min, max] = std::minmax_element(result.begin(), result.end()); + + if (0 == input.m_MPIRank) + std::cout << name << "\t\t " << p.m_Counter << "\t" << *min + << " \t " << *max << " \t :peek " << result[0] << " " + << result[input.m_MPISize - 1] << std::endl; + } } MPI_Finalize(); - return 0; } @@ -586,7 +616,8 @@ int main(int argc, char *argv[]) * Class BasicParticlePattern * @param input: (user input class) */ -BasicParticlePattern::BasicParticlePattern(const TestInput &input) : m_Input(input) +BasicParticlePattern::BasicParticlePattern(const TestInput &input) + : m_Input(input) {} /* @@ -599,8 +630,8 @@ void BasicParticlePattern::run() if (m_Input.m_Encoding == openPMD::IterationEncoding::fileBased) { // file based std::ostringstream s; - s << m_Input.m_Prefix << "/" <(), ProperExtent(np, true)); - auto const realDataSet = - openPMD::Dataset(openPMD::determineDatatype(), ProperExtent(np, true)); + auto const intDataSet = openPMD::Dataset( + openPMD::determineDatatype(), ProperExtent(np, true)); + auto const realDataSet = openPMD::Dataset( + openPMD::determineDatatype(), ProperExtent(np, true)); currSpecies["id"].resetDataset(intDataSet); currSpecies["charge"].resetDataset(realDataSet); currSpecies["position"]["x"].resetDataset(realDataSet); - //currSpecies["positionOffset"]["x"].resetDataset(realDataSet); - //currSpecies["positionOffset"]["x"].makeConstant(0.); - + // currSpecies["positionOffset"]["x"].resetDataset(realDataSet); + // currSpecies["positionOffset"]["x"].makeConstant(0.); { - Checkpoint remove2(" SP_Barrier_2", m_Input.m_MPIRank); - MPI_Barrier(MPI_COMM_WORLD); + Checkpoint remove2(" SP_Barrier_2", m_Input.m_MPIRank); + MPI_Barrier(MPI_COMM_WORLD); } - Checkpoint remove3(" SP_cs", m_Input.m_MPIRank); if (count > 0) - { - auto ids = createData(count, offset, 1); - currSpecies["id"].storeChunk(ids, ProperExtent(offset, false), {count}); + { + auto ids = createData(count, offset, 1); + currSpecies["id"].storeChunk(ids, ProperExtent(offset, false), {count}); - auto charges = createData(count, 0.1 * step, 0.0001); - currSpecies["charge"].storeChunk(charges, ProperExtent(offset, false), {count}); + auto charges = createData(count, 0.1 * step, 0.0001); + currSpecies["charge"].storeChunk( + charges, ProperExtent(offset, false), {count}); - auto mx = createData(count, 1.0 * step, 0.0002); - currSpecies["position"]["x"].storeChunk(mx, ProperExtent(offset, false), {count}); - } + auto mx = createData(count, 1.0 * step, 0.0002); + currSpecies["position"]["x"].storeChunk( + mx, ProperExtent(offset, false), {count}); + } } // storeParticles /* @@ -738,85 +769,98 @@ void BasicParticlePattern::storeParticles(ParticleSpecies &currSpecies, int &ste * set to be a multiple of mesh size * */ -void BasicParticlePattern::getParticleLayout(unsigned long& offset, unsigned long &count, unsigned long &total) +void BasicParticlePattern::getParticleLayout( + unsigned long &offset, unsigned long &count, unsigned long &total) { - { - Checkpoint x1(" ComputeLayout", m_Input.m_MPIRank); - if (m_Input.m_PtlMin >= m_Input.m_PtlMax) - { - count = m_Input.m_PtlMax; - } - else { - std::random_device rd; // a seed source for the random number engine - std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() - std::uniform_int_distribution<> distrib(m_Input.m_PtlMin, m_Input.m_PtlMax); - - //for (int n = 0; n != 10; ++n) - // std::cout << distrib(gen) << ' '; - count = distrib(gen); - } - - // gather from all ranks to get offset/total - - if (m_Input.m_UseJoinedDim) - return; - } - //Timer g("Gather Particle logistics ", m_Input.m_MPIRank); - Checkpoint x(" GetPTLOffset", m_Input.m_MPIRank); + Checkpoint x1(" ComputeLayout", m_Input.m_MPIRank); + if (m_Input.m_PtlMin >= m_Input.m_PtlMax) + { + count = m_Input.m_PtlMax; + } + else + { + std::random_device rd; // a seed source for the random number engine + std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() + std::uniform_int_distribution<> distrib( + m_Input.m_PtlMin, m_Input.m_PtlMax); + + // for (int n = 0; n != 10; ++n) + // std::cout << distrib(gen) << ' '; + count = distrib(gen); + } - std::vector result(m_Input.m_MPISize, 0); - //unsigned long buffer[m_Input.m_MPISize]; - MPI_Allgather (&count, 1, MPI_UNSIGNED_LONG, result.data(), 1, MPI_UNSIGNED_LONG, MPI_COMM_WORLD); + // gather from all ranks to get offset/total - total = 0; - auto const num_results = static_cast(result.size()); - for (int i=0; i result(m_Input.m_MPISize, 0); + // unsigned long buffer[m_Input.m_MPISize]; + MPI_Allgather( + &count, + 1, + MPI_UNSIGNED_LONG, + result.data(), + 1, + MPI_UNSIGNED_LONG, + MPI_COMM_WORLD); + + total = 0; + auto const num_results = static_cast(result.size()); + for (int i = 0; i < num_results; i++) { - total += result[i]; - if (i < m_Input.m_MPIRank) { - offset += result[i]; - } + total += result[i]; + if (i < m_Input.m_MPIRank) + { + offset += result[i]; + } } - } const std::string BasicParticlePattern::getBaseFileName() const { - if (m_Input.m_UseJoinedDim) - return "8a_parallel_ptl_joined"; - return "8a_parallel_ptl"; + if (m_Input.m_UseJoinedDim) + return "8a_parallel_ptl_joined"; + return "8a_parallel_ptl"; } - -openPMD::Extent BasicParticlePattern::ProperExtent (unsigned long long n, bool init) const +openPMD::Extent +BasicParticlePattern::ProperExtent(unsigned long long n, bool init) const { - if (!m_Input.m_UseJoinedDim) - return {n}; + if (!m_Input.m_UseJoinedDim) + return {n}; - if (init) - return {openPMD::Dataset::JOINED_DIMENSION}; - else - return {}; + if (init) + return {openPMD::Dataset::JOINED_DIMENSION}; + else + return {}; } /* * Print pattern layout */ void BasicParticlePattern::printMe() { - if ( 0 < m_Input.m_MPIRank ) - return; - - std::string pdw_status=" just EndStep"; - if (m_Input.m_CallPDW) - pdw_status=" PDW + EndStep"; + if (0 < m_Input.m_MPIRank) + return; - if (m_Input.m_UseJoinedDim) - std::cout << " ====> This is a Particle Only test, With Joined Dimension applied to ADIOS."< This is a Particle Only test. " < This is a Particle Only test, With Joined " + "Dimension applied to ADIOS." + << pdw_status << std::endl; + else + std::cout << " ====> This is a Particle Only test. " << pdw_status + << std::endl; - std::cout << "\t Num steps: "< Date: Fri, 20 Sep 2024 14:11:09 -0700 Subject: [PATCH 12/13] touch up after CodeQL --- examples/8c_benchmark_ptl_parallel.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index 20aa9c6303..ca95f0f6ef 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -498,8 +498,9 @@ int parseArgs(int argc, char *argv[], TestInput &input) { if (argc == 2) { + std::string filename = argv[1]; std::fstream infile; - infile.open(argv[1], std::ios::in); + infile.open(filename, std::ios::in); if (!infile.is_open()) { if (input.m_MPIRank == 0) @@ -685,7 +686,6 @@ void BasicParticlePattern::store(Series &series, int step) std::cout<<" STEP: "< result(m_Input.m_MPISize, 0); - // unsigned long buffer[m_Input.m_MPISize]; MPI_Allgather( &count, 1, From ab0ba02a2c5e185df5f2ad3dfe4d663e2f090cb7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 21:12:02 +0000 Subject: [PATCH 13/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/8c_benchmark_ptl_parallel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/8c_benchmark_ptl_parallel.cpp b/examples/8c_benchmark_ptl_parallel.cpp index ca95f0f6ef..165efa2401 100644 --- a/examples/8c_benchmark_ptl_parallel.cpp +++ b/examples/8c_benchmark_ptl_parallel.cpp @@ -498,7 +498,7 @@ int parseArgs(int argc, char *argv[], TestInput &input) { if (argc == 2) { - std::string filename = argv[1]; + std::string filename = argv[1]; std::fstream infile; infile.open(filename, std::ios::in); if (!infile.is_open())