diff --git a/foam/cavityParallel/0/U b/foam/cavityParallel/0/U new file mode 100644 index 00000000..cf8ef735 --- /dev/null +++ b/foam/cavityParallel/0/U @@ -0,0 +1,40 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: plus | +| \\ / A nd | Web: www.OpenFOAM.com | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class volVectorField; + object U; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +dimensions [0 1 -1 0 0 0 0]; + +internalField uniform (0 0 0); + +boundaryField +{ + movingWall + { + type fixedValue; + value uniform (1 0 0); + } + + fixedWalls + { + type noSlip; + } + + frontAndBack + { + type empty; + } +} + +// ************************************************************************* // diff --git a/foam/cavityParallel/0/p b/foam/cavityParallel/0/p new file mode 100644 index 00000000..0976329c --- /dev/null +++ b/foam/cavityParallel/0/p @@ -0,0 +1,39 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: plus | +| \\ / A nd | Web: www.OpenFOAM.com | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class volScalarField; + object p; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +dimensions [0 2 -2 0 0 0 0]; + +internalField uniform 0; + +boundaryField +{ + movingWall + { + type zeroGradient; + } + + fixedWalls + { + type zeroGradient; + } + + frontAndBack + { + type empty; + } +} + +// ************************************************************************* // diff --git a/foam/cavityParallel/Allclean b/foam/cavityParallel/Allclean new file mode 100755 index 00000000..ea5e2d94 --- /dev/null +++ b/foam/cavityParallel/Allclean @@ -0,0 +1 @@ +rm -rf processor* diff --git a/foam/cavityParallel/ExecutionTime-512.csv b/foam/cavityParallel/ExecutionTime-512.csv new file mode 100644 index 00000000..e206b75f --- /dev/null +++ b/foam/cavityParallel/ExecutionTime-512.csv @@ -0,0 +1,7 @@ +,amount_processor,time_max,speedup,efficiency +0,1,880.53,0.0,0.0 +1,2,507.05,1.7365743023370475,0.8682871511685237 +2,4,414.59,2.1238573048071587,0.5309643262017897 +3,8,215.96,4.077282830153732,0.5096603537692165 +4,12,162.28,5.4259921123983235,0.45216600936652696 +5,16,153.04,5.753593831677993,0.35959961447987454 diff --git a/foam/cavityParallel/README.md b/foam/cavityParallel/README.md new file mode 100644 index 00000000..270f2f7f --- /dev/null +++ b/foam/cavityParallel/README.md @@ -0,0 +1,44 @@ +Benchmarking for cluster environment with openFOAM and SLURM as job Scheduler + +The environment should be run as follows: + +1 - OpenMPI/3.0.0 or another MPI compiler (such as MPICH and Intel) configured; + +2 - OpenFOAM tool; + +3 - Cluster environment configured; + +4 - scipy or matplotlib, numpy and pandas dependencies (python >= 2.7); + +steps to test: +``` +$ source /foam-directory/etc/bashrc + +$ ./Allclean +``` + +Edit system/decomposeParDict where numberOfSubdomains = $number_of_processors_available + +and + +``` +simpleCoeffs +{ + n ( 4 4 1 ); + delta 0.0001; +} +``` +for this example has been used n ( 4 4 ...) this parameters should be change as of total of your processors. In this case, the total is 4 x 4 = 16 + +``` +$ blockMesh (to create mesh for processing) + +$ decomposePar -force (to split the mesh as of total processors able) +``` + +Configure "jobslurm.sh" according to what your hardware and resources. + +``` +$ sbatch jobslurm.sh (to running simulation) +``` + diff --git a/foam/cavityParallel/constant/polyMesh/blockMeshDict b/foam/cavityParallel/constant/polyMesh/blockMeshDict new file mode 100644 index 00000000..ae6dce83 --- /dev/null +++ b/foam/cavityParallel/constant/polyMesh/blockMeshDict @@ -0,0 +1,75 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: plus | +| \\ / A nd | Web: www.OpenFOAM.com | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + object blockMeshDict; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +scale 0.1; + +vertices +( + (0 0 0) + (1 0 0) + (1 1 0) + (0 1 0) + (0 0 0.1) + (1 0 0.1) + (1 1 0.1) + (0 1 0.1) +); + +blocks +( + hex (0 1 2 3 4 5 6 7) (1024 1024 1) simpleGrading (1 1 1) +); + +edges +( +); + +boundary +( + movingWall + { + type wall; + faces + ( + (3 7 6 2) + ); + } + fixedWalls + { + type wall; + faces + ( + (0 4 7 3) + (2 6 5 1) + (1 5 4 0) + ); + } + frontAndBack + { + type empty; + faces + ( + (0 3 2 1) + (4 5 6 7) + ); + } +); + +mergePatchPairs +( +); + +// ************************************************************************* // diff --git a/foam/cavityParallel/constant/transportProperties b/foam/cavityParallel/constant/transportProperties new file mode 100644 index 00000000..12a4fd4b --- /dev/null +++ b/foam/cavityParallel/constant/transportProperties @@ -0,0 +1,21 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: 2.3.0 | +| \\ / A nd | Web: www.OpenFOAM.org | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + location "constant"; + object transportProperties; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +nu nu [ 0 2 -1 0 0 0 0 ] 0.001; + + +// ************************************************************************* // diff --git a/foam/cavityParallel/jobslurm.sh b/foam/cavityParallel/jobslurm.sh new file mode 100755 index 00000000..606a2749 --- /dev/null +++ b/foam/cavityParallel/jobslurm.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copy/paste this job script into a text file and submit with the command: +# sbatch thefilename + +#SBATCH --time=20:00:00 # walltime limit (HH:MM:SS) +#SBATCH --nodes=4 # number of nodes +#SBATCH --ntasks-per-node=4 # 4 processor core(s) per node +#SBATCH --job-name="nativa-jobs" +#SBATCH --output="slurm-%j.out.log" # job standard output file (%j replaced by job id) +#SBATCH --error="slurm-%j.err.out.log" # job standard error file (%j replaced by job id) + +# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE +module load gnu7 + +mpiexec --hostfile machines -np 16 --allow-run-as-root /home/admceltab/OpenFOAM-v1712/bin/foamExec icoFoam -parallel diff --git a/foam/cavityParallel/machines b/foam/cavityParallel/machines new file mode 100644 index 00000000..22bb84a5 --- /dev/null +++ b/foam/cavityParallel/machines @@ -0,0 +1,4 @@ +n1 +n2 +n3 +n4 diff --git a/foam/cavityParallel/metrics.sh b/foam/cavityParallel/metrics.sh new file mode 100755 index 00000000..bd73e45c --- /dev/null +++ b/foam/cavityParallel/metrics.sh @@ -0,0 +1,18 @@ +#!/bin/bash +echo "Filtering log files of openFOAM simulation..." + +if [ $# -lt 1 ]; then + echo "Faltou utilizar pelo menos um argumento!" + exit 1 +fi + +ls $1 | grep out > /tmp/files +cd $1 +while read line; do + progs=($line) + cut -d: -f 10,10 ${progs[0]} | grep ExecutionTime | sed 's/[a-z A-Z]\+//g' | cut -d= -f 1,2 | sed 's/[a-z A-Z =,]\+//g' > log-${progs[0]} + sed -i '1 i\time' log-${progs[0]} +done < /tmp/files +rm /tmp/files + +echo "Finishing filter" diff --git a/foam/cavityParallel/plot.py b/foam/cavityParallel/plot.py new file mode 100644 index 00000000..b1cf674d --- /dev/null +++ b/foam/cavityParallel/plot.py @@ -0,0 +1,96 @@ +''' +This file is a test for shared mpi4py libraries in a cluster environment. +The application can be run... +''' +__author__ = 'Fernando Demarchi Natividade Luiz' +__email__ = 'nativanando@gmail.com' + +import matplotlib.pyplot as plt +import csv +import pandas as pd +import numpy as np + +class BenchMarkMetrics: + def __init__(self): + self.df1 = pd.read_csv('results-512/log-out-1x1.txt') + self.df2 = pd.read_csv('results-512/log-out-2x1.txt') + self.df3 = pd.read_csv('results-512/log-out-2x2.txt') + self.df4 = pd.read_csv('results-512/log-out-4x2.txt') + self.df5 = pd.read_csv('results-512/log-out-6x2.txt') + self.df6 = pd.read_csv('results-512/log-out-4x4.txt') + + def plot_graph_execution_time(self): + # plt.plot(self.df1['time']) + plt.plot(self.df1['time'], label='1 processor') + plt.plot(self.df2['time'], label='2 processors') + plt.plot(self.df3['time'], label='4 processors') + plt.plot(self.df4['time'], label='8 processors') + plt.plot(self.df5['time'], label='12 processors') + plt.plot(self.df6['time'], label='16 processors') + plt.title('CELTAB Cluster Metrics') + plt.legend(loc='upper left') + plt.xlabel('epochs') + plt.ylabel('execution time (sec)') + plt.grid(True) + plt.savefig('assets/benchmark-512.png') + + def print_execution_time(self): + with open('ExecutionTime-512.csv', 'w') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames= ['amount_processor', 'time_max', 'speedup', 'efficiency']) + writer.writeheader() + writer.writerow({'amount_processor': 1, 'time_max': self.df1['time'].max()}) + writer.writerow({'amount_processor': 2, 'time_max': self.df2['time'].max()}) + writer.writerow({'amount_processor': 4, 'time_max': self.df3['time'].max()}) + writer.writerow({'amount_processor': 8, 'time_max': self.df4['time'].max()}) + writer.writerow({'amount_processor': 12, 'time_max': self.df5['time'].max()}) + writer.writerow({'amount_processor': 16, 'time_max': self.df6['time'].max()}) + self.clear_buffer_plt() + print(self.df1.describe()) + + def speedup_calculation(self): + # s(p) = t(1) / t(p) + data = pd.read_csv('ExecutionTime-512.csv') + for i in range(1, data['amount_processor'].count()): + data['speedup'][i] = (data['time_max'][0] / data['time_max'][i]) + data['efficiency'][i] = (data['speedup'][i] / data['amount_processor'][i]) + data = self.clean_nan_values(data) + data.to_csv('ExecutionTime-512.csv') + + def plot_graph_speedup(self): + data = pd.read_csv('ExecutionTime-512.csv') + plt.plot(data['amount_processor'], data['speedup'], label="IcoFoam Cavity Simulation", marker='o', linestyle='--', color='g') + plt.title('CELTAB Cluster Metrics') + plt.legend(loc='upper left') + plt.xlabel('cores') + plt.ylabel('speedup') + plt.grid(True) + plt.savefig('assets/speedup-512.png') + self.clear_buffer_plt() + + def plot_graph_efficiency(self): + data = pd.read_csv('ExecutionTime-512.csv') + plt.plot(data['amount_processor'], data['efficiency'], label="IcoFoam Cavity Simulation", marker='o', linestyle='--', color='r') + plt.title('CELTAB Cluster Metrics') + plt.legend(loc='upper right') + plt.xlabel('cores') + plt.ylabel('efficiency') + plt.grid(True) + plt.savefig('assets/efficiency-512.png') + self.clear_buffer_plt() + + def clean_nan_values(self, dataset): + dataset = dataset.fillna(0) + return dataset + + def clear_buffer_plt(self): + plt.cla() + plt.clf() + + +if __name__ == '__main__': + plot = BenchMarkMetrics() + plot.plot_graph_execution_time() + plot.print_execution_time() + plot.speedup_calculation() + plot.plot_graph_speedup() + plot.plot_graph_efficiency() diff --git a/foam/cavityParallel/system/.#decomposeParDict b/foam/cavityParallel/system/.#decomposeParDict new file mode 120000 index 00000000..aaccfd78 --- /dev/null +++ b/foam/cavityParallel/system/.#decomposeParDict @@ -0,0 +1 @@ +admceltab@sms.990:1526471467 \ No newline at end of file diff --git a/foam/cavityParallel/system/blockMeshDict b/foam/cavityParallel/system/blockMeshDict new file mode 100644 index 00000000..6c09069c --- /dev/null +++ b/foam/cavityParallel/system/blockMeshDict @@ -0,0 +1,75 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: plus | +| \\ / A nd | Web: www.OpenFOAM.com | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + object blockMeshDict; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +scale 0.1; + +vertices +( + (0 0 0) + (1 0 0) + (1 1 0) + (0 1 0) + (0 0 0.1) + (1 0 0.1) + (1 1 0.1) + (0 1 0.1) +); + +blocks +( + hex (0 1 2 3 4 5 6 7) (20 20 1) simpleGrading (1 1 1) +); + +edges +( +); + +boundary +( + movingWall + { + type wall; + faces + ( + (3 7 6 2) + ); + } + fixedWalls + { + type wall; + faces + ( + (0 4 7 3) + (2 6 5 1) + (1 5 4 0) + ); + } + frontAndBack + { + type empty; + faces + ( + (0 3 2 1) + (4 5 6 7) + ); + } +); + +mergePatchPairs +( +); + +// ************************************************************************* // diff --git a/foam/cavityParallel/system/controlDict b/foam/cavityParallel/system/controlDict new file mode 100644 index 00000000..b0bb9d24 --- /dev/null +++ b/foam/cavityParallel/system/controlDict @@ -0,0 +1,49 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: 2.3.0 | +| \\ / A nd | Web: www.OpenFOAM.org | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + location "system"; + object controlDict; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +application icoFoam; + +startFrom startTime; + +startTime 0; + +stopAt endTime; + +endTime 0.001; + +deltaT 0.00001; + +writeControl runTime; + +writeInterval 0.1; + +purgeWrite 0; + +writeFormat ascii; + +writePrecision 6; + +writeCompression off; + +timeFormat general; + +timePrecision 6; + +runTimeModifiable false; + +// ************************************************************************* // + diff --git a/foam/cavityParallel/system/decomposeParDict b/foam/cavityParallel/system/decomposeParDict new file mode 100644 index 00000000..fa3892c3 --- /dev/null +++ b/foam/cavityParallel/system/decomposeParDict @@ -0,0 +1,34 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: 2.1.1 | +| \\ / A nd | Web: www.OpenFOAM.org | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + location "system"; + object decomposeParDict; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +numberOfSubdomains 16; + +method simple; + +simpleCoeffs +{ + n ( 8 2 1 ); + delta 0.0001; +} + +distributed no; + +roots ( ); + + +// ************************************************************************* // + diff --git a/foam/cavityParallel/system/fvSchemes b/foam/cavityParallel/system/fvSchemes new file mode 100644 index 00000000..b43ea748 --- /dev/null +++ b/foam/cavityParallel/system/fvSchemes @@ -0,0 +1,51 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: plus | +| \\ / A nd | Web: www.OpenFOAM.com | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + location "system"; + object fvSchemes; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +ddtSchemes +{ + default Euler; +} + +gradSchemes +{ + default Gauss linear; + grad(p) Gauss linear; +} + +divSchemes +{ + default none; + div(phi,U) Gauss linear; +} + +laplacianSchemes +{ + default Gauss linear orthogonal; +} + +interpolationSchemes +{ + default linear; +} + +snGradSchemes +{ + default orthogonal; +} + + +// ************************************************************************* // diff --git a/foam/cavityParallel/system/fvSolution b/foam/cavityParallel/system/fvSolution new file mode 100644 index 00000000..fb6e8e46 --- /dev/null +++ b/foam/cavityParallel/system/fvSolution @@ -0,0 +1,52 @@ +/*--------------------------------*- C++ -*----------------------------------*\ +| ========= | | +| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox | +| \\ / O peration | Version: plus | +| \\ / A nd | Web: www.OpenFOAM.com | +| \\/ M anipulation | | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + version 2.0; + format ascii; + class dictionary; + location "system"; + object fvSolution; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +solvers +{ + p + { + solver PCG; + preconditioner DIC; + tolerance 1e-06; + relTol 0.05; + } + + pFinal + { + $p; + relTol 0; + } + + U + { + solver smoothSolver; + smoother symGaussSeidel; + tolerance 1e-05; + relTol 0; + } +} + +PISO +{ + nCorrectors 2; + nNonOrthogonalCorrectors 0; + pRefCell 0; + pRefValue 0; +} + + +// ************************************************************************* // diff --git a/foam/cavityParallel/system/singleGraph b/foam/cavityParallel/system/singleGraph new file mode 100644 index 00000000..0202307d --- /dev/null +++ b/foam/cavityParallel/system/singleGraph @@ -0,0 +1,27 @@ +/*--------------------------------*- C++ -*----------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox +| \\ / O peration | Version: plus | +| \\ / A nd | Web: www.OpenFOAM.com | + \\/ M anipulation | +------------------------------------------------------------------------------- +Description + Writes graph data for specified fields along a line, specified by start + and end points. + +\*---------------------------------------------------------------------------*/ + +start (0 1e-06 0); +end (1 1e-06 0); +fields (U p); + +// Sampling and I/O settings +#includeEtc "caseDicts/postProcessing/graphs/sampleDict.cfg" + +// Override settings here, e.g. +// setConfig { type midPoint; } + +// Must be last entry +#includeEtc "caseDicts/postProcessing/graphs/graph.cfg" + +// ************************************************************************* //