Skip to content

Commit 0d54b4b

Browse files
maorz1998maorz1998
authored
Merge pull request #306 from maorz1998/GPU
update docs, configues, and installation
2 parents 425721d + 6c53c42 commit 0d54b4b

10 files changed

Lines changed: 101 additions & 106 deletions

File tree

‎Allwclean‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,4 @@ wclean ./applications/solvers/dfHighSpeedFoam
2525
rm -rf src_orig/
2626
rm -rf bin/
2727
rm -rf lib/
28+
rm -rf src_gpu/build

‎applications/solvers/dfLowMachFoam/EEqn.H‎

Lines changed: 22 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,5 @@
11
{
22
volScalarField& he = thermo.he();
3-
4-
#ifdef CPUSolver_
5-
start1 = std::clock();
6-
//debug
7-
// {
8-
// const fvPatchScalarField& hew = he.boundaryField()[5];
9-
// const basicThermo& bThermo = basicThermo::lookupThermo(hew);
10-
// const scalarField& pw = bThermo.p().boundaryField()[5];
11-
// fvPatchScalarField& Tw =
12-
// const_cast<fvPatchScalarField&>(bThermo.T().boundaryField()[5]);
13-
// scalarField& Tw_v = Tw;
14-
15-
// Tw.evaluate();
16-
17-
// Info << "internal field" <<bThermo.he(pw, Tw, mesh.boundary()[5].faceCells()) << endl;
18-
// Info << "boundary field" <<bThermo.he(pw, Tw, 5) << endl;
19-
// Info << "calculated grad" << mesh.boundary()[5].deltaCoeffs() * (bThermo.he(pw, Tw, 5) - bThermo.he(pw, Tw, mesh.boundary()[5].faceCells())) << endl;
20-
// }
21-
22-
23-
fvScalarMatrix EEqn
24-
(
25-
fvm::ddt(rho, he) + fvm::div(phi, he)
26-
+ fvc::ddt(rho, K) + fvc::div(phi, K)
27-
- dpdt
28-
- fvm::laplacian(turbulence->alphaEff(), he)
29-
+ diffAlphaD
30-
==
31-
fvc::div(hDiffCorrFlux)
32-
);
33-
end1 = std::clock();
34-
time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
35-
time_monitor_EEqn_mtxAssembly += double(end1 - start1) / double(CLOCKS_PER_SEC);
36-
37-
// EEqn.relax();
38-
start1 = std::clock();
39-
EEqn.solve();
40-
end1 = std::clock();
41-
time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
42-
time_monitor_EEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC);
43-
#endif
44-
453
#ifdef GPUSolver_
464
start1 = std::clock();
475
UEqn_GPU.updatePsi(&U[0][0]);
@@ -51,9 +9,7 @@
519
end1 = std::clock();
5210
time_monitor_UEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
5311
time_monitor_UEqn_correctBC += double(end1 - start1) / double(CLOCKS_PER_SEC);
54-
#endif
5512

56-
#ifdef GPUSolver_
5713
// prepare data on CPU
5814
start1 = std::clock();
5915
start2 = std::clock();
@@ -148,5 +104,27 @@
148104
end1 = std::clock();
149105
time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
150106
time_monitor_EEqn_correctBC += double(end1 - start1) / double(CLOCKS_PER_SEC);
107+
#else
108+
start1 = std::clock();
109+
fvScalarMatrix EEqn
110+
(
111+
fvm::ddt(rho, he) + fvm::div(phi, he)
112+
+ fvc::ddt(rho, K) + fvc::div(phi, K)
113+
- dpdt
114+
- fvm::laplacian(turbulence->alphaEff(), he)
115+
+ diffAlphaD
116+
==
117+
fvc::div(hDiffCorrFlux)
118+
);
119+
end1 = std::clock();
120+
time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
121+
time_monitor_EEqn_mtxAssembly += double(end1 - start1) / double(CLOCKS_PER_SEC);
122+
123+
// EEqn.relax();
124+
start1 = std::clock();
125+
EEqn.solve();
126+
end1 = std::clock();
127+
time_monitor_EEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
128+
time_monitor_EEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC);
151129
#endif
152130
}

‎applications/solvers/dfLowMachFoam/Make/options‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ EXE_INC = -std=c++14 \
99
$(PFLAGS) $(PINC) \
1010
$(if $(LIBTORCH_ROOT),-DUSE_LIBTORCH,) \
1111
$(if $(PYTHON_INC_DIR),-DUSE_PYTORCH,) \
12+
$(if $(AMGX_DIR),-DGPUSolver_,) \
1213
-I$(LIB_SRC)/transportModels/compressible/lnInclude \
1314
-I$(LIB_SRC)/thermophysicalModels/basic/lnInclude \
1415
-I$(LIB_SRC)/TurbulenceModels/turbulenceModels/lnInclude \
@@ -26,9 +27,9 @@ EXE_INC = -std=c++14 \
2627
$(if $(LIBTORCH_ROOT),-I$(LIBTORCH_ROOT)/include,) \
2728
$(if $(LIBTORCH_ROOT),-I$(LIBTORCH_ROOT)/include/torch/csrc/api/include,) \
2829
$(PYTHON_INC_DIR) \
29-
-I$(DF_ROOT)/src_gpu \
30-
-I/usr/local/cuda-11.6/include \
31-
-I$(AMGX_DIR)/include
30+
$(if $(AMGX_DIR), -I$(DF_ROOT)/src_gpu,) \
31+
$(if $(AMGX_DIR), -I/usr/local/cuda-11.6/include,) \
32+
$(if $(AMGX_DIR), -I$(AMGX_DIR)/include,)
3233

3334
EXE_LIBS = \
3435
-lcompressibleTransportModels \
@@ -50,7 +51,7 @@ EXE_LIBS = \
5051
$(if $(LIBTORCH_ROOT),$(DF_SRC)/dfChemistryModel/DNNInferencer/build/libDNNInferencer.so,) \
5152
$(if $(PYTHON_LIB_DIR),-L$(PYTHON_LIB_DIR),) \
5253
$(if $(PYTHON_LIB_DIR),-lpython3.8,) \
53-
$(DF_ROOT)/src_gpu/build/libdfMatrix.so \
54-
/usr/local/cuda-11.6/lib64/libcudart.so \
55-
$(AMGX_DIR)/build/libamgxsh.so
54+
$(if $(AMGX_DIR), /usr/local/cuda-11.6/lib64/libcudart.so,) \
55+
$(if $(AMGX_DIR), $(DF_ROOT)/src_gpu/build/libdfMatrix.so,) \
56+
$(if $(AMGX_DIR), $(AMGX_DIR)/build/libamgxsh.so,)
5657

‎applications/solvers/dfLowMachFoam/UEqn.H‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -79,16 +79,6 @@
7979
// }
8080
// }
8181
// }
82-
// const tmp<volTensorField> tgradU(fvc::grad(U));
83-
// const volTensorField& gradU = tgradU();
84-
// Pout << "gradU_of[1]\n" << gradU[1] << nl;
85-
// Pout << "gradU_of[0][64]\n" << gradU.boundaryField()[0][64] << nl;
86-
// Pout << "gradU_of[1][64]\n" << gradU.boundaryField()[1][64] << nl;
87-
// Pout << "gradU_of[5][1]\n" << gradU.boundaryField()[5][1] << nl;
88-
// Pout << "U[1][1]\n" << U[1] << nl;
89-
// Pout << "Ubou[0][64]\n" << U.boundaryField()[0][64] << nl;
90-
// Pout << "Ubou[1][64]\n" << U.boundaryField()[1][64] << nl;
91-
// Pout << "Ubou[5][1]\n" << U.boundaryField()[5][1] << nl;
9282
// if (pimple.momentumPredictor())
9383
// {
9484
// solve(UEqn);

‎applications/solvers/dfLowMachFoam/YEqn.H‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,20 +12,6 @@ tmp<fv::convectionScheme<scalar>> mvConvection
1212
mesh.divScheme("div(phi,Yi_h)")
1313
)
1414
);
15-
16-
#ifdef CPUSolver_
17-
start1 = std::clock();
18-
forAll(Y, i)
19-
{
20-
sumYDiffError += chemistry->rhoD(i)*fvc::grad(Y[i]);
21-
}
22-
// Info << "sumYDiffError\n" << sumYDiffError << endl;
23-
const surfaceScalarField phiUc = linearInterpolate(sumYDiffError) & mesh.Sf();
24-
start1 = std::clock();
25-
time_monitor_YEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
26-
time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC);
27-
#endif
28-
2915
#ifdef GPUSolver_
3016
start1 = std::clock();
3117
UEqn_GPU.solve();
@@ -124,6 +110,17 @@ time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC);
124110
end1 = std::clock();
125111
time_monitor_YEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
126112
time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC);
113+
#else
114+
start1 = std::clock();
115+
forAll(Y, i)
116+
{
117+
sumYDiffError += chemistry->rhoD(i)*fvc::grad(Y[i]);
118+
}
119+
// Info << "sumYDiffError\n" << sumYDiffError << endl;
120+
const surfaceScalarField phiUc = linearInterpolate(sumYDiffError) & mesh.Sf();
121+
start1 = std::clock();
122+
time_monitor_YEqn += double(end1 - start1) / double(CLOCKS_PER_SEC);
123+
time_monitor_YEqn_solve += double(end1 - start1) / double(CLOCKS_PER_SEC);
127124
#endif
128125

129126
//MPI_Barrier(PstreamGlobals::MPI_COMM_FOAM);

‎applications/solvers/dfLowMachFoam/dfLowMachFoam.C‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,15 @@ Description
5959
#include "basicThermo.H"
6060
#include "CombustionModel.H"
6161

62+
#ifdef GPUSolver_
6263
#include "dfUEqn.H"
6364
#include "dfYEqn.H"
6465
#include "dfRhoEqn.H"
6566
#include "dfEEqn.H"
6667
#include <cuda_runtime.h>
6768
#include <thread>
6869
#include "upwind.H"
69-
70-
#define GPUSolver_
71-
// #define CPUSolver_
70+
#endif
7271

7372
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
7473

@@ -79,10 +78,10 @@ int main(int argc, char *argv[])
7978
#endif
8079
#include "postProcess.H"
8180

82-
unsigned int flags = 0;
83-
checkCudaErrors(cudaGetDeviceFlags(&flags));
84-
flags |= cudaDeviceScheduleYield;
85-
checkCudaErrors(cudaSetDeviceFlags(flags));
81+
// unsigned int flags = 0;
82+
// checkCudaErrors(cudaGetDeviceFlags(&flags));
83+
// flags |= cudaDeviceScheduleYield;
84+
// checkCudaErrors(cudaSetDeviceFlags(flags));
8685

8786
// #include "setRootCaseLists.H"
8887
#include "listOptions.H"
@@ -158,7 +157,9 @@ int main(int argc, char *argv[])
158157
}
159158

160159
start1 = std::clock();
160+
#ifdef GPUSolver_
161161
#include "createdfSolver.H"
162+
#endif
162163
end1 = std::clock();
163164
time_monitor_init += double(end1 - start1) / double(CLOCKS_PER_SEC);
164165

‎bashrc.in‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,11 @@ export CANTERA_ROOT=LIBCANTERA_DIR
88
export CANTERA_DATA=$CANTERA_ROOT/share/cantera/data
99
export LD_LIBRARY_PATH=$LIBTORCH_ROOT/lib:$LD_LIBRARY_PATH
1010
export LD_LIBRARY_PATH=$CANTERA_ROOT/lib:$LD_LIBRARY_PATH
11+
export AMGX_DIR=@AMGX_DIR@
1112

1213
export DF_APPBIN=pwd/bin
1314
export DF_LIBBIN=pwd/lib
1415
export PATH=$DF_APPBIN:$PATH
1516
export LD_LIBRARY_PATH=$DF_LIBBIN:$LD_LIBRARY_PATH
1617
export LD_LIBRARY_PATH=$DF_ROOT/src_gpu/build:$LD_LIBRARY_PATH
17-
export LD_LIBRARY_PATH=path-to-amgx/build:$LD_LIBRARY_PATH
18-
export AMGX_DIR=path-to-amgx
19-
18+
export LD_LIBRARY_PATH=$AMGX_DIR/build:$LD_LIBRARY_PATH

‎configure.sh‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,19 @@ unset LIBTORCH_DIR
77
unset LIBCANTERA_DIR
88
unset PYTORCH_INC
99
unset PYTORCH_LIB
10+
unset USE_GPUSOLVER
11+
unset AMGX_DIR
1012

1113
print_usage() {
12-
echo "Usage: . install.sh --libtorch_no (default) | --libtorch_dir _path_to_libtorch | --libtorch_autodownload | --use_pytorch | --libcantera_dir _path_to_libcantera"
14+
echo "Usage: . install.sh --libtorch_no (default) | --libtorch_dir _path_to_libtorch | --libtorch_autodownload | --use_pytorch | --libcantera_dir _path_to_libcantera
15+
| --amgx_dir _path_to_amgx"
1316
}
1417

1518
# default
1619
LIBTORCH_AUTO=false
1720
USE_LIBTORCH=false
1821
USE_PYTORCH=false
22+
USE_GPUSOLVER=false
1923

2024
while test $# -gt 0; do
2125
case "$1" in
@@ -60,6 +64,17 @@ while test $# -gt 0; do
6064
fi
6165
shift
6266
;;
67+
--amgx_dir)
68+
shift
69+
if test $# -gt 0; then
70+
AMGX_DIR=$1
71+
USE_GPUSOLVER=true
72+
else
73+
print_usage
74+
return
75+
fi
76+
shift
77+
;;
6378
-h|--help)
6479
shift
6580
print_usage
@@ -148,13 +163,17 @@ if [ $USE_PYTORCH = true ]; then
148163
echo PYTORCH_LIB=$PYTORCH_LIB
149164
echo LIBTORCH_DIR=""
150165
fi
166+
if [ $USE_GPUSOLVER = true ]; then
167+
echo AMGX_DIR=$AMGX_DIR
168+
fi
151169

152170
cp bashrc.in bashrc
153171
sed -i "s#pwd#$PWD#g" ./bashrc
154172
sed -i "s#LIBTORCH_DIR#$LIBTORCH_DIR#g" ./bashrc
155173
sed -i "s#PYTORCH_INC#$PYTORCH_INC#g" ./bashrc
156174
sed -i "s#PYTORCH_LIB#$PYTORCH_LIB#g" ./bashrc
157175
sed -i "s#LIBCANTERA_DIR#$LIBCANTERA_DIR#g" ./bashrc
176+
sed -i "s#@AMGX_DIR@#$AMGX_DIR#g" ./bashrc
158177

159178

160179

‎docs/source/qs/install.rst‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -142,27 +142,29 @@ If ``df-notorch`` not activated (or you have a self-complied libcantera), specif
142142
. install.sh
143143
144144
145-
**3. If you wish to employ the AMGX library for accelerating PDE solving using GPU:**
145+
**3. If you wish to employ dfMatrix and the AMGX library for accelerating PDE solving using GPU:**
146+
147+
.. Note:: This is still under developement.
146148

147149
To begin, you will need to install AMGX. You can find the instructions for installing AMGX on its official website. Follow the instructions provided to install AMGX on your system. Once you have installed AMGX, navigate to the DeepFlame directory and follow the commands below.
148150

149151
.. code-block:: bash
150152
151-
cd ${DF_ROOT}/src/dfMatrix/solver/amgx/
152-
export AMGX_DIR=/your/path/to/AMGX/
153-
cmake -B build
154-
cd build
155-
make
156-
157-
After this, two libraries for enabling DeepFlame with AMGX are available in ``${DF_ROOT}/src/dfMatrix/solver/amgx/build``.
158-
Beforing using AMGX, run:
153+
cd deepflame-dev
154+
. configure.sh --amgx_dir /your/path/to/AMGX/
155+
source ./bashrc
156+
. install.sh
159157
160-
.. code-block:: bash
158+
Also, you will need to add configuration files for AMGX for each euqation under ``system`` folder and name them in the pattern of ``amgxpOptions``, ``amgxUOptions`` . Please refer to the AMGX official website to find out detailed instructions.
161159

162-
export LD_LIBRARY_PATH=${DF_ROOT}/src/dfMatrix/solver/amgx/build:$LD_LIBRARY_PATH
160+
**If you have compiled DeepFlame with GPU solver successfully, you should see the print message in your terminal:**
163161

162+
.. code-block::
164163
165-
If you want to use AMGX, you will need to add configuration files for AMGX for each euqation under ``system`` folder and name them in the pattern of ``amgxpOptions``, ``amgxUOptions`` . Please refer to the AMGX official website to find out detailed instructions.
164+
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
165+
| deepflame (linked with libcantera) compiled successfully! Enjoy!! |
166+
| select the GPU solver coupled with AMGx library to solve PDE |
167+
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
166168
167169
168170
**4. If you wish to install DeepFlame with CMake**

‎install.sh‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@ print_finish() {
44
if [ ! -z "$LIBTORCH_ROOT" ]; then
55
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
66
echo "| deepflame (linked with libcantera and libtorch) compiled successfully! Enjoy!! |"
7+
elif [ ! -z "$PYTHON_LIB_DIR" ]; then
78
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
8-
return
9+
echo "| deepflame (linked with libcantera and pytorch) compiled successfully! Enjoy!! |"
10+
else
11+
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
12+
echo "| deepflame (linked with libcantera) compiled successfully! Enjoy!! |"
913
fi
10-
if [ ! -z "$PYTHON_LIB_DIR" ]; then
11-
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
12-
echo "| deepflame (linked with libcantera and pytorch) compiled successfully! Enjoy!! | "
13-
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
14-
return
14+
if [ ! -z "$AMGX_DIR" ]; then
15+
echo "| select the GPU solver coupled with AMGx library to solve PDE |"
1516
fi
16-
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
17-
echo "| deepflame (linked with libcantera) compiled successfully! Enjoy!! |"
18-
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
17+
echo " = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = ="
1918
}
2019
if [ $USE_LIBTORCH = true ]; then
2120
cd "$DF_SRC/dfChemistryModel/DNNInferencer"
@@ -25,5 +24,13 @@ if [ $USE_LIBTORCH = true ]; then
2524
make
2625
export LD_LIBRARY_PATH=$DF_SRC/dfChemistryModel/DNNInferencer/build:$LD_LIBRARY_PATH
2726
fi
27+
if [ $USE_GPUSOLVER = true ]; then
28+
cd "$DF_ROOT/src_gpu"
29+
mkdir build
30+
cd build
31+
cmake ..
32+
make
33+
export LD_LIBRARY_PATH=$DF_ROOT/src_gpu/build:$LD_LIBRARY_PATH
34+
fi
2835
cd $DF_ROOT
2936
./Allwmake -j && print_finish

0 commit comments

Comments
 (0)