Skip to content

Commit 6f94483

Browse files
authored
work around a ROCm 7.2.0 compiler bug (#3350)
this just moves some if-logic out of the GPU kernel.
1 parent fcd4fff commit 6f94483

4 files changed

Lines changed: 57 additions & 63 deletions

File tree

Source/hydro/Castro_ctu_hydro.cpp

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -583,7 +583,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
583583
areay_arr,
584584
vol_arr,
585585
hdt, hdtdy);
586-
587586
reset_edge_state_thermo(xbx, ql_arr);
588587

589588
reset_edge_state_thermo(xbx, qr_arr);
@@ -626,7 +625,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
626625
areax_arr,
627626
vol_arr,
628627
hdt, hdtdx);
629-
630628
reset_edge_state_thermo(ybx, ql_arr);
631629

632630
reset_edge_state_thermo(ybx, qr_arr);
@@ -708,7 +706,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
708706
#endif
709707
qgdnvtmp1_arr,
710708
hdt, cdtdx);
711-
712709
reset_edge_state_thermo(tyxbx, qmyx.array());
713710

714711
reset_edge_state_thermo(tyxbx, qpyx.array());
@@ -734,7 +731,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
734731
#endif
735732
qgdnvtmp1_arr,
736733
hdt, cdtdx);
737-
738734
reset_edge_state_thermo(tzxbx, qmzx.array());
739735

740736
reset_edge_state_thermo(tzxbx, qpzx.array());
@@ -755,7 +751,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
755751
qgdnvtmp1_arr,
756752
qaux_arr, shk_arr,
757753
1, false);
758-
759754
// [lo(1), lo(2), lo(3)-1], [hi(1)+1, hi(2), lo(3)+1]
760755
const Box& txybx = amrex::grow(xbx, IntVect(AMREX_D_DECL(0,0,1)));
761756

@@ -780,7 +775,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
780775
#endif
781776
qgdnvtmp1_arr,
782777
hdt, cdtdy);
783-
784778
reset_edge_state_thermo(txybx, qmxy.array());
785779

786780
reset_edge_state_thermo(txybx, qpxy.array());
@@ -809,7 +803,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
809803
#endif
810804
qgdnvtmp1_arr,
811805
hdt, cdtdy);
812-
813806
reset_edge_state_thermo(tzybx, qmzy.array());
814807

815808
reset_edge_state_thermo(tzybx, qpzy.array());
@@ -830,7 +823,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
830823
qgdnvtmp1_arr,
831824
qaux_arr, shk_arr,
832825
2, false);
833-
834826
// [lo(1)-1, lo(2)-1, lo(3)], [hi(1)+1, hi(2)+1, lo(3)]
835827
const Box& txzbx = amrex::grow(xbx, IntVect(AMREX_D_DECL(0,1,0)));
836828

@@ -855,7 +847,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
855847
#endif
856848
qgdnvtmp1_arr,
857849
hdt, cdtdz);
858-
859850
reset_edge_state_thermo(txzbx, qmxz.array());
860851

861852
reset_edge_state_thermo(txzbx, qpxz.array());
@@ -884,7 +875,6 @@ Castro::construct_ctu_hydro_source(Real time, Real dt) // NOLINT(readability-co
884875
#endif
885876
qgdnvtmp1_arr,
886877
hdt, cdtdz);
887-
888878
reset_edge_state_thermo(tyzbx, qmyz.array());
889879

890880
reset_edge_state_thermo(tyzbx, qpyz.array());

Source/hydro/riemann.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ Castro::cmpflx_plus_godunov(const Box& bx,
6666
const auto domlo = geom.Domain().loVect3d();
6767
const auto domhi = geom.Domain().hiVect3d();
6868

69+
const bool do_ppm_temp_fix = ppm_temp_fix == 2;
70+
const Real temp_guess = T_guess;
71+
6972
amrex::ParallelFor(bx,
7073
[=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
7174
{
@@ -82,7 +85,8 @@ Castro::cmpflx_plus_godunov(const Box& bx,
8285
qm, qp, qaux_arr,
8386
qint,
8487
special_bnd_lo, special_bnd_hi,
85-
domlo, domhi);
88+
domlo, domhi,
89+
do_ppm_temp_fix, temp_guess);
8690

8791
// now use the interface state to compute and store the flux
8892

Source/hydro/riemann_solvers.H

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,8 @@ riemann_state(const int i, const int j, const int k, const int idir,
224224
Array4<Real const> const& qaux_arr,
225225
RiemannState& qint,
226226
const bool special_bnd_lo, const bool special_bnd_hi,
227-
GpuArray<int, 3> const& domlo, GpuArray<int, 3> const& domhi) {
227+
GpuArray<int, 3> const& domlo, GpuArray<int, 3> const& domhi,
228+
const bool do_ppm_temp_fix, const Real temp_guess) {
228229

229230
// just compute the hydrodynamic state on the interfaces
230231
// don't compute the fluxes
@@ -235,7 +236,7 @@ riemann_state(const int i, const int j, const int k, const int idir,
235236
// Riemann problems
236237

237238

238-
if (ppm_temp_fix == 2) {
239+
if (do_ppm_temp_fix) {
239240
// recompute the thermodynamics on the interface to make it
240241
// all consistent
241242

@@ -247,7 +248,7 @@ riemann_state(const int i, const int j, const int k, const int idir,
247248

248249
// this is an initial guess for iterations, since we
249250
// can't be certain what temp is on interfaces
250-
eos_state.T = T_guess;
251+
eos_state.T = temp_guess;
251252

252253
// minus state
253254
eos_state.rho = qm(i,j,k,QRHO);

Source/hydro/trans.cpp

Lines changed: 48 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ Castro::trans_single(const Box& bx,
6060
vol,
6161
#endif
6262
hdt, cdtdx);
63+
6364
}
6465

6566

@@ -107,7 +108,37 @@ Castro::actual_trans_single(const Box& bx, // NOLINT(readability-convert-member
107108

108109
bool reset_density = transverse_reset_density;
109110
bool reset_rhoe = transverse_reset_rhoe;
111+
bool use_eos = transverse_use_eos;
110112
Real small_p = small_pres;
113+
Real temp_guess = T_guess;
114+
Real abundance_tol = abundance_failure_tolerance;
115+
116+
int ilo = 0;
117+
int jlo = 0;
118+
int klo = 0;
119+
120+
int ihi = 0;
121+
int jhi = 0;
122+
int khi = 0;
123+
124+
if (idir_t == 0) {
125+
ihi = 1;
126+
} else if (idir_t == 1) {
127+
jhi = 1;
128+
} else {
129+
khi = 1;
130+
}
131+
132+
if (idir_n == 0) {
133+
ilo += d;
134+
ihi += d;
135+
} else if (idir_n == 1) {
136+
jlo += d;
137+
jhi += d;
138+
} else {
139+
klo += d;
140+
khi += d;
141+
}
111142

112143
amrex::ParallelFor(bx,
113144
[=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
@@ -117,48 +148,13 @@ Castro::actual_trans_single(const Box& bx, // NOLINT(readability-convert-member
117148
// (i, i+1) in the x-direction, and similarly for
118149
// the y- and z- directions.
119150

120-
int il = i;
121-
int jl = j; // NOLINT(misc-confusable-identifiers)
122-
int kl = k;
123-
124-
int ir = i;
125-
int jr = j;
126-
int kr = k;
151+
const int il = i + ilo;
152+
const int jl = j + jlo; // NOLINT(misc-confusable-identifiers)
153+
const int kl = k + klo;
127154

128-
// set the face indices in the transverse direction
129-
130-
if (idir_t == 0) {
131-
ir = i+1;
132-
jr = j;
133-
kr = k;
134-
135-
} else if (idir_t == 1) {
136-
ir = i;
137-
jr = j+1;
138-
kr = k;
139-
140-
} else {
141-
ir = i;
142-
jr = j;
143-
kr = k+1;
144-
}
145-
146-
// We're handling both the plus and minus states;
147-
// for the minus state we're shifting one zone to
148-
// the left in our chosen direction.
149-
150-
if (idir_n == 0) {
151-
il += d;
152-
ir += d;
153-
154-
} else if (idir_n == 1) {
155-
jl += d;
156-
jr += d;
157-
158-
} else {
159-
kl += d;
160-
kr += d;
161-
}
155+
const int ir = i + ihi;
156+
const int jr = j + jhi;
157+
const int kr = k + khi;
162158

163159
// Update all of the passively-advected quantities with the
164160
// transverse term and convert back to the primitive quantity.
@@ -384,8 +380,8 @@ Castro::actual_trans_single(const Box& bx, // NOLINT(readability-convert-member
384380
// Reset to original value if adding transverse terms made any mass fraction invalid.
385381

386382
for (int n = 0; n < NumSpec; ++n) {
387-
if (qo_arr(i,j,k,n+QFS) > 1.0_rt + castro::abundance_failure_tolerance ||
388-
qo_arr(i,j,k,n+QFS) < -castro::abundance_failure_tolerance) {
383+
if (qo_arr(i,j,k,n+QFS) > 1.0_rt + abundance_tol ||
384+
qo_arr(i,j,k,n+QFS) < -abundance_tol) {
389385
rrnewn = rrn;
390386
runewn = run;
391387
rvnewn = rvn;
@@ -443,13 +439,13 @@ Castro::actual_trans_single(const Box& bx, // NOLINT(readability-convert-member
443439
// Pretend QREINT has been fixed
444440
// We can get pressure update via eos or using p-evolution equation
445441

446-
if (transverse_use_eos) {
442+
if (use_eos) {
447443

448444
// With the EOS route:
449445
eos_rep_t eos_state;
450446
eos_state.rho = rrnewn;
451447
eos_state.e = qo_arr(i,j,k,QREINT) * rhoinv;
452-
eos_state.T = T_guess;
448+
eos_state.T = temp_guess;
453449

454450
for (int n = 0; n < NumSpec; n++) {
455451
eos_state.xn[n] = qo_arr(i,j,k,QFS+n);
@@ -581,7 +577,10 @@ Castro::actual_trans_final(const Box& bx, // NOLINT(readability-convert-member-
581577

582578
bool reset_density = transverse_reset_density;
583579
bool reset_rhoe = transverse_reset_rhoe;
580+
bool use_eos = transverse_use_eos;
584581
Real small_p = small_pres;
582+
Real temp_guess = T_guess;
583+
Real abundance_tol = abundance_failure_tolerance;
585584

586585
amrex::ParallelFor(bx,
587586
[=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
@@ -860,8 +859,8 @@ Castro::actual_trans_final(const Box& bx, // NOLINT(readability-convert-member-
860859
// Reset to original value if adding transverse terms made any mass fraction invalid.
861860

862861
for (int n = 0; n < NumSpec; ++n) {
863-
if (qo_arr(i,j,k,n+QFS) > 1.0_rt + castro::abundance_failure_tolerance ||
864-
qo_arr(i,j,k,n+QFS) < -castro::abundance_failure_tolerance) {
862+
if (qo_arr(i,j,k,n+QFS) > 1.0_rt + abundance_tol ||
863+
qo_arr(i,j,k,n+QFS) < -abundance_tol) {
865864
rrnewn = rrn;
866865
runewn = run;
867866
rvnewn = rvn;
@@ -914,13 +913,13 @@ Castro::actual_trans_final(const Box& bx, // NOLINT(readability-convert-member-
914913
// Pretend QREINT has been fixed
915914
// We can get pressure update via eos or using p-evolution equation
916915

917-
if (transverse_use_eos) {
916+
if (use_eos) {
918917

919918
// With the EOS route:
920919
eos_rep_t eos_state;
921920
eos_state.rho = rrnewn;
922921
eos_state.e = qo_arr(i,j,k,QREINT) / rrnewn;
923-
eos_state.T = T_guess;
922+
eos_state.T = temp_guess;
924923

925924
for (int n = 0; n < NumSpec; n++) {
926925
eos_state.xn[n] = qo_arr(i,j,k,QFS+n);

0 commit comments

Comments
 (0)