Merge branch 'development' into MagneX_hybrid_ML

Author: ajnonaka

Exec/DMI_diagnostic/DMI_diagnostic.cpp

@@ -118,10 +118,10 @@ main (int   argc,
             int k = (hi.z+1)/2;
             int j = (hi.y+1)/2;
             for (auto i = (hi.x+1)/2; i <= hi.x; ++i) {
-                std::cout << i << " " << mfdata(i,j,k,3)/1.1e6 << " " << mfdata(i,j,k,16)+offset << "\n";
+                std::cout << i << " " << mfdata(i,j,k,2)/1.1e6 << " " << mfdata(i,j,k,3)+offset << "\n";
                 // 2pi cyclic fix
                 if (i<hi.x) {
-                    if (offset==0. && mfdata(i,j,k,16)>5. && mfdata(i+1,j,k,16)<1.) {
+                    if (offset==0. && mfdata(i,j,k,3)>5. && mfdata(i+1,j,k,3)<1.) {
                         offset = 2.*M_PI;
                     }
                 }

Exec/GNUmakefile

@@ -74,43 +74,14 @@ endif
 # else
 #   LDFLAGS += -Wl,--no-as-needed,-rpath=$(TORCH_LIBPATH) $(TORCH_LIBS)
 # endif
+SUNDIALS_HOME ?= ../../sundials/instdir
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
 include ../Source/Make.package
 VPATH_LOCATIONS  += ../Source
 INCLUDE_LOCATIONS += ../Source
 
-ifeq ($(USE_SUNDIALS),TRUE)
-  ifeq ($(USE_CUDA),TRUE)
-    SUNDIALS_ROOT ?= $(TOP)../../sundials/instdir_cuda
-  else
-    SUNDIALS_ROOT ?= $(TOP)../../sundials/instdir
-  endif
-  ifeq ($(NERSC_HOST),perlmutter)
-    SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib64
-  else
-    SUNDIALS_LIB_DIR ?= $(SUNDIALS_ROOT)/lib
-  endif
-
-  USE_CVODE_LIBS ?= TRUE
-  USE_ARKODE_LIBS ?= TRUE
-
-  DEFINES += -DAMREX_USE_SUNDIALS
-  INCLUDE_LOCATIONS += $(SUNDIALS_ROOT)/include
-  LIBRARY_LOCATIONS += $(SUNDIALS_LIB_DIR)
-
-  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_cvode
-  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_arkode
-  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nvecmanyvector
-  LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_core
-
-  ifeq ($(USE_CUDA),TRUE)
-    LIBRARIES += -L$(SUNDIALS_LIB_DIR) -lsundials_nveccuda
-  endif
-
-endif
-
 include $(AMREX_HOME)/Src/Base/Make.package
 include $(AMREX_HOME)/Src/FFT/Make.package
 include $(AMREX_HOME)/Src/Boundary/Make.package 

Exec/README_sundials

@@ -1,94 +1,4 @@
-SUNDIALS installation guide:
-https://computing.llnl.gov/projects/sundials/faq#inst
+Refer to https://amrex-codes.github.io/amrex/docs_html/TimeIntegration_Chapter.html
+for SNUDIALS installation, build, and usage instructions.
 
-Installation
-
-# You need SUNDIALS v7.1.1 or later.
-# Check https://computing.llnl.gov/projects/sundials/sundials-software to see if it's available for download.
-# If so, download sundials-x.y.z.tar.gz and extract it at the same level as amrex using
->> tar -xzvf sundials-x.y.z.tar.gz # where x.y.z is the version of sundials
->> mv sundials-x.y.z sundials-src
-
-# If v7.1.1. is not available on the website, clone the git repo directly and use the latest version
-# At the same level that amrex is cloned, do:
-
->> git clone https://github.com/LLNL/sundials.git
->> mv sundials sundials-src
-
-# Next
-
->> mkdir sundials
->> cd sundials
-
-######################
-HOST BUILD 
-######################
-
->> mkdir instdir
->> mkdir builddir
->> cd builddir
-
->> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir/examples -DENABLE_MPI=ON ../../sundials-src
-
-######################
-NVIDIA/CUDA BUILD
-######################
-
-# Navigate back to the 'sundials' directory and do:
-
->> mkdir instdir_cuda
->> mkdir builddir_cuda
->> cd builddir_cuda
-
->> cmake -DCMAKE_INSTALL_PREFIX=/pathto/sundials/instdir_cuda -DEXAMPLES_INSTALL_PATH=/pathto/sundials/instdir_cuda/examples -DENABLE_CUDA=ON -DENABLE_MPI=ON ../../sundials-src
-
-######################
-
->> make -j4
->> make install
-
-# in your .bashrc or preferred configuration file, add the following (and then "source ~/.bashrc")
-
-# If you have a CPU build:
-
->> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir/lib/
-
-# If you have a NVIDIA/CUDA build:
-
->> export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pathto/sundials/instdir_cuda/lib/
-
-# now you are ready to compile MagneX with:
-
->> make -j4 USE_SUNDIALS=TRUE # optional to have 'USE_CUDA=TRUE' as well
-
-# in your inputs file, you will need to have:
-
-TimeIntegratorOption = 4 #amrex/sundials backend integrators
-
-# INTEGRATION
-## integration.type can take on the following values:
-## 0 or "ForwardEuler" => Native AMReX Forward Euler integrator
-## 1 or "RungeKutta"   => Native AMReX Explicit Runge Kutta controlled by integration.rk.type
-## 2 or "SUNDIALS"     => SUNDIALS backend controlled by integration.sundials.strategy
-integration.type = SUNDIALS
-
-## Native AMReX Explicit Runge-Kutta parameters
-#
-## integration.rk.type can take the following values:
-### 0 = User-specified Butcher Tableau
-### 1 = Forward Euler
-### 2 = Trapezoid Method
-### 3 = SSPRK3 Method
-### 4 = RK4 Method
-integration.rk.type = 1
-
-# Set the SUNDIALS method type:
-# ERK      = Explicit Runge-Kutta method
-# DIRK     = Diagonally Implicit Runge-Kutta method
-#
-# Optionally select a specific SUNDIALS method by name, see the SUNDIALS
-# documentation for the supported method names
-
-# Use forward Euler (fixed step sizes only)
-integration.sundials.type = ERK
-integration.sundials.method = ARKODE_FORWARD_EULER_1_1
+Make sure that SUNDIALS_HOME in the GNUmakefile points to the installation directory.

Exec/scaling_inputs/inputs_scaling_cpu_0.25

@@ -0,0 +1,52 @@
+amrex.use_gpu_aware_mpi=1
+
+n_cell = 128 128 128
+max_grid_size_x = 32
+max_grid_size_y = 32
+max_grid_size_z = 64
+
+dt = 1.e-15
+nsteps = 100
+plot_int = -1
+chk_int = -1
+restart = -1
+
+prob_lo = 0. 0. 0.
+prob_hi = 512.e-9 512.e-9 512.e-9
+
+mu0 = 1.25663e-6
+
+Mx_parser(x,y,z) = "8.e5 * (x>0.)*(x<512.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9)"
+My_parser(x,y,z) = "0."
+Mz_parser(x,y,z) = "0."
+
+# Field 1:  mu_0 Hx=-24.6 mT, mu_0 Hy= 4.3 mT, mu_0 Hz= 0.0 mT
+# which is a field approximately 25 mT, directed 170 degrees counterclockwise from the positive x axis
+timedependent_Hbias = 0
+Hx_bias_parser(x,y,z,t) = "1.e5"
+Hy_bias_parser(x,y,z,t) = "1.e5"
+Hz_bias_parser(x,y,z,t) = "1.e5"
+
+timedependent_alpha = 0
+alpha_parser(x,y,z,t) =    "(x>0.)*(x<512.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * 0.5"
+Ms_parser(x,y,z) =         "(x>0.)*(x<512.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * 8.e5"
+gamma_parser(x,y,z) =      "(x>0.)*(x<512.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * -1.759e11"
+exchange_parser(x,y,z) =   "(x>0.)*(x<512.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * 1.3e-11"
+anisotropy_parser(x,y,z) = "(x>0.)*(x<512.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * -139.26"
+DMI_parser(x,y,z) =        "(x>0.)*(x<512.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * -4.5e-3"
+
+precession = 1
+demag_coupling = 1
+FFT_solver = 1
+M_normalization = 1  # 0 = unsaturated case; 1 = saturated case
+exchange_coupling = 1
+anisotropy_coupling = 1
+anisotropy_axis = 0.0 0.0 1.0
+DMI_coupling = 1
+
+# INTEGRATION
+
+TimeIntegratorOption = 1 #Forward Euler
+
+integration.type = RungeKutta
+integration.rk.type = 1

Exec/scaling_inputs/inputs_scaling_cpu_0.5

@@ -0,0 +1,52 @@
+amrex.use_gpu_aware_mpi=1
+
+n_cell = 256 128 128
+max_grid_size_x = 32
+max_grid_size_y = 32
+max_grid_size_z = 64
+
+dt = 1.e-15
+nsteps = 100
+plot_int = -1
+chk_int = -1
+restart = -1
+
+prob_lo = 0. 0. 0.
+prob_hi = 1024.e-9 512.e-9 512.e-9
+
+mu0 = 1.25663e-6
+
+Mx_parser(x,y,z) = "8.e5 * (x>0.)*(x<1024.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9)"
+My_parser(x,y,z) = "0."
+Mz_parser(x,y,z) = "0."
+
+# Field 1:  mu_0 Hx=-24.6 mT, mu_0 Hy= 4.3 mT, mu_0 Hz= 0.0 mT
+# which is a field approximately 25 mT, directed 170 degrees counterclockwise from the positive x axis
+timedependent_Hbias = 0
+Hx_bias_parser(x,y,z,t) = "1.e5"
+Hy_bias_parser(x,y,z,t) = "1.e5"
+Hz_bias_parser(x,y,z,t) = "1.e5"
+
+timedependent_alpha = 0
+alpha_parser(x,y,z,t) =    "(x>0.)*(x<1024.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * 0.5"
+Ms_parser(x,y,z) =         "(x>0.)*(x<1024.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * 8.e5"
+gamma_parser(x,y,z) =      "(x>0.)*(x<1024.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * -1.759e11"
+exchange_parser(x,y,z) =   "(x>0.)*(x<1024.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * 1.3e-11"
+anisotropy_parser(x,y,z) = "(x>0.)*(x<1024.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * -139.26"
+DMI_parser(x,y,z) =        "(x>0.)*(x<1024.e-9)*(y>0.)*(y<512.e-9)*(z>0.)*(z<512.e-9) * -4.5e-3"
+
+precession = 1
+demag_coupling = 1
+FFT_solver = 1
+M_normalization = 1  # 0 = unsaturated case; 1 = saturated case
+exchange_coupling = 1
+anisotropy_coupling = 1
+anisotropy_axis = 0.0 0.0 1.0
+DMI_coupling = 1
+
+# INTEGRATION
+
+TimeIntegratorOption = 1 #Forward Euler
+
+integration.type = RungeKutta
+integration.rk.type = 1

Exec/scaling_inputs/inputs_scaling_cpu_1

@@ -0,0 +1,52 @@
+amrex.use_gpu_aware_mpi=1
+
+n_cell = 256 256 128
+max_grid_size_x = 32
+max_grid_size_y = 32
+max_grid_size_z = 64
+
+dt = 1.e-15
+nsteps = 100
+plot_int = -1
+chk_int = -1
+restart = -1
+
+prob_lo = 0. 0. 0.
+prob_hi = 1024.e-9 1024.e-9 512.e-9
+
+mu0 = 1.25663e-6
+
+Mx_parser(x,y,z) = "8.e5 * (x>0.)*(x<1024.e-9)*(y>0.)*(y<1024.e-9)*(z>0.)*(z<512.e-9)"
+My_parser(x,y,z) = "0."
+Mz_parser(x,y,z) = "0."
+
+# Field 1:  mu_0 Hx=-24.6 mT, mu_0 Hy= 4.3 mT, mu_0 Hz= 0.0 mT
+# which is a field approximately 25 mT, directed 170 degrees counterclockwise from the positive x axis
+timedependent_Hbias = 0
+Hx_bias_parser(x,y,z,t) = "1.e5"
+Hy_bias_parser(x,y,z,t) = "1.e5"
+Hz_bias_parser(x,y,z,t) = "1.e5"
+
+timedependent_alpha = 0
+alpha_parser(x,y,z,t) =    "(x>0.)*(x<1024.e-9)*(y>0.)*(y<1024.e-9)*(z>0.)*(z<512.e-9) * 0.5"
+Ms_parser(x,y,z) =         "(x>0.)*(x<1024.e-9)*(y>0.)*(y<1024.e-9)*(z>0.)*(z<512.e-9) * 8.e5"
+gamma_parser(x,y,z) =      "(x>0.)*(x<1024.e-9)*(y>0.)*(y<1024.e-9)*(z>0.)*(z<512.e-9) * -1.759e11"
+exchange_parser(x,y,z) =   "(x>0.)*(x<1024.e-9)*(y>0.)*(y<1024.e-9)*(z>0.)*(z<512.e-9) * 1.3e-11"
+anisotropy_parser(x,y,z) = "(x>0.)*(x<1024.e-9)*(y>0.)*(y<1024.e-9)*(z>0.)*(z<512.e-9) * -139.26"
+DMI_parser(x,y,z) =        "(x>0.)*(x<1024.e-9)*(y>0.)*(y<1024.e-9)*(z>0.)*(z<512.e-9) * -4.5e-3"
+
+precession = 1
+demag_coupling = 1
+FFT_solver = 1
+M_normalization = 1  # 0 = unsaturated case; 1 = saturated case
+exchange_coupling = 1
+anisotropy_coupling = 1
+anisotropy_axis = 0.0 0.0 1.0
+DMI_coupling = 1
+
+# INTEGRATION
+
+TimeIntegratorOption = 1 #Forward Euler
+
+integration.type = RungeKutta
+integration.rk.type = 1

Exec/scaling_inputs/inputs_scaling_cpu_128

@@ -0,0 +1,52 @@
+amrex.use_gpu_aware_mpi=1
+
+n_cell = 1024 1024 1024
+max_grid_size_x = 32
+max_grid_size_y = 32
+max_grid_size_z = 64
+
+dt = 1.e-15
+nsteps = 100
+plot_int = -1
+chk_int = -1
+restart = -1
+
+prob_lo = 0. 0. 0.
+prob_hi = 4096.e-9 4096.e-9 4096.e-9
+
+mu0 = 1.25663e-6
+
+Mx_parser(x,y,z) = "8.e5 * (x>0.)*(x<4096.e-9)*(y>0.)*(y<4096.e-9)*(z>0.)*(z<4096.e-9)"
+My_parser(x,y,z) = "0."
+Mz_parser(x,y,z) = "0."
+
+# Field 1:  mu_0 Hx=-24.6 mT, mu_0 Hy= 4.3 mT, mu_0 Hz= 0.0 mT
+# which is a field approximately 25 mT, directed 170 degrees counterclockwise from the positive x axis
+timedependent_Hbias = 0
+Hx_bias_parser(x,y,z,t) = "1.e5"
+Hy_bias_parser(x,y,z,t) = "1.e5"
+Hz_bias_parser(x,y,z,t) = "1.e5"
+
+timedependent_alpha = 0
+alpha_parser(x,y,z,t) =    "(x>0.)*(x<4096.e-9)*(y>0.)*(y<4096.e-9)*(z>0.)*(z<4096.e-9) * 0.5"
+Ms_parser(x,y,z) =         "(x>0.)*(x<4096.e-9)*(y>0.)*(y<4096.e-9)*(z>0.)*(z<4096.e-9) * 8.e5"
+gamma_parser(x,y,z) =      "(x>0.)*(x<4096.e-9)*(y>0.)*(y<4096.e-9)*(z>0.)*(z<4096.e-9) * -1.759e11"
+exchange_parser(x,y,z) =   "(x>0.)*(x<4096.e-9)*(y>0.)*(y<4096.e-9)*(z>0.)*(z<4096.e-9) * 1.3e-11"
+anisotropy_parser(x,y,z) = "(x>0.)*(x<4096.e-9)*(y>0.)*(y<4096.e-9)*(z>0.)*(z<4096.e-9) * -139.26"
+DMI_parser(x,y,z) =        "(x>0.)*(x<4096.e-9)*(y>0.)*(y<4096.e-9)*(z>0.)*(z<4096.e-9) * -4.5e-3"
+
+precession = 1
+demag_coupling = 1
+FFT_solver = 1
+M_normalization = 1  # 0 = unsaturated case; 1 = saturated case
+exchange_coupling = 1
+anisotropy_coupling = 1
+anisotropy_axis = 0.0 0.0 1.0
+DMI_coupling = 1
+
+# INTEGRATION
+
+TimeIntegratorOption = 1 #Forward Euler
+
+integration.type = RungeKutta
+integration.rk.type = 1