Made some changes to make it run with CUDA 4.2

Author: jirikraus

posts/cuda-aware-mpi-example/src/CUDA_Aware_MPI.c

@@ -23,7 +23,7 @@
 void SetDeviceBeforeInit()
 {
 	char * localRankStr = NULL;
-	int rank = 0, devCount = 0;	
+	int rank = 0, devCount = 0;
 
 	// We extract the local rank initialization using an environment variable
 	if ((localRankStr = getenv(ENV_LOCAL_RANK)) != NULL)

posts/cuda-aware-mpi-example/src/Host.c

@@ -261,7 +261,7 @@ void InitializeDataChunk(int topSizeY, int topIdxY, const int2 * domSize, const
 	SafeCudaCall(cudaMalloc((void **)devResidue, sizeof(real)));
 	SafeCudaCall(cudaMemcpy(devBlocks[0], hostBlock, blockBytes, cudaMemcpyHostToDevice));
 	SafeCudaCall(cudaMemcpy(devBlocks[1], devBlocks[0], blockBytes, cudaMemcpyDeviceToDevice));
-	SafeCudaCall(cudaStreamCreateWithFlags(copyStream, cudaStreamNonBlocking));			
+	SafeCudaCall(cudaStreamCreate(copyStream));			
 
 	SafeHostFree(hostBlock);
 }

posts/cuda-aware-mpi-example/src/Makefile

@@ -21,7 +21,7 @@ GENCODE_SM13    := -gencode arch=compute_13,code=sm_13
 GENCODE_SM20    := -gencode arch=compute_20,code=sm_20
 GENCODE_SM30    := -gencode arch=compute_30,code=sm_30
 GENCODE_SM35    := -gencode arch=compute_35,code=sm_35
-GENCODE_FLAGS   := $(GENCODE_SM13) $(GENCODE_SM20) $(GENCODE_SM30) $(GENCODE_SM35)
+GENCODE_FLAGS   := $(GENCODE_SM13) $(GENCODE_SM20)
 
 NVCCFLAGS=-O3 $(GENCODE_FLAGS) -Xcompiler -march=native
 
@@ -55,9 +55,11 @@ cuda_aware_mpi.o: Jacobi.h CUDA_Aware_MPI.c Makefile
 	$(MPICC) $(MPICFLAGS) $(CFLAGS) $(CUDACFLAGS) -c CUDA_Aware_MPI.c -o cuda_aware_mpi.o
 
 $(JACOBI_CUDA_NORMAL_MPI): jacobi.o input.o host.o device.o cuda_normal_mpi.o Makefile
+	mkdir -p $(BINDIR)
 	$(MPICC) $(CUDALDFLAGS) -o $(JACOBI_CUDA_NORMAL_MPI) jacobi.o input.o host.o device.o  cuda_normal_mpi.o  
 
 $(JACOBI_CUDA_AWARE_MPI): jacobi.o input.o host.o device.o cuda_aware_mpi.o Makefile
+	mkdir -p $(BINDIR)
 	$(MPICC) $(CUDALDFLAGS) -o $(JACOBI_CUDA_AWARE_MPI) jacobi.o input.o host.o device.o cuda_aware_mpi.o  
 
 doc: CUDA_Normal_MPI.c CUDA_Aware_MPI.c Device.cu Host.c Input.c Makefile Jacobi.c Jacobi.h Jacobi.doxygen

posts/cuda-aware-mpi-example/src/README

@@ -15,8 +15,9 @@ Description document for the GPU-based Jacobi solver
 Contents:
 ---------
 1)	Application overview
-2)  Build instructions
-3)	Documentation
+2)	Build instructions
+3)	Run instructions
+4)	Documentation
 
 
 =======================
@@ -56,8 +57,6 @@ c)	-fs 		-	optional argument indicating that the replacement of the old block wi
 					a fast pointer swap rather than a full block copy (which is the default behavior)
 d)	-h | --help	-	optional argument for printing help information; this overrides all other arguments
 
-
-
 =====================
 2) Build instructions
 =====================
@@ -72,11 +71,22 @@ You can build the CUDA-aware MPI an the normal MPI version by calling make in sr
 cd src
 make
 
-To find mpi.h and the CUDA runtime library the provied Makefile relies on CUDA_INSTALL_PATH and MPI_HOME beeing set correctly.
+To find mpi.h and the CUDA runtime library the provied Makefile relies on CUDA_INSTALL_PATH and MPI_HOME beeing set correctly. If you are using CUDA 5 or newer and running on a device with compute capability 3.0 or 3.5 you should also add GENCODE_SM30 and GENCODE_SM35 to the GENCODE_FLAGS in the Makefile. Also the macro ENV_LOCAL_RANK might need to
+be changed in Jacobi.h to handle the GPU affinit properly. It defaults to MV2_COMM_WORLD_LOCAL_RANK which works with MVAPICH2.
 The generated binaries can then be found found the bin directory.
+
+===================
+3) Run instructions
+===================
+
+To run the normal MPI version use: 
+mpiexec -np 2 ./jacobi_cuda_normal_mpi -t 2 1
+
+To run the CUDA-aware MPI version depending on the MPI implementation you are using you need to activate the CUDA-aware feature, e.g. for MVAPICH2 use
+MV2_USE_CUDA=1 mpiexec -np 2 --exports=MV2_USE_CUDA ./jacobi_cuda_aware_mpi -t 2 1
 
 ================
-3) Documentation
+4) Documentation
 ================
 
 Documentation for this project may be generated automatically using Doxygen by calling make doc. A configuration file for this may be found in the "doc" folder. If