# usage
# Make several versions, plus extract individual versions
# make clean; make
# Make and scalar GPU version
# make clean GPU=1 ; make GPU=1
# ./bpc_gpu.exe
#
# Make and run mananged memory version for GPU
# make clean MANAGED=1 ; make MANAGED=1
# ./bpc_gpu_managed.exe
#
# Make and run OpenACC CPU version
# make clean MULTI=1 ; make MULTI=1
# ACC_NUM_CORES=20 ./bpc_multi.exe
#
# Make and run CPU/OpenMP version
# make clean CPU=1 ; make CPU=1
# OMP_NUM_THREADS=20 ./bpc_cpu.exe

FC=pgf90

# Default flags are the GPU with managed memory
MANAGED=1
TARGET=managed
FMACPU=

ifdef MANAGED
  MANAGE=,managed -DMANAGED
  TARGET=managed
endif

ifdef GPU 
  TARGET=gpu
  MANAGE=
endif

ifdef ALL
TARGET=*
endif

FLAGS = -gopt -Mpreprocess -fast -acc -Minfo=acc -ta=tesla,cc70$(MANAGE)
LFLAGS= -gopt -Mpreprocess -fast -acc -Minfo=acc -ta=tesla,cc70$(MANAGE)

ifdef CPU 
  FLAGS = -gopt -Mpreprocess -fast -mp=nonuma -Minfo=mp $(FMACPU)
  LFLAGS= -gopt -Mpreprocess -fast -mp=nonuma -Minfo=mp $(FMACPU)
  TARGET=cpu
  MANAGE=
endif

ifdef MULTI
  FLAGS= -gopt -O2 -Mpreprocess -fast -acc -Minfo=acc -ta=multicore
  FLAGS= -gopt -Mpreprocess -fast -acc -Minfo=acc -ta=multicore
  TARGET=multi
  MANAGE=
endif

ifdef DEBUG
  FLAGS = -O0 -g -C -Mchkstk -Ktrap=fp -Mchkfpstk -Mchkptr
  LFLAGS= -O0 -g -C -Mchkstk -Ktrap=fp -Mchkfpstk -Mchkptr
endif

# These flags will be system dependent
NETCDF=-I$(NETCDF_INC) -L$(NETCDF_LIB) -lnetcdff -lnetcdf
NETCDF=-I$(CPATH) -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/pgi-19.1/netcdf-4.6.1-snrb2ywx62gybndihod5cjn2cgqv6g5b/lib -L/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20180914/linux-rhel7-ppc64le/pgi-19.1/netcdf-fortran-4.4.4-3pcdzmpp5luuvp72ddzuc24dldq45pfw/lib -lnetcdff -lnetcdf

bpc_$(TARGET).exe: par_kind.o phycst.o len_oce.o in_out_man_cut.o eos_insitu.o zpshde.o dynhpgs.o zdfmxl.o ldfslp.o traldf_iso.o main_bpc_hpg.o
	$(FC) par_kind.o phycst.o len_oce.o in_out_man_cut.o eos_insitu.o zpshde.o dynhpgs.o zdfmxl.o ldfslp.o traldf_iso.o tiletype.o traldf_iso_tile.o main_bpc_hpg.o $(LFLAGS) $(NETCDF) -o bpc_$(TARGET).exe

par_kind.o: par_kind.F90
	$(FC) -c $(FLAGS) $(NETCDF) par_kind.F90

phycst.o: phycst.F90 par_kind.o
	$(FC) -c $(FLAGS) $(NETCDF) phycst.F90

len_oce.o: len_oce.F90
	$(FC) -c $(FLAGS) $(NETCDF) len_oce.F90

in_out_man_cut.o: in_out_man_cut.F90 par_kind.o
	$(FC) -c $(FLAGS) $(NETCDF) in_out_man_cut.F90

eos_insitu.o: eos_insitu.F90 phycst.o in_out_man_cut.o len_oce.o vectopt_loop_substitute.h90
	$(FC) -c $(FLAGS) $(NETCDF) eos_insitu.F90

zpshde.o: zpshde.F90 len_oce.o phycst.o in_out_man_cut.o eos_insitu.o
	$(FC) -c $(FLAGS) $(NETCDF) zpshde.F90

dynhpgs.o: dynhpgs.F90 par_kind.o len_oce.o in_out_man_cut.o vectopt_loop_substitute.h90
	$(FC) -c $(FLAGS) $(NETCDF) dynhpgs.F90

zdfmxl.o: zdfmxl.F90 in_out_man_cut.o phycst.o len_oce.o
	$(FC) -c $(FLAGS) $(NETCDF) zdfmxl.F90

ldfslp.o: ldfslp.F90 len_oce.o phycst.o in_out_man_cut.o
	$(FC) -c $(FLAGS) $(NETCDF) ldfslp.F90

tiletype.o: tiletype.F90
	$(FC) -c $(FLAGS) $(NETCDF) tiletype.F90

traldf_iso.o: traldf_iso.F90  len_oce.o phycst.o in_out_man_cut.o
	$(FC) -c $(FLAGS) $(NETCDF) traldf_iso.F90

traldf_iso_tile.o: traldf_iso_tile.F90  len_oce.o phycst.o in_out_man_cut.o tiletype.o
	$(FC) -c $(FLAGS) $(NETCDF) traldf_iso_tile.F90

main_bpc_hpg.o: main_bpc_hpg.F90 par_kind.o len_oce.o phycst.o in_out_man_cut.o eos_insitu.o zpshde.o dynhpgs.o zdfmxl.o ldfslp.o traldf_iso.o traldf_iso_tile.o tiletype.o
	$(FC) -c $(FLAGS) $(NETCDF) main_bpc_hpg.F90

clean:
	rm -f *.o *.mod bpc_$(TARGET).exe

bpc_all: bpc_OpenMP bpc_OpenACC_managed bpc_OpenACC

bpc_OpenMP:
	mkdir -p ../OpenMP
	for i in *90; do  grep -v ACC $$i > ../OpenMP/$$i ; done
	cp Makefile ../OpenMP
bpc_OpenACC_managed:
	mkdir -p ../OpenACC_managed
	for i in *90; do  grep -v OMP $$i > ../OpenACC_managed/$$i ; done
	cp Makefile ../OpenACC_managed
bpc_OpenACC:
	mkdir -p ../OpenACC
	for i in *90; do  grep -v OMP $$i > ../OpenACC/$$i ; done
	cp Makefile ../OpenACC