# usage
# Make several versions, plus extract individual versions
# make clean; make
# Make and scalar CPU version
# make clean; make CPU=1
# ./bpc.exe
#
# Make and run mananged memory version for GPU
# make clean MANAGED=1 ; make MANAGED=1
# ./bpc_gpu_managed.exe
#
# Make and run OpenACC CPU version
# make clean MULTI=1 ; make MULTI=1
# ACC_NUM_CORES=20 ./bpc_multi.exe
#
# Make and run CPU/OpenMP version
# make clean CPU=1 ; make CPU=1
# OMP_NUM_THREADS=20 ./bpc_cpu.exe

FC=pgf90

# Default flags are the GPU with managed memory
MANAGED=1
TARGET=gpu
FMACPU=
ifdef MANAGED
  MANAGE=,managed -DMANAGED
endif

ifdef ALL
TARGET=*
endif


FLAGS = -gopt -Mpreprocess -fast -acc -Minfo=acc -ta=tesla,cc70$(MANAGE)
LFLAGS= -gopt -Mpreprocess -fast -acc -Minfo=acc -ta=tesla,cc70$(MANAGE)

ifdef CPU 
  FLAGS = -gopt -Mpreprocess -fast -mp=nonuma -Minfo=mp $(FMACPU)
  LFLAGS= -gopt -Mpreprocess -fast -mp=nonuma -Minfo=mp $(FMACPU)
  TARGET=cpu
endif

ifdef MULTI
  FLAGS= -gopt -O2 -Mpreprocess -fast -acc -Minfo=acc -ta=multicore
  FLAGS= -gopt -Mpreprocess -fast -acc -Minfo=acc -ta=multicore
  TARGET=multi
endif

ifdef DEBUG
  FLAGS = -O0 -g -C -Mchkstk -Ktrap=fp -Mchkfpstk -Mchkptr
  LFLAGS= -O0 -g -C -Mchkstk -Ktrap=fp -Mchkfpstk -Mchkptr
endif

# These flags will be system dependent
NETCDF=-I$(NETCDF_INC) -L$(NETCDF_LIB) -lnetcdff -lnetcdf

bpc_$(TARGET).exe: par_kind.o phycst.o len_oce.o in_out_man_cut.o eos_insitu.o zpshde.o dynhpgs.o zdfmxl.o ldfslp.o traldf_iso.o main_bpc_hpg.o
	$(FC) par_kind.o phycst.o len_oce.o in_out_man_cut.o eos_insitu.o zpshde.o dynhpgs.o zdfmxl.o ldfslp.o traldf_iso.o tiletype.o traldf_iso_tile.o main_bpc_hpg.o $(LFLAGS) $(NETCDF) -o bpc_$(TARGET).exe

par_kind.o: par_kind.F90
	$(FC) -c $(FLAGS) $(NETCDF) par_kind.F90

phycst.o: phycst.F90 par_kind.o
	$(FC) -c $(FLAGS) $(NETCDF) phycst.F90

len_oce.o: len_oce.F90
	$(FC) -c $(FLAGS) $(NETCDF) len_oce.F90

in_out_man_cut.o: in_out_man_cut.F90 par_kind.o
	$(FC) -c $(FLAGS) $(NETCDF) in_out_man_cut.F90

eos_insitu.o: eos_insitu.F90 phycst.o in_out_man_cut.o len_oce.o vectopt_loop_substitute.h90
	$(FC) -c $(FLAGS) $(NETCDF) eos_insitu.F90

zpshde.o: zpshde.F90 len_oce.o phycst.o in_out_man_cut.o eos_insitu.o
	$(FC) -c $(FLAGS) $(NETCDF) zpshde.F90

dynhpgs.o: dynhpgs.F90 par_kind.o len_oce.o in_out_man_cut.o vectopt_loop_substitute.h90
	$(FC) -c $(FLAGS) $(NETCDF) dynhpgs.F90

zdfmxl.o: zdfmxl.F90 in_out_man_cut.o phycst.o len_oce.o
	$(FC) -c $(FLAGS) $(NETCDF) zdfmxl.F90

ldfslp.o: ldfslp.F90 len_oce.o phycst.o in_out_man_cut.o
	$(FC) -c $(FLAGS) $(NETCDF) ldfslp.F90

tiletype.o: tiletype.F90
	$(FC) -c $(FLAGS) $(NETCDF) tiletype.F90

traldf_iso.o: traldf_iso.F90  len_oce.o phycst.o in_out_man_cut.o
	$(FC) -c $(FLAGS) $(NETCDF) traldf_iso.F90

traldf_iso_tile.o: traldf_iso_tile.F90  len_oce.o phycst.o in_out_man_cut.o tiletype.o
	$(FC) -c $(FLAGS) $(NETCDF) traldf_iso_tile.F90

main_bpc_hpg.o: main_bpc_hpg.F90 par_kind.o len_oce.o phycst.o in_out_man_cut.o eos_insitu.o zpshde.o dynhpgs.o zdfmxl.o ldfslp.o traldf_iso.o traldf_iso_tile.o tiletype.o
	$(FC) -c $(FLAGS) $(NETCDF) main_bpc_hpg.F90

clean:
	rm -f *.o *.mod bpc_$(TARGET).exe

bpc_all: bpc_OpenMP bpc_OpenACC_managed bpc_OpenACC

bpc_OpenMP:
	mkdir -p ../OpenMP
	for i in *90; do  grep -v ACC $$i > ../OpenMP/$$i ; done
	cp Makefile ../OpenMP
bpc_OpenACC_managed:
	mkdir -p ../OpenACC_managed
	for i in *90; do  grep -v OMP $$i > ../OpenACC_managed/$$i ; done
	cp Makefile ../OpenACC_managed
bpc_OpenACC:
	mkdir -p ../OpenACC
	for i in *90; do  grep -v OMP $$i > ../OpenACC/$$i ; done
	cp Makefile ../OpenACC