Optimize build for parallel make. Move away from "all-recursive" cons…

…truct

in favor of a "subdirectories" construct without a loop.

$(SUBDIRS):
  $(MAKE) --directory=$@

This allows make to parallelize building files from different directories
at the same time. Up to now, make only built files from each directory in
parallel, resulting in the "operator" directory taking a long time to build
with highly optimizing compilers.

In addition, shuffle build order to encourage modules that take a long time
to compile to start early in the compilation process.

Makefile.global

@@ -23,23 +23,13 @@ $(abs_top_builddir)/config.h: $(top_srcdir)/config.h.in $(abs_top_builddir)/conf
 $(top_srcdir)/configure: $(top_srcdir)/configure.in 
 	-( cd $(top_srcdir) && $(AUTOCONF) )
 
-#extern modules
-
-$(addsuffix .o,$(EXTERNMODULES)): %.o:
-	( cd $(dir $@) && ${MAKE} $(notdir $@) )
-
-#extern libs
-
-$(LINKLIBS): %.a: Makefile $(abs_top_builddir)/config.status $(top_srcdir)/configure
-	@( cd $(dir $@) && ${MAKE} $(notdir $@) )
-
 #dep rules
 
 # PROGRAMS_WITH_GIT_HASH require git_hash.h which is dynamically built by a phony make target
 # to prevent too frequent building of git_hash (slowing down the build)
 # we filter the list of all objects and treat these separately
 $(addsuffix .d, $(filter-out ${PROGRAMS_WITH_GIT_HASH},${ALLOBJ})): %.d: ${srcdir}/%.c Makefile
-	@ $(CCDEP) ${DEPFLAGS} ${CPPFLAGS} ${INCLUDES} ${DEFS} $< > $@
+	@ $(CCDEP) ${DEPFLAGS} ${DEFS} ${INCLUDES} $< > $@
 
 # dirty hack to prevent make from entering an infinite loop because a phony target is given as a real
 # dependency (make will build invert.d and hmc_tm.d indefinitely)
@@ -49,13 +39,12 @@ $(addsuffix .d, $(filter-out ${PROGRAMS_WITH_GIT_HASH},${ALLOBJ})): %.d: ${srcdi
 # irrelevant because it will be rebuilt during the compilation of either invert or hmc_tm
 ifneq (git_hash.h, $(findstring git_hash.h,$(wildcard $(top_srcdir)/git_has*.h)))
 $(addsuffix .d, $(filter ${PROGRAMS_WITH_GIT_HASH},${ALLOBJ})): %.d: ${srcdir}/%.c ${top_srcdir}/git_hash.h Makefile
-	@ $(CCDEP) ${DEPFLAGS} ${CPPFLAGS} ${INCLUDES} ${DEFS} $< > $@
+	@ $(CCDEP) ${DEPFLAGS} ${DEFS} ${INCLUDES} $< > $@
 else
 $(addsuffix .d, $(filter ${PROGRAMS_WITH_GIT_HASH},${ALLOBJ})): %.d: ${srcdir}/%.c Makefile
-	@ $(CCDEP) ${DEPFLAGS} ${CPPFLAGS} ${INCLUDES} ${DEFS} $< > $@
+	@ $(CCDEP) ${DEPFLAGS} ${DEFS} ${INCLUDES} $< > $@
 endif
 
-
 ${top_builddir}/fixed_volume.h: ${top_srcdir}/fixed_volume.h.in ${top_builddir}/config.status
 	cd ${abs_top_builddir} && CONFIG_FILES=fixed_volume.h CONFIG_HEADERS= $(SHELL) ${top_builddir}/config.status
 

Makefile.in

@@ -31,12 +31,8 @@ USESUBDIRS = @USESUBDIRS@
 NVCC = @NVCC@
 GPUMPICOMPILER = @GPUMPICOMPILER@
 
-
 INCLUDES = @INCLUDES@
 LINK = $(CCLD) -o $@ ${LDFLAGS}
-LINKLIBS = ${top_builddir}/linalg/liblinalg.a  \
-	${top_builddir}/solver/libsolver.a ${top_builddir}/io/libio.a \
-	${top_builddir}/buffers/libbuffers.a $(top_builddir)/cu/libcu.a
 
 COMPILE = ${CC} ${DEFS} ${INCLUDES} -o $@ ${CFLAGS}
 
@@ -66,7 +62,6 @@ GPUOBJECTS := $(patsubst $(srcdir)/$(GPUDIR)/%.cu, $(GPUDIR)/%.o, $(GPUSOURCES))
 #GPUSOURCES_C := $(wildcard $(srcdir)/$(GPUDIR)/*.c)
 #GPUOBJECTS_C := $(patsubst $(srcdir)/$(GPUDIR)/%.c, $(GPUDIR)/%.o, $(GPUSOURCES_C))
 
-
 NOOPTMOD = test/check_xchange test/check_geometry
 
 PROGRAMS = hmc_tm benchmark invert gen_sources  \
@@ -78,18 +73,15 @@ SUBDIRS = ${USESUBDIRS}
 # delete the default suffix rules
 .SUFFIXES:
 
-all: Makefile all-recursive dep hmc_tm invert
+all: Makefile dep $(SUBDIRS) hmc_tm invert benchmark
 
-#ifneq (,$(findstring lapack,${LIBS}))
-#all: Makefile all-recursive dep hmc_tm invert
-#else
-#all: Makefile all-recursive dep hmc_tm invert
-#endif
+$(SUBDIRS):
+	$(MAKE) --directory=$@
 
 # run the GIT-VERSION-GEN script to generate version information in git_hash.h
 # making sure that we run in the correct directory
 ${top_srcdir}/git_hash.h:
-	@cd @srcdir@ && sh GIT-VERSION-GEN
+	@ ( cd @srcdir@ && sh GIT-VERSION-GEN )
 
 -include $(addsuffix .d,$(ALLOBJ))
 
@@ -106,9 +98,6 @@ libhmc.a: ${addsuffix .o, ${MODULES} ${SMODULES}} Makefile
 	@$(RANLIB) libhmc.a
 	@cp libhmc.a ${top_builddir}/lib/libhmc.a
 
-#${addsuffix .o, ${ALLOBJ}}: %.o: ${srcdir}/%.c %.d Makefile $(abs_top_builddir)/config.h
-#	${COMPILE} ${OPTARGS} -c $<
-
 $(addsuffix .o,$(filter-out ${NOOPTMOD},${MODULES})): %.o: ${srcdir}/%.c %.d Makefile $(abs_top_builddir)/config.h
 	${COMPILE} ${OPTARGS} -c $<
 
@@ -122,7 +111,7 @@ ${addsuffix .o, ${SMODULES}}: %.o: ${srcdir}/%.c %.d Makefile $(abs_top_builddir
 ${addsuffix .o, ${PROGRAMS}}: %.o: ${srcdir}/%.c %.d Makefile $(abs_top_builddir)/config.h ${top_srcdir}/git_hash.h
 	${COMPILE} ${OPTARGS} -c $<
 
-${PROGRAMS}: %: %.o libhmc.a all-recursive
+${PROGRAMS}: %: %.o libhmc.a $(SUBDIRS)
 	 ${LINK} $@.o $(GPUOBJECTS) $(GPUOBJECTS_C) $(LIBS)
 
 # The rules for unit tests are kept in a separate file for tidyness
@@ -152,15 +141,13 @@ compile-clean: compile-clean-recursive Makefile
 	rm -f *.o *.d test/*.o test/*.d tests/*.o tests/*.d
 
 clean: clean-recursive Makefile
-	rm -f hmc_tm invert *.o *.d test/*.o test/*.d tests/*.o tests/*.d
+	rm -f benchmark hmc_tm invert *.o *.d test/*.o test/*.d tests/*.o tests/*.d
 
 distclean: distclean-recursive Makefile
-	rm -f hmc_tm hybrid *.o *.d *~ Makefile config.log config.status fixed_volume.h
+	rm -f benchmark hmc_tm invert *.o *.d *~ Makefile config.log config.status fixed_volume.h
 	rm -f config.h
 
-.PHONY: all ${top_srcdir}/git_hash.h clean compile-clean distclean dep install \
-	$(PROGRAMS) all-recursive \
-	all-debug-recursive all-profile-recursive \
+.PHONY: all ${SUBDIRS} ${top_srcdir}/git_hash.h clean compile-clean distclean dep install \
+	all-recursive all-debug-recursive all-profile-recursive \
 	clean-recursive distclean-recursive \
-	compile-clean-recursive $(LINKLIBS) \
-	tests libhmc.a
+	compile-clean-recursive

configure.in

@@ -40,7 +40,8 @@ AC_CHECK_PROG(CCDEP, gcc, "gcc", "$CC")
 LDFLAGS="$LDFLAGS -L\${HOME}/lib -L\${top_builddir}/lib"
 CCLD=${CC}
 
-USESUBDIRS="buffers cu io solver linalg monomial xchange operator init"
+# compilation in operator is slowest so we do it first, saves time in parallel compiles
+USESUBDIRS="operator linalg solver monomial buffers cu io xchange init"
 
 AC_CHECK_HEADERS([stdint.h],
 [ dnl for inttypes.h and stdint.h for uint_xxx types

operator/Makefile.in

@@ -30,8 +30,8 @@ LDADD =
 COMPILE = ${CC} $(DEFS) ${INCLUDES} ${CFLAGS}
 
 LIBRARIES = liboperator
-liboperator_TARGETS = clovertm_operators clover_leaf tm_operators_nd clover_term clover_invert \
-	clover_deriv clover_accumulate_deriv clover_det
+liboperator_TARGETS = clover_accumulate_deriv clover_deriv clovertm_operators clover_leaf \
+  tm_operators_nd clover_term clover_invert clover_det
 
 liboperator_STARGETS = Hopping_Matrix_nocom tm_times_Hopping_Matrix Hopping_Matrix \
 	tm_operators tm_sub_Hopping_Matrix D_psi Dov_psi Dov_proj
@@ -51,37 +51,30 @@ debug all-debug: all
 profile all-profile: CFLAGS := $(filter-out -fomit-frame-pointer,${CFLAGS}) @PROFILE_FLAG@
 profile all-profile: all
 
-
 #include dep rules
-
 -include $(addsuffix .d,${liboperator_TARGETS})
 
 include ${top_srcdir}/Makefile.global
 
 # rule to compile objects
-
 ${liboperator_OBJECTS}: %.o: ${srcdir}/%.c %.d Makefile ${abs_top_builddir}/config.h
 	$(COMPILE) ${OPTARGS} -c $<
 
 ${liboperator_SOBJECTS}: %.o: ${srcdir}/%.c %.d Makefile ${abs_top_builddir}/config.h
 	$(COMPILE) ${SOPTARGS} -c $<
 
 # rule to make liboperator
-
 liboperator.a: ${liboperator_OBJECTS} ${liboperator_SOBJECTS} Makefile
 	@rm -f liboperator.a
 	@${AR} cru liboperator.a ${liboperator_OBJECTS} ${liboperator_SOBJECTS}
 	@$(RANLIB) liboperator.a
 	@cp liboperator.a ../lib/liboperator.a
 
 # rule to generate .d files
-
 $(addsuffix .d, $(liboperator_TARGETS) ${liboperator_STARGETS}): %.d: ${srcdir}/%.c Makefile
 	@${CCDEP} ${DEFS} ${DEPFLAGS} ${INCLUDES} $< > $@
 
-
 # rule to make dependencies
-
 dep: ${addsuffix .d, ${liboperator_TARGETS} ${liboperator_STARGETS}}
 
 # rules to clean