From 5420c67114a66f3f4ad627632d91c5f02534cc25 Mon Sep 17 00:00:00 2001
From: Tony Cook <tony@develop-help.com>
Date: Thu, 7 Jul 2016 14:50:52 +1000
Subject: [PATCH] Increase parallelization for GNU make builds

The main improvement XS extensions are now built in parallel, non-XS
extensions are also built in parallel, but this is a much smaller
win.

- for gmake/dmake builds, put lib back at the end of @INC, since that
  change depended on builds not being parallel on Win32, but they are
  now.  Without the change I had random build failures, one case
  tracked down to make_ext.pl trying to load a partly written
  lib/File/Find.pm.

- fixes GNU make builds with a static Unicode::Normalize, previously
  this would fail because we always built a dynamic Unicode::Normalize.
---
 MANIFEST                |  1 +
 make_ext.pl             | 58 ++++++++++++++++--------------
 win32/.gitignore        |  1 +
 win32/GNUmakefile       | 23 ++++++------
 win32/Makefile          |  2 +-
 win32/mkexts.pl         | 95 +++++++++++++++++++++++++++++++++++++++++++++++++
 write_buildcustomize.pl | 11 ++++--
 7 files changed, 149 insertions(+), 42 deletions(-)
 create mode 100644 win32/mkexts.pl

diff --git a/MANIFEST b/MANIFEST
index 3240e37..07f7f9e 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -5790,6 +5790,7 @@ win32/list_static_libs.pl	prints libraries for static linking
 win32/Makefile			Win32 makefile for NMAKE (Visual C++ build)
 win32/Makefile.ce		WinCE port
 win32/makefile.mk		Win32 makefile for DMAKE (BC++, VC++ builds)
+win32/mkexts.pl			Builds exts.mk
 win32/perlexe.ico		perlexe.ico image file
 win32/perlexe.manifest		Assembly manifest file
 win32/perlexe.rc		associated perl binary with icon
diff --git a/make_ext.pl b/make_ext.pl
index 8270092..27c2ded 100644
--- a/make_ext.pl
+++ b/make_ext.pl
@@ -84,6 +84,7 @@ my $dynaloader = $opts{dynaloader} || $opts{all};
 # canonise into X/Y form (pname)
 
 foreach (@extspec) {
+    s(^\.\./)() if IS_WIN32;
     if (s{^lib/auto/}{}) {
 	# Remove lib/auto prefix and /*.* suffix
 	s{/[^/]+\.[^/]+$}{};
@@ -141,7 +142,6 @@ my %extra_passthrough;
 
 if (IS_WIN32) {
     require Cwd;
-    require FindExt;
     my $build = Cwd::getcwd();
     $perl = $^X;
     if ($perl =~ m#^\.\.#) {
@@ -160,35 +160,39 @@ if (IS_WIN32) {
     }
 
     print "In $build" if $verbose;
-    foreach my $dir (@dirs) {
-	chdir($dir) or die "Cannot cd to $dir: $!\n";
-	(my $ext = Cwd::getcwd()) =~ s{/}{\\}g;
-	FindExt::scan_ext($ext);
-	FindExt::set_static_extensions(split ' ', $Config{static_ext});
-	chdir $build
-	    or die "Couldn't chdir to '$build': $!"; # restore our start directory
-    }
+    if ($static || $dynamic || $nonxs) {
+        require FindExt;
+        foreach my $dir (@dirs) {
+            chdir($dir) or die "Cannot cd to $dir: $!\n";
+            (my $ext = Cwd::getcwd()) =~ s{/}{\\}g;
+            FindExt::scan_ext($ext);
+            FindExt::set_static_extensions(split ' ', $Config{static_ext});
+            chdir $build
+              or die "Couldn't chdir to '$build': $!"; # restore our start directory
+        }
 
-    my @ext;
-    push @ext, FindExt::static_ext() if $static;
-    push @ext, FindExt::dynamic_ext() if $dynamic;
-    push @ext, FindExt::nonxs_ext() if $nonxs;
-    push @ext, 'DynaLoader' if $dynaloader;
+        my @ext;
+        push @ext, FindExt::static_ext() if $static;
+        push @ext, FindExt::dynamic_ext() if $dynamic;
+        push @ext, FindExt::nonxs_ext() if $nonxs;
+        push @ext, 'DynaLoader' if $dynaloader;
 
-    foreach (sort @ext) {
-	if (%incl and !exists $incl{$_}) {
-	    #warn "Skipping extension $_, not in inclusion list\n";
-	    next;
-	}
-	if (exists $excl{$_}) {
-	    warn "Skipping extension $_, not ported to current platform";
-	    next;
-	}
-	push @extspec, $_;
-	if($_ ne 'DynaLoader' && FindExt::is_static($_)) {
-	    push @{$extra_passthrough{$_}}, 'LINKTYPE=static';
-	}
+        foreach (sort @ext) {
+            if (%incl and !exists $incl{$_}) {
+                #warn "Skipping extension $_, not in inclusion list\n";
+                next;
+            }
+            if (exists $excl{$_}) {
+                warn "Skipping extension $_, not ported to current platform";
+                next;
+            }
+            push @extspec, $_;
+            if($_ ne 'DynaLoader' && FindExt::is_static($_)) {
+                push @{$extra_passthrough{$_}}, 'LINKTYPE=static';
+            }
+        }
     }
+    push @extspec, "DynaLoader" if $dynaloader;
 
     chdir '..'
 	or die "Couldn't chdir to build directory: $!"; # now in the Perl build
diff --git a/win32/.gitignore b/win32/.gitignore
index 56dee9f..96a5415 100644
--- a/win32/.gitignore
+++ b/win32/.gitignore
@@ -5,6 +5,7 @@
 bin/*.bat
 html/
 mini/
+exts.mk
 Extensions_static
 .coreheaders
 dlutils.c
diff --git a/win32/GNUmakefile b/win32/GNUmakefile
index 0fc3e58..a1bb7a1 100644
--- a/win32/GNUmakefile
+++ b/win32/GNUmakefile
@@ -746,10 +746,6 @@ BLINK_FLAGS	= $(PRIV_LINK_FLAGS) $(LINK_FLAGS)
 #################### do not edit below this line #######################
 ############# NO USER-SERVICEABLE PARTS BEYOND THIS POINT ##############
 
-#prevent -j from reaching EUMM/make_ext.pl/"sub makes", Win32 EUMM not parallel
-#compatible yet
-unexport MAKEFLAGS
-
 a ?= .lib
 
 .SUFFIXES : .c .i $(o) .dll $(a) .exe .rc .res
@@ -1463,21 +1459,24 @@ endif
 # DynaLoader.pm, so this will have to do
 
 #most of deps of this target are in DYNALOADER and therefore omitted here
-Extensions : $(PERLDEP) $(DYNALOADER) $(GLOBEXE) MakePPPort
-	$(MINIPERL) -I..\lib ..\make_ext.pl "MAKE=$(PLMAKE)" --dir=$(CPANDIR) --dir=$(DISTDIR) --dir=$(EXTDIR) --dynamic !Unicode/Normalize
+Extensions : $(PERLDEP) $(DYNALOADER) $(GLOBEXE) MakePPPort exts.mk
+	$(MAKE) -f exts.mk Extensions
 
-Extensions_normalize : $(PERLDEP) $(DYNALOADER) $(GLOBEXE) $(UNIDATAFILES)
-	$(MINIPERL) -I..\lib ..\make_ext.pl "MAKE=$(PLMAKE)" --dir=$(CPANDIR) --dir=$(DISTDIR) --dir=$(EXTDIR) --dynamic +Unicode/Normalize
+Extensions_normalize : $(PERLDEP) $(DYNALOADER) $(GLOBEXE) $(UNIDATAFILES) exts.mk
+	$(MAKE) -f exts.mk Extensions_normalize
 
 Extensions_reonly : $(PERLDEP) $(DYNALOADER)
 	$(MINIPERL) -I..\lib ..\make_ext.pl "MAKE=$(PLMAKE)" --dir=$(CPANDIR) --dir=$(DISTDIR) --dir=$(EXTDIR) --dynamic +re
 
-Extensions_static : ..\make_ext.pl list_static_libs.pl $(CONFIGPM) $(GLOBEXE) $(HAVE_COREDIR) MakePPPort
-	$(MINIPERL) -I..\lib ..\make_ext.pl "MAKE=$(PLMAKE)" --dir=$(CPANDIR) --dir=$(DISTDIR) --dir=$(EXTDIR) --static
+Extensions_static : ..\make_ext.pl list_static_libs.pl $(CONFIGPM) $(GLOBEXE) $(HAVE_COREDIR) MakePPPort exts.mk
+	$(MAKE) -f exts.mk Extensions_static
 	$(MINIPERL) -I..\lib list_static_libs.pl > Extensions_static
 
-Extensions_nonxs : ..\make_ext.pl ..\pod\perlfunc.pod $(CONFIGPM) $(GLOBEXE)
-	$(MINIPERL) -I..\lib ..\make_ext.pl "MAKE=$(PLMAKE)" --dir=$(CPANDIR) --dir=$(DISTDIR) --dir=$(EXTDIR) --nonxs !libs
+exts.mk : $(CONFIGPM) FindExt.pm $(HAVEMINIPERL) mkexts.pl $(HAVE_COREDIR)
+	$(MINIPERL) -I..\lib mkexts.pl MINIPERL=$(MINIPERL) CPANDIR=$(CPANDIR) DISTDIR=$(DISTDIR) EXTDIR=$(EXTDIR)
+
+Extensions_nonxs : ..\make_ext.pl ..\pod\perlfunc.pod $(CONFIGPM) $(GLOBEXE) exts.mk
+	$(MAKE) -f exts.mk Extensions_nonxs
 
 #lib must be built, it can't be buildcustomize.pl-ed, and is required for XS building
 $(DYNALOADER) : ..\make_ext.pl $(CONFIGPM) $(HAVE_COREDIR)
diff --git a/win32/Makefile b/win32/Makefile
index 0092dda..edd1158 100644
--- a/win32/Makefile
+++ b/win32/Makefile
@@ -1046,7 +1046,7 @@ $(MINIPERL) : ..\lib\buildcustomize.pl
 	$(BLINK_FLAGS) $(DELAYLOAD) $(MINIDELAYLOAD) $(LIBFILES) $(MINI_OBJ)
 <<
 	$(EMBED_EXE_MANI:..\lib\buildcustomize.pl=..\miniperl.exe)
-	$(MINIPERL) -I..\lib -f ..\write_buildcustomize.pl ..
+	$(MINIPERL) -I..\lib -f ..\write_buildcustomize.pl --single ..
 
 $(MINIDIR) :
 	if not exist "$(MINIDIR)" mkdir "$(MINIDIR)"
diff --git a/win32/mkexts.pl b/win32/mkexts.pl
new file mode 100644
index 0000000..22b9351
--- /dev/null
+++ b/win32/mkexts.pl
@@ -0,0 +1,95 @@
+use strict;
+use FindExt;
+use Config;
+
+open my $mk, ">", "exts.tmp"
+  or die "Cannot create exts.tmp: $!";
+
+my @ext_dirs = qw(../ext ../dist ../cpan);
+FindExt::scan_ext($_) for @ext_dirs;
+FindExt::set_static_extensions(split ' ', $Config{static_ext});
+
+my %dynamic_ext = map {
+    $_ => "../lib/auto/$_" =~ s((\w+)$)($1/$1.dll)ra;
+} FindExt::dynamic_ext();
+my %nonxs_ext;
+# lib is built when DynaLoader is built
+for my $ext (grep $_ ne "lib", FindExt::nonxs_ext()) {
+    my $subdir = join "-", split m(/), $ext;
+    my ($dir) = grep -d "$_/$subdir", @ext_dirs;
+    $dir
+      or die "Cannot find non-xs extension $ext";
+    $nonxs_ext{$ext} = "$dir/$subdir/pm_to_blib";
+}
+my %static_ext = map {
+    $_ => "../lib/auto/$_" =~ s!(\w+)$!$1/$1\$(LIB_EXT)!ra;
+} FindExt::static_ext();
+
+my @normalize_dyn = grep defined, delete $dynamic_ext{"Unicode/Normalize"};
+
+# used for dependency calculations
+my %all = ( %nonxs_ext, %dynamic_ext, %static_ext );
+
+my @nonxs_ext = sort values %nonxs_ext;
+my @static_ext = reorder(sort values %static_ext);
+my @dynamic_ext = reorder(sort values %dynamic_ext);
+
+print $mk <<EXPAND;
+LIB_EXT=$Config{_a}
+static_ext=@static_ext
+nonxs_ext=@nonxs_ext
+dynamic_ext=@dynamic_ext
+normalize_dynamic=@normalize_dyn
+EXPAND
+
+for my $def (grep /=/, @ARGV) {
+    print $mk "$def\n";
+}
+
+print $mk <<'!NO!SUBS!';
+
+SHELL := cmd.exe
+
+Extensions_nonxs : $(nonxs_ext)
+
+Extensions : $(dynamic_ext)
+
+Extensions_static : $(static_ext)
+
+Extensions_normalize : $(normalize_dynamic)
+
+d_dummy $(dynamic_ext) $(normalize_dynamic) :
+	$(MINIPERL) -I..\lib ..\make_ext.pl $@ $(MAKE_EXT_ARGS) MAKE="$(MAKE) -j1" LINKTYPE=dynamic
+
+s_dummy $(static_ext) :
+	$(MINIPERL) -I..\lib ..\make_ext.pl $@ $(MAKE_EXT_ARGS) MAKE="$(MAKE) -j1" LINKTYPE=static
+
+n_dummy $(nonxs_ext) :
+	$(MINIPERL) -I..\lib ..\make_ext.pl $@ $(MAKE_EXT_ARGS) MAKE="$(MAKE) -j1"
+
+!NO!SUBS!
+
+my %deps =
+  (
+   "Pod/Functions" => [ "Pod/Simple", "Pod/Escapes" ],
+   "Math/BigInt/FastCalc" => [ "List/Util" ]
+  );
+for my $dep (sort grep $all{$_}, keys %deps) {
+    my @in = @{$deps{$dep}};
+    print $mk "$all{$dep} : ", join(" ", map $all{$_}, @in), "\n\n";
+}
+
+close $mk;
+
+unlink "exts.mk";
+rename "exts.tmp", "exts.mk"
+  or die "Cannot rename exts.tmp to exts.mk: $!";
+
+# put some larger modules at the front of the list
+# gmake will tend to build them sooner, so we don't have eg Encode
+# start building just as everything else finishes
+sub reorder {
+    my @big = grep /\b(?:Encode|re)\b/, @_;
+    my @other = grep !/\b(?:Encode|re)\b/, @_;
+    return ( @big, @other );
+}
diff --git a/write_buildcustomize.pl b/write_buildcustomize.pl
index 8666a6b..f7588bc 100644
--- a/write_buildcustomize.pl
+++ b/write_buildcustomize.pl
@@ -5,6 +5,14 @@ use strict;
 my $osname = $^O;
 my $file = 'lib/buildcustomize.pl';
 
+# for non-parallel builds, like with nmake on Win32, there's
+# no change of a race on "toolchain" modules, so put lib first
+my $single;
+if (@ARGV && $ARGV[0] eq "--single") {
+    shift;
+    ++$single;
+}
+
 if ( @ARGV % 2 ) {
     my $dir = shift;
     chdir $dir or die "Can't chdir '$dir': $!";
@@ -66,8 +74,7 @@ my $cwd  = Cwd::getcwd();
 my $inc = join ",\n        ",
     map { "q\0$_\0" }
     (map {File::Spec::Functions::rel2abs($_, $cwd)} (
-# faster build on the non-parallel Win32 build process
-        $^O eq 'MSWin32' ? ('lib', @toolchain ) : (@toolchain, 'lib')
+        $single ? ( "lib", @toolchain) : ( @toolchain, 'lib' )
     ));
 
 open my $fh, '>', $file
-- 
2.7.0.windows.1