[PATCH] make_ext.pl: run all Makefile.PLes in 1 process #15054
Comments
From @bulk88Created by @bulk88..\miniperl.exe -I..\lib ..\make_ext.pl "MAKE=dmake" --dir=..\cpan before Version Number: Windows NT 5.2 (Build 3790) Version Number: Windows NT 5.2 (Build 3790) Version Number: Windows NT 5.2 (Build 3790) after Version Number: Windows NT 5.2 (Build 3790) Version Number: Windows NT 5.2 (Build 3790) Version Number: Windows NT 5.2 (Build 3790) notice make_ext.pl's CPU time increased from 0.5 sec to 6.75 sec, since The before and after %INC module list of make_ext.pl process, does not ..\lib/buildcustomize.pl make_ext.pl before this patch, already loaded all of EUMM into the How make_ext.pl before, loaded all of EUMM. at C:\p523\src\cpan\ExtUtils-MakeMaker\lib/ExtUtils/MakeMaker.pm line make_ext.pl contains ---------------------------------------------------------- ExtUtils::MM_Unix contains ---------------------------------------------------- Now all of EUMM was sucked in. "require ExtUtils::MM_Unix;" was added With this change of running the Makefile.PL inside make_ext.pl, 1 Perl Info |
From @bulk880001-make_ext.pl-run-all-Makefile.PLes-in-1-process.patchFrom 03b715f0d82386dbed79e0c571fa544e0893b675 Mon Sep 17 00:00:00 2001
From: Daniel Dragan <bulk88@hotmail.com>
Date: Thu, 19 Nov 2015 22:27:25 -0500
Subject: [PATCH] make_ext.pl: run all Makefile.PLes in 1 process
This saves 13 wall seconds for "..\miniperl.exe -I..\lib ..\make_ext.pl
"MAKE=dmake" --dir=..\cpan --dir=..\dist --dir=..\ext --dynamic
!Unicode/Normalize" to complete on Win32. This patch catches all exit()
calls from Makefile.PLs, including exit(0) but no core modules do exit(0)
as normal flow in their Makefile.PLs. It also catches exceptions too. See
Perl RT ticket for details.
---
make_ext.pl | 476 +++++++++++++++++++++++++++++++-----------------------------
1 file changed, 247 insertions(+), 229 deletions(-)
diff --git a/make_ext.pl b/make_ext.pl
index 223f67e..7ac17bb 100644
--- a/make_ext.pl
+++ b/make_ext.pl
@@ -258,143 +258,147 @@ foreach my $spec (@extspec) {
[@pass_through, @{$extra_passthrough{$spec} || []}]);
}
-sub build_extension {
- my ($ext_dir, $perl, $mname, $target, $pass_through) = @_;
+{
+ my($del_makefile_on_exit, $ext_dir, $makefile);
- unless (chdir "$ext_dir") {
- warn "Cannot cd to $ext_dir: $!";
- return;
- }
+ sub build_extension { #build_extensions is not recursion safe
+ $ext_dir = shift;
+ my ($perl, $mname, $target, $pass_through) = @_;
+
+ unless (chdir "$ext_dir") {
+ warn "Cannot cd to $ext_dir: $!";
+ return;
+ }
- my $up = $ext_dir;
- $up =~ s![^/]+!..!g;
+ my $up = $ext_dir;
+ $up =~ s![^/]+!..!g;
- $perl ||= "$up/miniperl";
- my $return_dir = $up;
- my $lib_dir = "$up/lib";
+ $perl ||= "$up/miniperl";
+ my $return_dir = $up;
+ my $lib_dir = "$up/lib";
- my ($makefile, $makefile_no_minus_f);
- if (IS_VMS) {
- $makefile = 'descrip.mms';
- if ($target =~ /clean$/
- && !-f $makefile
- && -f "${makefile}_old") {
- $makefile = "${makefile}_old";
+ my ($makefile_no_minus_f);
+ if (IS_VMS) {
+ $makefile = 'descrip.mms';
+ if ($target =~ /clean$/
+ && !-f $makefile
+ && -f "${makefile}_old") {
+ $makefile = "${makefile}_old";
+ }
+ } else {
+ $makefile = 'Makefile';
}
- } else {
- $makefile = 'Makefile';
- }
-
- if (-f $makefile) {
- $makefile_no_minus_f = 0;
- open my $mfh, $makefile or die "Cannot open $makefile: $!";
- while (<$mfh>) {
- # Plagiarised from CPAN::Distribution
- last if /MakeMaker post_initialize section/;
- next unless /^#\s+VERSION_FROM\s+=>\s+(.+)/;
- my $vmod = eval $1;
- my $oldv;
+
+ if (-f $makefile) {
+ $makefile_no_minus_f = 0;
+ open my $mfh, $makefile or die "Cannot open $makefile: $!";
while (<$mfh>) {
- next unless /^XS_VERSION = (\S+)/;
- $oldv = $1;
- last;
+ # Plagiarised from CPAN::Distribution
+ last if /MakeMaker post_initialize section/;
+ next unless /^#\s+VERSION_FROM\s+=>\s+(.+)/;
+ my $vmod = eval $1;
+ my $oldv;
+ while (<$mfh>) {
+ next unless /^XS_VERSION = (\S+)/;
+ $oldv = $1;
+ last;
+ }
+ last unless defined $oldv;
+ require ExtUtils::MM_Unix;
+ defined (my $newv = parse_version MM $vmod) or last;
+ if ($newv ne $oldv) {
+ close $mfh or die "close $makefile: $!";
+ _unlink($makefile);
+ {
+ no warnings 'deprecated';
+ goto NO_MAKEFILE;
+ }
+ }
}
- last unless defined $oldv;
- require ExtUtils::MM_Unix;
- defined (my $newv = parse_version MM $vmod) or last;
- if ($newv ne $oldv) {
- close $mfh or die "close $makefile: $!";
- _unlink($makefile);
- {
- no warnings 'deprecated';
- goto NO_MAKEFILE;
+
+ if (IS_CROSS) {
+ # If we're cross-compiling, it's possible that the host's
+ # Makefiles are around.
+ seek($mfh, 0, 0) or die "Cannot seek $makefile: $!";
+
+ my $cross_makefile;
+ while (<$mfh>) {
+ # XXX This might not be throughout enough.
+ # For example, it's possible to cause a false-positive
+ # if cross compiling on and for the Raspberry Pi,
+ # which is insane but plausible.
+ # False positives are really not troublesome, though;
+ # all they mean is that the module gets rebuilt.
+ if (/^CC = \Q$Config{cc}\E/) {
+ $cross_makefile = 1;
+ last;
+ }
+ }
+
+ if (!$cross_makefile) {
+ print "Deleting non-Cross makefile\n";
+ close $mfh or die "close $makefile: $!";
+ _unlink($makefile);
}
}
+ } else {
+ $makefile_no_minus_f = 1;
}
- if (IS_CROSS) {
- # If we're cross-compiling, it's possible that the host's
- # Makefiles are around.
- seek($mfh, 0, 0) or die "Cannot seek $makefile: $!";
-
- my $cross_makefile;
- while (<$mfh>) {
- # XXX This might not be throughout enough.
- # For example, it's possible to cause a false-positive
- # if cross compiling on and for the Raspberry Pi,
- # which is insane but plausible.
- # False positives are really not troublesome, though;
- # all they mean is that the module gets rebuilt.
- if (/^CC = \Q$Config{cc}\E/) {
- $cross_makefile = 1;
- last;
- }
- }
-
- if (!$cross_makefile) {
- print "Deleting non-Cross makefile\n";
- close $mfh or die "close $makefile: $!";
- _unlink($makefile);
- }
- }
- } else {
- $makefile_no_minus_f = 1;
- }
+ if ($makefile_no_minus_f || !-f $makefile) {
+ NO_MAKEFILE:
+ if (!-f 'Makefile.PL') {
+ unless (just_pm_to_blib($target, $ext_dir, $mname, $return_dir)) {
+ # No problems returned, so it has faked everything for us. :-)
+ chdir $return_dir || die "Cannot cd to $return_dir: $!";
+ return;
+ }
- if ($makefile_no_minus_f || !-f $makefile) {
- NO_MAKEFILE:
- if (!-f 'Makefile.PL') {
- unless (just_pm_to_blib($target, $ext_dir, $mname, $return_dir)) {
- # No problems returned, so it has faked everything for us. :-)
- chdir $return_dir || die "Cannot cd to $return_dir: $!";
- return;
- }
+ print "\nCreating Makefile.PL in $ext_dir for $mname\n" if $verbose;
+ my ($fromname, $key, $value);
+ if ($mname eq 'podlators') {
+ # We need to special case this somewhere, and this is fewer
+ # lines of code than a core-only Makefile.PL, and no more
+ # complex
+ $fromname = 'VERSION';
+ $key = 'DISTNAME';
+ $value = 'podlators';
+ $mname = 'Pod';
+ } else {
+ $key = 'ABSTRACT_FROM';
+ # We need to cope well with various possible layouts
+ my @dirs = split /::/, $mname;
+ my $leaf = pop @dirs;
+ my $leafname = "$leaf.pm";
+ my $pathname = join '/', @dirs, $leafname;
+ my @locations = ($leafname, $pathname, "lib/$pathname");
+ foreach (@locations) {
+ if (-f $_) {
+ $fromname = $_;
+ last;
+ }
+ }
- print "\nCreating Makefile.PL in $ext_dir for $mname\n" if $verbose;
- my ($fromname, $key, $value);
- if ($mname eq 'podlators') {
- # We need to special case this somewhere, and this is fewer
- # lines of code than a core-only Makefile.PL, and no more
- # complex
- $fromname = 'VERSION';
- $key = 'DISTNAME';
- $value = 'podlators';
- $mname = 'Pod';
- } else {
- $key = 'ABSTRACT_FROM';
- # We need to cope well with various possible layouts
- my @dirs = split /::/, $mname;
- my $leaf = pop @dirs;
- my $leafname = "$leaf.pm";
- my $pathname = join '/', @dirs, $leafname;
- my @locations = ($leafname, $pathname, "lib/$pathname");
- foreach (@locations) {
- if (-f $_) {
- $fromname = $_;
- last;
+ unless ($fromname) {
+ die "For $mname tried @locations in $ext_dir but can't find source";
}
+ ($value = $fromname) =~ s/\.pm\z/.pod/;
+ $value = $fromname unless -e $value;
}
- unless ($fromname) {
- die "For $mname tried @locations in $ext_dir but can't find source";
+ if ($mname eq 'Pod::Checker') {
+ # the abstract in the .pm file is unparseable by MM,
+ # so special-case it. We can't use the package's own
+ # Makefile.PL, as it doesn't handle the executable scripts
+ # right.
+ $key = 'ABSTRACT';
+ # this is copied from the CPAN Makefile.PL v 1.171
+ $value = 'Pod::Checker verifies POD documentation contents for compliance with the POD format specifications';
}
- ($value = $fromname) =~ s/\.pm\z/.pod/;
- $value = $fromname unless -e $value;
- }
- if ($mname eq 'Pod::Checker') {
- # the abstract in the .pm file is unparseable by MM,
- # so special-case it. We can't use the package's own
- # Makefile.PL, as it doesn't handle the executable scripts
- # right.
- $key = 'ABSTRACT';
- # this is copied from the CPAN Makefile.PL v 1.171
- $value = 'Pod::Checker verifies POD documentation contents for compliance with the POD format specifications';
- }
-
- open my $fh, '>', 'Makefile.PL'
- or die "Can't open Makefile.PL for writing: $!";
- printf $fh <<'EOM', $0, $mname, $fromname, $key, $value;
+ open my $fh, '>', 'Makefile.PL'
+ or die "Can't open Makefile.PL for writing: $!";
+ printf $fh <<'EOM', $0, $mname, $fromname, $key, $value;
#-*- buffer-read-only: t -*-
# This Makefile.PL was written by %s.
@@ -447,99 +451,104 @@ WriteMakefile(
# ex: set ro:
EOM
- close $fh or die "Can't close Makefile.PL: $!";
- # As described in commit 23525070d6c0e51f:
- # Push the atime and mtime of generated Makefile.PLs back 4
- # seconds. In certain circumstances ( on virtual machines ) the
- # generated Makefile.PL can produce a Makefile that is older than
- # the Makefile.PL. Altering the atime and mtime backwards by 4
- # seconds seems to resolve the issue.
- eval {
- my $ftime = (stat('Makefile.PL'))[9] - 4;
- utime $ftime, $ftime, 'Makefile.PL';
- };
- } elsif ($mname =~ /\A(?:Carp
- |ExtUtils::CBuilder
- |Safe
- |Search::Dict)\z/x) {
- # An explicit list of dual-life extensions that have a Makefile.PL
- # for CPAN, but we have verified can also be built using the fakery.
- my ($problem) = just_pm_to_blib($target, $ext_dir, $mname, $return_dir);
- # We really need to sanity test that we can fake it.
- # Otherwise "skips" will go undetected, and the build slow down for
- # everyone, defeating the purpose.
- if (defined $problem) {
- if (-d "$return_dir/.git") {
- # Get the list of files that git isn't ignoring:
- my @files = `git ls-files --cached --others --exclude-standard 2>/dev/null`;
- # on error (eg no git) we get nothing, but that's not a
- # problem. The goal is to see if git thinks that the problem
- # file is interesting, by getting a positive match with
- # something git told us about, and if so bail out:
- foreach (@files) {
- chomp;
- # We really need to sanity test that we can fake it.
- # The intent is that this should only fail because
- # you've just added a file to the dual-life dist that
- # we can't handle. In which case you should either
- # 1) remove the dist from the regex a few lines above.
- # or
- # 2) add the file to regex of "safe" filenames earlier
- # in this function, that starts with ChangeLog
- die "FATAL - $0 has $mname in the list of simple extensions, but it now contains file '$problem' which we can't handle"
- if $problem eq $_;
- }
- # There's an unexpected file, but it seems to be something
- # that git will ignore. So fall through to the regular
- # Makefile.PL handling code below, on the assumption that
- # we won't get here for a clean build.
- }
- warn "WARNING - $0 is building $mname using EU::MM, as it found file '$problem'";
- } else {
- # It faked everything for us.
- chdir $return_dir || die "Cannot cd to $return_dir: $!";
- return;
- }
- }
-
- # We are going to have to use Makefile.PL:
- print "\nRunning Makefile.PL in $ext_dir\n" if $verbose;
+ close $fh or die "Can't close Makefile.PL: $!";
+ # As described in commit 23525070d6c0e51f:
+ # Push the atime and mtime of generated Makefile.PLs back 4
+ # seconds. In certain circumstances ( on virtual machines ) the
+ # generated Makefile.PL can produce a Makefile that is older than
+ # the Makefile.PL. Altering the atime and mtime backwards by 4
+ # seconds seems to resolve the issue.
+ eval {
+ my $ftime = (stat('Makefile.PL'))[9] - 4;
+ utime $ftime, $ftime, 'Makefile.PL';
+ };
+ } elsif ($mname =~ /\A(?:Carp
+ |ExtUtils::CBuilder
+ |Safe
+ |Search::Dict)\z/x) {
+ # An explicit list of dual-life extensions that have a Makefile.PL
+ # for CPAN, but we have verified can also be built using the fakery.
+ my ($problem) = just_pm_to_blib($target, $ext_dir, $mname, $return_dir);
+ # We really need to sanity test that we can fake it.
+ # Otherwise "skips" will go undetected, and the build slow down for
+ # everyone, defeating the purpose.
+ if (defined $problem) {
+ if (-d "$return_dir/.git") {
+ # Get the list of files that git isn't ignoring:
+ my @files = `git ls-files --cached --others --exclude-standard 2>/dev/null`;
+ # on error (eg no git) we get nothing, but that's not a
+ # problem. The goal is to see if git thinks that the problem
+ # file is interesting, by getting a positive match with
+ # something git told us about, and if so bail out:
+ foreach (@files) {
+ chomp;
+ # We really need to sanity test that we can fake it.
+ # The intent is that this should only fail because
+ # you've just added a file to the dual-life dist that
+ # we can't handle. In which case you should either
+ # 1) remove the dist from the regex a few lines above.
+ # or
+ # 2) add the file to regex of "safe" filenames earlier
+ # in this function, that starts with ChangeLog
+ die "FATAL - $0 has $mname in the list of simple extensions, but it now contains file '$problem' which we can't handle"
+ if $problem eq $_;
+ }
+ # There's an unexpected file, but it seems to be something
+ # that git will ignore. So fall through to the regular
+ # Makefile.PL handling code below, on the assumption that
+ # we won't get here for a clean build.
+ }
+ warn "WARNING - $0 is building $mname using EU::MM, as it found file '$problem'";
+ } else {
+ # It faked everything for us.
+ chdir $return_dir || die "Cannot cd to $return_dir: $!";
+ return;
+ }
+ }
- my @args = ("-I$lib_dir", 'Makefile.PL');
- if (IS_VMS) {
- my $libd = VMS::Filespec::vmspath($lib_dir);
- push @args, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
- } else {
- push @args, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
- 'INSTALLMAN3DIR=none';
- }
- push @args, @$pass_through;
- _quote_args(\@args) if IS_VMS;
- print join(' ', $perl, @args), "\n" if $verbose;
- my $code = do {
- local $ENV{PERL_MM_USE_DEFAULT} = 1;
- system $perl, @args;
- };
- if($code != 0){
- #make sure next build attempt/run of make_ext.pl doesn't succeed
- _unlink($makefile);
- die "Unsuccessful Makefile.PL($ext_dir): code=$code";
- }
+ # We are going to have to use Makefile.PL:
+ print "\nRunning Makefile.PL in $ext_dir\n" if $verbose;
+
+ my $err;
+ {
+ local @ARGV = (); #fake a system($^X) for perf
+ if (IS_VMS) {
+ my $libd = VMS::Filespec::vmspath($lib_dir);
+ push @ARGV, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
+ } else {
+ push @ARGV, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
+ 'INSTALLMAN3DIR=none';
+ }
+ push @ARGV, @$pass_through;
+ print join(' ', @ARGV), "\n" if $verbose;
+ local $ENV{PERL_MM_USE_DEFAULT} = 1;
+ $del_makefile_on_exit = 1;
+ #last statement not guarenteed to be true, see XSLoader Makefile.PL
+ do '.\Makefile.PL';
+ $del_makefile_on_exit = 0;
+ }
+ if($@) {
+ my $err = $@;
+ #make sure next build attempt/run of make_ext.pl doesn't succeed
+ #even though Makefile.PL failed
+ _unlink($makefile);
+ die "Unsuccessful Makefile.PL($ext_dir): error=$err";
+ }
- # Right. The reason for this little hack is that we're sitting inside
- # a program run by ./miniperl, but there are tasks we need to perform
- # when the 'realclean', 'distclean' or 'veryclean' targets are run.
- # Unfortunately, they can be run *after* 'clean', which deletes
- # ./miniperl
- # So we do our best to leave a set of instructions identical to what
- # we would do if we are run directly as 'realclean' etc
- # Whilst we're perfect, unfortunately the targets we call are not, as
- # some of them rely on a $(PERL) for their own distclean targets.
- # But this always used to be a problem with the old /bin/sh version of
- # this.
- if (IS_UNIX) {
- foreach my $clean_target ('realclean', 'veryclean') {
- fallback_cleanup($return_dir, $clean_target, <<"EOS");
+ # Right. The reason for this little hack is that we're sitting inside
+ # a program run by ./miniperl, but there are tasks we need to perform
+ # when the 'realclean', 'distclean' or 'veryclean' targets are run.
+ # Unfortunately, they can be run *after* 'clean', which deletes
+ # ./miniperl
+ # So we do our best to leave a set of instructions identical to what
+ # we would do if we are run directly as 'realclean' etc
+ # Whilst we're perfect, unfortunately the targets we call are not, as
+ # some of them rely on a $(PERL) for their own distclean targets.
+ # But this always used to be a problem with the old /bin/sh version of
+ # this.
+ if (IS_UNIX) {
+ foreach my $clean_target ('realclean', 'veryclean') {
+ fallback_cleanup($return_dir, $clean_target, <<"EOS");
cd $ext_dir
if test ! -f Makefile -a -f Makefile.old; then
echo "Note: Using Makefile.old"
@@ -552,32 +561,41 @@ else
fi
cd $return_dir
EOS
+ }
}
}
- }
- if (not -f $makefile) {
- print "Warning: No Makefile!\n";
- }
+ if (not -f $makefile) {
+ print "Warning: No Makefile!\n";
+ }
- if (IS_VMS) {
- _quote_args($pass_through);
- @$pass_through = (
- "/DESCRIPTION=$makefile",
- '/MACRO=(' . join(',',@$pass_through) . ')'
- );
- }
+ if (IS_VMS) {
+ _quote_args($pass_through);
+ @$pass_through = (
+ "/DESCRIPTION=$makefile",
+ '/MACRO=(' . join(',',@$pass_through) . ')'
+ );
+ }
+
+ my @targ = ($target, @$pass_through);
+ print "Making $target in $ext_dir\n@make @targ\n" if $verbose;
+ local $ENV{PERL_INSTALL_QUIET} = 1;
+ my $code = system(@make, @targ);
+ if($code >> 8 != 0){ # probably cleaned itself, try again once more time
+ $code = system(@make, @targ);
+ }
+ die "Unsuccessful make($ext_dir): code=$code" if $code != 0;
- my @targ = ($target, @$pass_through);
- print "Making $target in $ext_dir\n@make @targ\n" if $verbose;
- local $ENV{PERL_INSTALL_QUIET} = 1;
- my $code = system(@make, @targ);
- if($code >> 8 != 0){ # probably cleaned itself, try again once more time
- $code = system(@make, @targ);
+ chdir $return_dir || die "Cannot cd to $return_dir: $!";
}
- die "Unsuccessful make($ext_dir): code=$code" if $code != 0;
- chdir $return_dir || die "Cannot cd to $return_dir: $!";
+ END {
+ if($del_makefile_on_exit) { #exit() called, probably non-zero, delete
+ #makefile so next run of make_ext.pl doesn't succeed
+ _unlink($makefile);
+ die "Unsuccessful Makefile.PL($ext_dir): error=exit($?) ";
+ }
+ }
}
sub _quote_args {
--
1.8.0.msysgit.0
|
From @jkeenanOn Thu Nov 19 21:24:00 2015, bulk88 wrote:
The patch, which touches only one file, is over 500 lines along -- but my quick scan of it suggests that many of the changes are whitespace only. That makes it more difficult to review. Would it be possible to re-draw the patch against blead without making any whitespace changes? (I'm not opposed to tidying the code, but because the file being revised is not well known, I'd like to get discussion of the changes going before worrying about tidying.) Thank you very much. -- |
|
The RT System itself - Status changed from 'new' to 'open' |
From @bulk88On Mon Nov 23 15:46:49 2015, jkeenan wrote:
-- |
From @bulk880001-make_ext.pl-run-all-Makefile.PLes-in-1-process.patchFrom 69dd64f723b005a1f67e9bf8b83bab953e14b9f4 Mon Sep 17 00:00:00 2001
From: Daniel Dragan <bulk88@hotmail.com>
Date: Fri, 27 Nov 2015 17:39:36 -0500
Subject: [PATCH] make_ext.pl: run all Makefile.PLes in 1 process
This saves 13 wall seconds for "..\miniperl.exe -I..\lib ..\make_ext.pl
"MAKE=dmake" --dir=..\cpan --dir=..\dist --dir=..\ext --dynamic
!Unicode/Normalize" to complete on Win32. This patch catches all exit()
calls from Makefile.PLs, including exit(0) but no core modules do exit(0)
as normal flow in their Makefile.PLs. It also catches exceptions too. See
Perl RT ticket for details.
---
make_ext.pl | 56 +++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 37 insertions(+), 19 deletions(-)
diff --git a/make_ext.pl b/make_ext.pl
index 223f67e..b01e59a 100644
--- a/make_ext.pl
+++ b/make_ext.pl
@@ -258,8 +258,12 @@ foreach my $spec (@extspec) {
[@pass_through, @{$extra_passthrough{$spec} || []}]);
}
-sub build_extension {
- my ($ext_dir, $perl, $mname, $target, $pass_through) = @_;
+{
+ my($del_makefile_on_exit, $ext_dir, $makefile);
+
+sub build_extension { #build_extensions is not recursion safe
+ $ext_dir = shift;
+ my ($perl, $mname, $target, $pass_through) = @_;
unless (chdir "$ext_dir") {
warn "Cannot cd to $ext_dir: $!";
@@ -273,7 +277,7 @@ sub build_extension {
my $return_dir = $up;
my $lib_dir = "$up/lib";
- my ($makefile, $makefile_no_minus_f);
+ my ($makefile_no_minus_f);
if (IS_VMS) {
$makefile = 'descrip.mms';
if ($target =~ /clean$/
@@ -505,25 +509,30 @@ EOM
# We are going to have to use Makefile.PL:
print "\nRunning Makefile.PL in $ext_dir\n" if $verbose;
- my @args = ("-I$lib_dir", 'Makefile.PL');
- if (IS_VMS) {
- my $libd = VMS::Filespec::vmspath($lib_dir);
- push @args, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
- } else {
- push @args, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
- 'INSTALLMAN3DIR=none';
+ my $err;
+ {
+ local @ARGV = (); #fake a system($^X) for perf
+ if (IS_VMS) {
+ my $libd = VMS::Filespec::vmspath($lib_dir);
+ push @ARGV, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
+ } else {
+ push @ARGV, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
+ 'INSTALLMAN3DIR=none';
+ }
+ push @ARGV, @$pass_through;
+ print join(' ', @ARGV), "\n" if $verbose;
+ local $ENV{PERL_MM_USE_DEFAULT} = 1;
+ $del_makefile_on_exit = 1;
+ #last statement not guarenteed to be true, see XSLoader Makefile.PL
+ do '.\Makefile.PL';
+ $del_makefile_on_exit = 0;
}
- push @args, @$pass_through;
- _quote_args(\@args) if IS_VMS;
- print join(' ', $perl, @args), "\n" if $verbose;
- my $code = do {
- local $ENV{PERL_MM_USE_DEFAULT} = 1;
- system $perl, @args;
- };
- if($code != 0){
+ if($@) {
+ my $err = $@;
#make sure next build attempt/run of make_ext.pl doesn't succeed
+ #even though Makefile.PL failed
_unlink($makefile);
- die "Unsuccessful Makefile.PL($ext_dir): code=$code";
+ die "Unsuccessful Makefile.PL($ext_dir): error=$err";
}
# Right. The reason for this little hack is that we're sitting inside
@@ -580,6 +589,15 @@ EOS
chdir $return_dir || die "Cannot cd to $return_dir: $!";
}
+ END {
+ if($del_makefile_on_exit) { #exit() called, probably non-zero, delete
+ #makefile so next run of make_ext.pl doesn't succeed
+ _unlink($makefile);
+ die "Unsuccessful Makefile.PL($ext_dir): error=exit($?) ";
+ }
+ }
+}
+
sub _quote_args {
my $args = shift; # must be array reference
--
1.9.5.msysgit.1
|
From @jkeenanOn Fri Nov 27 18:23:59 2015, bulk88 wrote:
Thanks for the revised patch. Unfortunately, when I applied the patch to a branch forked from blead at commit d61acc4 and then called, as I usually do, 'sh ./Configure -des -Dusedevel', 'make' failed. Transcript of 'make' attached. -- |
From @bulk88On Sun Nov 29 05:11:27 2015, jkeenan wrote:
Revised patch attached. -- |
From @bulk880001-make_ext.pl-run-all-Makefile.PLes-in-1-process.patchFrom 45141c2bfafd2e18dfb00d6feb88d3f6debf8995 Mon Sep 17 00:00:00 2001
From: Daniel Dragan <bulk88@hotmail.com>
Date: Mon, 30 Nov 2015 00:50:45 -0500
Subject: [PATCH] make_ext.pl: run all Makefile.PLes in 1 process
This saves 13 wall seconds for "..\miniperl.exe -I..\lib ..\make_ext.pl
"MAKE=dmake" --dir=..\cpan --dir=..\dist --dir=..\ext --dynamic
!Unicode/Normalize" to complete on Win32. This patch catches all exit()
calls from Makefile.PLs, including exit(0) but no core modules do exit(0)
as normal flow in their Makefile.PLs. It also catches exceptions too. See
Perl RT ticket for details.
---
make_ext.pl | 56 +++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 37 insertions(+), 19 deletions(-)
diff --git a/make_ext.pl b/make_ext.pl
index 223f67e..575239d 100644
--- a/make_ext.pl
+++ b/make_ext.pl
@@ -258,8 +258,12 @@ foreach my $spec (@extspec) {
[@pass_through, @{$extra_passthrough{$spec} || []}]);
}
-sub build_extension {
- my ($ext_dir, $perl, $mname, $target, $pass_through) = @_;
+{
+ my($del_makefile_on_exit, $ext_dir, $makefile);
+
+sub build_extension { #build_extensions is not recursion safe
+ $ext_dir = shift;
+ my ($perl, $mname, $target, $pass_through) = @_;
unless (chdir "$ext_dir") {
warn "Cannot cd to $ext_dir: $!";
@@ -273,7 +277,7 @@ sub build_extension {
my $return_dir = $up;
my $lib_dir = "$up/lib";
- my ($makefile, $makefile_no_minus_f);
+ my ($makefile_no_minus_f);
if (IS_VMS) {
$makefile = 'descrip.mms';
if ($target =~ /clean$/
@@ -505,25 +509,30 @@ EOM
# We are going to have to use Makefile.PL:
print "\nRunning Makefile.PL in $ext_dir\n" if $verbose;
- my @args = ("-I$lib_dir", 'Makefile.PL');
- if (IS_VMS) {
- my $libd = VMS::Filespec::vmspath($lib_dir);
- push @args, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
- } else {
- push @args, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
- 'INSTALLMAN3DIR=none';
+ my $err;
+ {
+ local @ARGV = (); #fake a system($^X) for perf
+ if (IS_VMS) {
+ my $libd = VMS::Filespec::vmspath($lib_dir);
+ push @ARGV, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
+ } else {
+ push @ARGV, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
+ 'INSTALLMAN3DIR=none';
+ }
+ push @ARGV, @$pass_through;
+ print join(' ', @ARGV), "\n" if $verbose;
+ local $ENV{PERL_MM_USE_DEFAULT} = 1;
+ $del_makefile_on_exit = 1;
+ #last statement not guarenteed to be true, see XSLoader Makefile.PL
+ do './Makefile.PL';
+ $del_makefile_on_exit = 0;
}
- push @args, @$pass_through;
- _quote_args(\@args) if IS_VMS;
- print join(' ', $perl, @args), "\n" if $verbose;
- my $code = do {
- local $ENV{PERL_MM_USE_DEFAULT} = 1;
- system $perl, @args;
- };
- if($code != 0){
+ if($@) {
+ my $err = $@;
#make sure next build attempt/run of make_ext.pl doesn't succeed
+ #even though Makefile.PL failed
_unlink($makefile);
- die "Unsuccessful Makefile.PL($ext_dir): code=$code";
+ die "Unsuccessful Makefile.PL($ext_dir): error=$err";
}
# Right. The reason for this little hack is that we're sitting inside
@@ -580,6 +589,15 @@ EOS
chdir $return_dir || die "Cannot cd to $return_dir: $!";
}
+ END {
+ if($del_makefile_on_exit) { #exit() called, probably non-zero, delete
+ #makefile so next run of make_ext.pl doesn't succeed
+ _unlink($makefile);
+ die "Unsuccessful Makefile.PL($ext_dir): error=exit($?) ";
+ }
+ }
+}
+
sub _quote_args {
my $args = shift; # must be array reference
--
1.9.5.msysgit.1
|
From @jkeenanOn Mon Nov 30 01:14:58 2015, bulk88 wrote:
This patch also experienced 'make' failures. On linux-x86_64 I forked a branch from blead at 4c29884 and applied https://rt-archive.perl.org/perl5/Ticket/Attachment/1377765/739931/0001-make_ext.pl-run-all-Makefile.PLes-in-1-process.patch to the branch. I then ran: sh ./Configure -des -Dusedevel && make Full transcript, gzipped, attached. Excerpt from its tail: ##### -- |
From @bulk88On Mon Nov 30 16:59:41 2015, jkeenan wrote:
Revised again. The last patch I smoked on a Unix OS but it apparently didn't have libdb so DB_File was never attempted to be built on it per the logic in FindExt.pm. -- |
From @bulk880001-make_ext.pl-run-all-Makefile.PLes-in-1-process.patchFrom f0267747fcc38be985926dc44bfa97fe94c1806d Mon Sep 17 00:00:00 2001
From: Daniel Dragan <bulk88@hotmail.com>
Date: Tue, 1 Dec 2015 16:44:12 -0500
Subject: [PATCH] make_ext.pl: run all Makefile.PLes in 1 process
This saves 13 wall seconds for "..\miniperl.exe -I..\lib ..\make_ext.pl
"MAKE=dmake" --dir=..\cpan --dir=..\dist --dir=..\ext --dynamic
!Unicode/Normalize" to complete on Win32. This patch catches all exit()
calls from Makefile.PLs, including exit(0). While exit(0) is techincally
legal for Makefile.PLs to do, it is rare, nearly all Makefile.PLs fall off
the end when they are done. So exit(0) for core purposes is really an error
since the build will prematurly stop. Fix the 1 case where exit(0) was
called. This patch also catches exceptions from the Makefile.PL.
See Perl RT ticket for details.
---
cpan/DB_File/Makefile.PL | 3 ---
make_ext.pl | 56 ++++++++++++++++++++++++++++++++----------------
2 files changed, 37 insertions(+), 22 deletions(-)
diff --git a/cpan/DB_File/Makefile.PL b/cpan/DB_File/Makefile.PL
index f5ef7c8..f203deb 100644
--- a/cpan/DB_File/Makefile.PL
+++ b/cpan/DB_File/Makefile.PL
@@ -147,9 +147,6 @@ else {
or die "Can't copy fallback.xs to constants.xs: $!";
}
-exit;
-
-
sub MY::libscan
{
my $self = shift ;
diff --git a/make_ext.pl b/make_ext.pl
index 223f67e..575239d 100644
--- a/make_ext.pl
+++ b/make_ext.pl
@@ -258,8 +258,12 @@ foreach my $spec (@extspec) {
[@pass_through, @{$extra_passthrough{$spec} || []}]);
}
-sub build_extension {
- my ($ext_dir, $perl, $mname, $target, $pass_through) = @_;
+{
+ my($del_makefile_on_exit, $ext_dir, $makefile);
+
+sub build_extension { #build_extensions is not recursion safe
+ $ext_dir = shift;
+ my ($perl, $mname, $target, $pass_through) = @_;
unless (chdir "$ext_dir") {
warn "Cannot cd to $ext_dir: $!";
@@ -273,7 +277,7 @@ sub build_extension {
my $return_dir = $up;
my $lib_dir = "$up/lib";
- my ($makefile, $makefile_no_minus_f);
+ my ($makefile_no_minus_f);
if (IS_VMS) {
$makefile = 'descrip.mms';
if ($target =~ /clean$/
@@ -505,25 +509,30 @@ EOM
# We are going to have to use Makefile.PL:
print "\nRunning Makefile.PL in $ext_dir\n" if $verbose;
- my @args = ("-I$lib_dir", 'Makefile.PL');
- if (IS_VMS) {
- my $libd = VMS::Filespec::vmspath($lib_dir);
- push @args, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
- } else {
- push @args, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
- 'INSTALLMAN3DIR=none';
+ my $err;
+ {
+ local @ARGV = (); #fake a system($^X) for perf
+ if (IS_VMS) {
+ my $libd = VMS::Filespec::vmspath($lib_dir);
+ push @ARGV, "INST_LIB=$libd", "INST_ARCHLIB=$libd";
+ } else {
+ push @ARGV, 'INSTALLDIRS=perl', 'INSTALLMAN1DIR=none',
+ 'INSTALLMAN3DIR=none';
+ }
+ push @ARGV, @$pass_through;
+ print join(' ', @ARGV), "\n" if $verbose;
+ local $ENV{PERL_MM_USE_DEFAULT} = 1;
+ $del_makefile_on_exit = 1;
+ #last statement not guarenteed to be true, see XSLoader Makefile.PL
+ do './Makefile.PL';
+ $del_makefile_on_exit = 0;
}
- push @args, @$pass_through;
- _quote_args(\@args) if IS_VMS;
- print join(' ', $perl, @args), "\n" if $verbose;
- my $code = do {
- local $ENV{PERL_MM_USE_DEFAULT} = 1;
- system $perl, @args;
- };
- if($code != 0){
+ if($@) {
+ my $err = $@;
#make sure next build attempt/run of make_ext.pl doesn't succeed
+ #even though Makefile.PL failed
_unlink($makefile);
- die "Unsuccessful Makefile.PL($ext_dir): code=$code";
+ die "Unsuccessful Makefile.PL($ext_dir): error=$err";
}
# Right. The reason for this little hack is that we're sitting inside
@@ -580,6 +589,15 @@ EOS
chdir $return_dir || die "Cannot cd to $return_dir: $!";
}
+ END {
+ if($del_makefile_on_exit) { #exit() called, probably non-zero, delete
+ #makefile so next run of make_ext.pl doesn't succeed
+ _unlink($makefile);
+ die "Unsuccessful Makefile.PL($ext_dir): error=exit($?) ";
+ }
+ }
+}
+
sub _quote_args {
my $args = shift; # must be array reference
--
1.9.5.msysgit.1
|
From @jkeenanOn Tue Dec 01 14:21:26 2015, bulk88 wrote:
This version of the patch built successfully. It is now smoking in this branch: smoke-me/jkeenan/bulk88/126686-make-ext-3 I'll leave to others the questions: (a) whether it speeds up 'make' on both Windows and other OSes; and (b) whether it should be applied. Thank you very much. -- |
From @bulk88bump -- |
From @tonycozOn Tue Dec 01 14:21:26 2015, bulk88 wrote:
That's pretty questionable. DB_File is CPAN upstream, I'm not sure it's worth breaking its Makefile.PL for a small build-time optimization. There is a build time advantage on Linux - full make -j6 build time (not include Configure) on a 4 core CPU: before: real 1m16.346s real 1m13.703s real 1m10.914s after: real 1m10.237s real 1m8.678s real 1m10.573s Tony |
|
@bulk88 Given this is not a bug but a minor build optimization, I'm going to suggest that further revisions of this idea be moved to a pull request. |
Migrated from rt.perl.org#126686 (status was 'open')
Searchable as RT126686$
The text was updated successfully, but these errors were encountered: