1377 lines
30 KiB
Perl
1377 lines
30 KiB
Perl
# ex:ts=8 sw=4:
|
|
# $OpenBSD: Port.pm,v 1.213 2023/07/07 11:34:16 espie Exp $
|
|
#
|
|
# Copyright (c) 2010-2013 Marc Espie <espie@openbsd.org>
|
|
#
|
|
# Permission to use, copy, modify, and distribute this software for any
|
|
# purpose with or without fee is hereby granted, provided that the above
|
|
# copyright notice and this permission notice appear in all copies.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
use v5.36;
|
|
|
|
use DPB::Job;
|
|
use DPB::Clock;
|
|
|
|
package DPB::Junk;
|
|
|
|
# job is normally attached to core, unless it's not attached,
|
|
# and then we pass it as an extra parameter
|
|
sub want($class, $core, $job = $core->job)
|
|
{
|
|
return 2 if $job->{v}->forcejunk;
|
|
# XXX let's wipe the slates at the start of the first tagged
|
|
# job, as we don't know the exact state of the host.
|
|
return 2 if $job->{v}{info}->has_property('tag') &&
|
|
!defined $core->prop->{last_junk};
|
|
return 0 unless defined $core->prop->{junk};
|
|
if ($core->prop->{depends_count} >= $core->prop->{junk}) {
|
|
return 1;
|
|
} else {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
package DPB::Task::BasePort;
|
|
our @ISA = qw(DPB::Task::Clocked);
|
|
use OpenBSD::Paths;
|
|
|
|
sub setup($task, $core)
|
|
{
|
|
return $task;
|
|
}
|
|
|
|
sub is_serialized($) { 0 }
|
|
sub want_frozen($) { 1 }
|
|
sub want_percent($) { 1 }
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
if ($job->{pos}) {
|
|
open(my $fh, "<&", $job->{logfh});
|
|
if (defined $fh && seek($fh, $job->{pos}, 0)) {
|
|
for my $i (1 .. 10) {
|
|
my $l = <$fh>;
|
|
last if !defined $l;
|
|
if ($l =~ m/^\=\=\=\>\s+Building from scratch/) {
|
|
$job->{fromscratch} = $i;
|
|
if (defined $job->{watched}) {
|
|
$job->{watched}->set_offset(
|
|
$job->{pos});
|
|
}
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
$job->{fromscratch} //= 0;
|
|
delete $job->{pos};
|
|
}
|
|
$self->SUPER::finalize($core);
|
|
$job->finished_task($self);
|
|
return $core->{status} == 0;
|
|
}
|
|
|
|
# note that tasks are using the "flyweight" pattern: they're
|
|
# just a name + behavior, and all the data is in job (which is
|
|
# obtained thru core)
|
|
sub new($class, $phase)
|
|
{
|
|
bless {phase => $phase}, $class;
|
|
}
|
|
|
|
sub fork($self, $core)
|
|
{
|
|
return $self->SUPER::fork($core);
|
|
}
|
|
|
|
sub handle_output($self, $job)
|
|
{
|
|
$self->redirect_fh($job->{logfh}, $job->{log});
|
|
print ">>> Running $self->{phase} in $job->{path} at ",
|
|
DPB::Util->current_ts, "\n";
|
|
}
|
|
|
|
sub tweak_args($self, $args, $job, $builder)
|
|
{
|
|
push(@$args,
|
|
"FETCH_PACKAGES=No",
|
|
"PREPARE_CHECK_ONLY=Yes",
|
|
"REPORT_PROBLEM='exit 1'", "BULK=No");
|
|
if ($job->{parallel}) {
|
|
push(@$args, "MAKE_JOBS=$job->{parallel}");
|
|
}
|
|
if ($job->{memsize}) {
|
|
push(@$args, "USE_MFS=Yes");
|
|
}
|
|
if ($builder->{nochecksum}) {
|
|
push(@$args, "NO_CHECKSUM=Yes");
|
|
}
|
|
}
|
|
|
|
sub run($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
my $t = $self->{phase};
|
|
my $builder = $job->{builder};
|
|
my $ports = $builder->ports;
|
|
my $fullpkgpath = $job->{path};
|
|
if ($core->prop->{syslog}) {
|
|
Sys::Syslog::syslog('info', "start $fullpkgpath($t)");
|
|
}
|
|
$self->handle_output($job);
|
|
close STDIN;
|
|
open STDIN, '</dev/null';
|
|
my @args = ($t);
|
|
$self->tweak_args(\@args, $job, $builder);
|
|
|
|
my @l = $builder->make_args;
|
|
my $make = $builder->make;
|
|
my @env = ();
|
|
if (defined $builder->{rsslog}) {
|
|
unless ($self->notime) {
|
|
$make = $builder->{wrapper};
|
|
$l[0] = $make;
|
|
push(@env, WRAPPER_OUTPUT => $builder->{rsslog});
|
|
}
|
|
}
|
|
|
|
unshift(@args, @l);
|
|
$core->shell
|
|
->as_root($self->{as_root})
|
|
->env(SUBDIR => $fullpkgpath,
|
|
PHASE => $t,
|
|
@env)
|
|
->exec(@args);
|
|
exit(1);
|
|
}
|
|
|
|
sub name($self)
|
|
{
|
|
return $self->{phase};
|
|
}
|
|
|
|
sub notime($) { 0 }
|
|
|
|
package DPB::Task::Port;
|
|
our @ISA = qw(DPB::Task::BasePort);
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$self->SUPER::finalize($core);
|
|
if ($core->prop->{syslog}) {
|
|
my $fullpkgpath = $core->job->{path};
|
|
my $t = $self->{phase};
|
|
Sys::Syslog::syslog('info', "end $fullpkgpath($t)");
|
|
}
|
|
if ($core->{status} == 0) {
|
|
return 1;
|
|
}
|
|
$core->job->{failed} = $core->{status};
|
|
if ($core->prop->{always_clean}) {
|
|
$core->job->replace_tasks(DPB::Port::TaskFactory->create('clean'));
|
|
return 1;
|
|
}
|
|
# XXX in case we taint the core, we will mark ourselves as cleaned
|
|
# so the tag and dependencies may vanish.
|
|
#
|
|
# this is a bit of a pain for fixing errors, but this ensures bulks
|
|
# *will* finish anyhow
|
|
#
|
|
if ($core->job->{v}{info}->has_property('tag')) {
|
|
$core->job->{lock}->write("cleaned");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
# return swallowed cores at the end of fake: package is inherently sequential
|
|
# and there's some "thundering herd" effect when we release lots of cores,
|
|
# so release them a bit early, so by the time we're finished packaging,
|
|
# they're mostly out of "waiting-for-lock"
|
|
package DPB::Task::Port::Fake;
|
|
our @ISA = qw(DPB::Task::Port);
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$core->unswallow;
|
|
delete $core->job->{nojunk};
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
# some ports prevent junking only up-to-configure
|
|
package DPB::Task::Port::Configure;
|
|
our @ISA = qw(DPB::Task::Port);
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
if ($job->{noconfigurejunk}) {
|
|
delete $core->job->{nojunk};
|
|
}
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
package DPB::Task::Port::Extract;
|
|
our @ISA = qw(DPB::Task::Port);
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
# XXX we only exist because there is nojunk involved
|
|
$job->{nojunk} = 1;
|
|
# XXX don't bother marking ourselves for configurejunk
|
|
if (!$job->{noconfigurejunk}) {
|
|
$job->{lock}->write("nojunk");
|
|
}
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
package DPB::Task::Port::Signature;
|
|
our @ISA =qw(DPB::Task::BasePort);
|
|
|
|
sub notime($) { 1 }
|
|
|
|
sub run($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
$self->handle_output($job);
|
|
exit($job->{builder}->check_signature($core, $job->{v}));
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$self->SUPER::finalize($core);
|
|
my $job = $core->job;
|
|
if ($core->{status} == 0) {
|
|
my $v = $job->{v};
|
|
my $builder = $job->{builder};
|
|
$job->add_normal_tasks($builder->should_clean($v), $core);
|
|
} else {
|
|
$job->{signature_only} = 1;
|
|
$job->{builder}->register_updates($job->{v});
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
package DPB::Task::Port::Checksum;
|
|
our @ISA = qw(DPB::Task::Port);
|
|
|
|
sub need_checksum($self, $log, $info)
|
|
{
|
|
my $need = 0;
|
|
for my $dist (values %{$info->{DIST}}) {
|
|
if (!$dist->cached_checksum($log, $dist->filename)) {
|
|
$need = 1;
|
|
} else {
|
|
unlink($dist->tempfilename);
|
|
}
|
|
}
|
|
return $need;
|
|
}
|
|
|
|
sub setup($task, $core)
|
|
{
|
|
my $job = $core->job;
|
|
my $info = $job->{v}{info};
|
|
if (defined $info->{distsize}) {
|
|
print {$job->{logfh}} "distfiles size=$info->{distsize}\n";
|
|
}
|
|
if ($task->need_checksum($job->{logfh}, $info)) {
|
|
return $task;
|
|
} else {
|
|
delete $info->{DIST};
|
|
return $job->next_task($core);
|
|
}
|
|
}
|
|
|
|
sub checksum($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
$self->handle_output($job);
|
|
my $exit = 0;
|
|
for my $dist (values %{$job->{v}{info}{DIST}}) {
|
|
if (!$dist->checksum($dist->filename)) {
|
|
$exit = 1;
|
|
} else {
|
|
unlink($dist->tempfilename);
|
|
}
|
|
}
|
|
return $exit;
|
|
}
|
|
|
|
sub run($self, $core)
|
|
{
|
|
exit($self->checksum($core));
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$self->SUPER::finalize($core);
|
|
if ($core->{status} == 0) {
|
|
delete $core->job->{v}{info}{DIST};
|
|
}
|
|
}
|
|
|
|
|
|
package DPB::Task::Port::Serialized;
|
|
our @ISA = qw(DPB::Task::Port);
|
|
|
|
sub is_serialized($) { 1 }
|
|
sub want_percent($) { 0 }
|
|
|
|
# note that serialized's setup will return its task only if lock
|
|
# happened succesfully, so we can use that in serialized tasks
|
|
sub setup($task, $core)
|
|
{
|
|
my $job = $core->job;
|
|
if (!$job->{locked}) {
|
|
$task->try_lock($core);
|
|
}
|
|
if (!$job->{locked}) {
|
|
unshift(@{$job->{tasks}}, $task);
|
|
$job->{wakemeup} = $core->prop->{waited_for_lock}++;
|
|
my $o = $job->{builder}->locker->lock_has_other_owner($core);
|
|
my $status = 'waiting-for-lock #'.$job->{wakemeup};
|
|
if (defined $o) {
|
|
$status ='locked by '.$o;
|
|
}
|
|
return DPB::Task::Port::Lock->new($status);
|
|
}
|
|
|
|
return $task;
|
|
}
|
|
|
|
sub try_lock($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
|
|
my $lock = $job->{builder}->locker->lock($core);
|
|
if ($lock) {
|
|
$lock->write("path", $job->{path});
|
|
print {$job->{logfh}} "(Junk lock obtained for ",
|
|
$core->hostname, " at ", DPB::Util->current_ts, ")\n";
|
|
$job->{locked} = 1;
|
|
}
|
|
}
|
|
|
|
sub junk_unlock($self, $core)
|
|
{
|
|
if ($core->job->{locked}) {
|
|
$core->job->{builder}->locker->unlock($core);
|
|
print {$core->job->{logfh}} "(Junk lock released for ",
|
|
$core->hostname, " at ", DPB::Util->current_ts, ")\n";
|
|
delete $core->job->{locked};
|
|
$core->job->wake_others($core);
|
|
}
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
my $task = $job->{tasks}[0];
|
|
# XXX if we didn't lock at the entrance, we locked here.
|
|
$job->{locked} = 1;
|
|
if ($core->{status} != 0 || !defined $task || !$task->is_serialized) {
|
|
$self->junk_unlock($core);
|
|
}
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
|
|
# this is the full locking task, the one that has to wait.
|
|
|
|
package DPB::Task::Port::Lock;
|
|
our @ISA = qw(DPB::Task::Port::Serialized);
|
|
|
|
sub setup($task, $core)
|
|
{
|
|
return $task
|
|
}
|
|
|
|
sub want_frozen($) { 0 }
|
|
|
|
sub run($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
$SIG{IO} = sub { print {$job->{logfh}} "Received IO\n"; };
|
|
my $start = Time::HiRes::time();
|
|
use POSIX;
|
|
|
|
while (1) {
|
|
$self->try_lock($core);
|
|
my $now = Time::HiRes::time();
|
|
if ($job->{locked}) {
|
|
print {$job->{builder}{lockperf}}
|
|
$now, ":", $core->hostname,
|
|
": $self->{phase}: ", $now - $start, " seconds\n";
|
|
exit(0);
|
|
}
|
|
print {$job->{logfh}} "(Junk lock failure for ",
|
|
$core->hostname, " at ", $now, ")\n";
|
|
pause;
|
|
}
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$core->job->{locked} = 1;
|
|
delete $core->job->{wakemeup};
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
package DPB::Task::Port::Depends;
|
|
our @ISA=qw(DPB::Task::Port::Serialized);
|
|
|
|
sub notime($) { 1 }
|
|
|
|
sub recompute_depends($self, $core)
|
|
{
|
|
# we're running this synchronously with other jobs, so
|
|
# let's try avoid running pkg_add if we can !
|
|
# compute all missing deps for all jobs currently waiting
|
|
my $deps = {};
|
|
|
|
# XXX not a "same_host_jobs" as we're in setup, so not
|
|
# actually running
|
|
for my $d (keys %{$core->job->{depends}}) {
|
|
$deps->{$d} = $d;
|
|
}
|
|
for my $job ($core->same_host_jobs) {
|
|
next if $job->{shunt_depends};
|
|
for my $d (keys %{$job->{depends}}) {
|
|
$deps->{$d} = $d;
|
|
$job->{shunt_depends} = $core->job->{path};
|
|
}
|
|
}
|
|
for my $job ($core->same_host_jobs) {
|
|
next unless defined $job->{live_depends};
|
|
for my $d (@{$job->{live_depends}}) {
|
|
delete $deps->{$d};
|
|
}
|
|
}
|
|
return $deps;
|
|
}
|
|
|
|
sub setup($task, $core)
|
|
{
|
|
my $job = $core->job;
|
|
|
|
# first, we must be sure to have the lock !
|
|
$task = $task->SUPER::setup($core);
|
|
if (!$job->{locked}) {
|
|
return $task;
|
|
}
|
|
|
|
if ($job->{shunt_depends}) {
|
|
print {$job->{logfh}} "Short-cut: depends already handled by ",
|
|
$job->{shunt_depends}, "\n";
|
|
return $job->next_task($core);
|
|
}
|
|
my $dep = $task->recompute_depends($core);
|
|
if (keys %$dep == 0) {
|
|
return $job->next_task($core);
|
|
} else {
|
|
$job->{dodeps} = $dep;
|
|
return $task;
|
|
}
|
|
}
|
|
|
|
sub run($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
$self->handle_output($job);
|
|
if ($core->prop->{syslog}) {
|
|
Sys::Syslog::syslog('info', "start $job->{path}(depends)");
|
|
}
|
|
|
|
if (defined $core->prop->{last_junk}) {
|
|
print " last junk was in ",
|
|
$core->prop->{last_junk}->fullpkgpath, "\n";
|
|
}
|
|
my @cmd = ('/usr/sbin/pkg_add', '-aI');
|
|
if ($job->{builder}{update}) {
|
|
push(@cmd, "-rqU", "-Dupdate", "-Dupdatedepends");
|
|
}
|
|
if ($job->{builder}{forceupdate}) {
|
|
push(@cmd, "-Dinstalled");
|
|
}
|
|
if ($core->prop->{repair}) {
|
|
push(@cmd, "-Drepair");
|
|
}
|
|
if ($job->{builder}{state}{localbase} ne '/usr/local') {
|
|
push(@cmd, "-L", $job->{builder}{state}{localbase});
|
|
}
|
|
my @l = (sort keys %{$job->{dodeps}});
|
|
print join(' ', @cmd, @l), "\n";
|
|
print "was: ", join(' ', @cmd, (sort keys %{$job->{depends}})), "\n";
|
|
print join(' ', @cmd, @l), "\n";
|
|
my $path = $job->{builder}{fullrepo}.'/';
|
|
$core->shell->env(TRUSTED_PKG_PATH => $path)->as_root->exec(@cmd, @l);
|
|
exit(1);
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$core->{status} = 0;
|
|
|
|
$self->SUPER::finalize($core);
|
|
return 1;
|
|
}
|
|
|
|
package DPB::Task::Port::PrepareResults;
|
|
our @ISA = qw(DPB::Task::Port::Serialized);
|
|
|
|
sub setup($task, $core)
|
|
{
|
|
my $job = $core->job;
|
|
$job->{pos} = tell($job->{logfh});
|
|
$job->track_lock;
|
|
return $task->SUPER::setup($core);
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
my $job = $core->{job};
|
|
my $v = $job->{v};
|
|
# peek at log at the right location
|
|
open(my $fh, "<&", $job->{logfh});
|
|
if (defined $fh && seek($fh, $job->{pos}, 0)) {
|
|
my @r;
|
|
while (<$fh>) {
|
|
last if m/^\>\>\>\s+Running\s+show-prepare-results/;
|
|
}
|
|
while (<$fh>) {
|
|
# zap headers
|
|
next if m/^\>\>\>\s/ || m/^\=\=\=\>\s/;
|
|
chomp;
|
|
# normal lines *only have one package name*
|
|
next if m/\s/;
|
|
push(@r, $_);
|
|
}
|
|
close $fh;
|
|
$job->save_depends(\@r);
|
|
# XXX we ran junk before us, so retaint *now*
|
|
# before losing the lock
|
|
if ($job->{v}{info}->has_property('tag') &&
|
|
!defined $core->prop->{tainted}) {
|
|
$core->prop->taint($v);
|
|
print {$job->{logfh}} "Forced junk, retainting: ",
|
|
$core->prop->{tainted}, "\n";
|
|
}
|
|
} else {
|
|
$core->{status} = 1;
|
|
}
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
package DPB::Task::Port::Uninstall;
|
|
our @ISA=qw(DPB::Task::Port::Serialized);
|
|
|
|
sub notime($) { 1 }
|
|
|
|
# uninstall is actually a "tentative" junk case
|
|
# it might not happen for various reasons:
|
|
# - a port that's building on the same host that says "nojunk"
|
|
# - something else went thru simultaneously and junked already
|
|
|
|
|
|
sub setup($task, $core)
|
|
{
|
|
# we got pre-empted
|
|
# no actual need to junk
|
|
if (!DPB::Junk->want($core)) {
|
|
$task->junk_unlock($core);
|
|
return $core->job->next_task($core);
|
|
}
|
|
# okay we have to make sure we're locked first
|
|
my $t2 = $task->SUPER::setup($core);
|
|
if ($t2 != $task) {
|
|
return $t2;
|
|
}
|
|
my $fh = $core->job->{logfh};
|
|
# so we're locked, let's boogie
|
|
my $still_tainted;
|
|
for my $job ($core->same_host_jobs) {
|
|
if ($job->{nojunk}) {
|
|
# we can't junk go next
|
|
print $fh "Don't run junk because nojunk in ",
|
|
$job->{path}, "\n";
|
|
$task->junk_unlock($core);
|
|
return $core->job->next_task($core);
|
|
}
|
|
if ($job->{v}{info}->has_property('tag')) {
|
|
$still_tainted //= "host tagged by $job->{path}";
|
|
}
|
|
}
|
|
if (!defined $still_tainted) {
|
|
my ($tag, $owner) =
|
|
$core->job->{builder}->locker->find_tag($core->hostname);
|
|
if (defined $tag) {
|
|
$still_tainted = "host tagged with $tag by $owner";
|
|
}
|
|
}
|
|
if (!defined $still_tainted) {
|
|
# XXX deal better with old nojunk stuff ?
|
|
# there are some decisions to take. For now,
|
|
# let's just make sure it's not broken
|
|
my ($nojunk, $h) = $core->job->{builder}->locker->
|
|
find_dependencies($core->hostname);
|
|
if ($nojunk) {
|
|
$still_tainted = "host marked nojunk by $nojunk";
|
|
}
|
|
}
|
|
# we are going along with junk, BUT we may still be tainted
|
|
if (defined $still_tainted) {
|
|
print $fh "Still tainted: $still_tainted\n";
|
|
} else {
|
|
print $fh "Still tainted: no\n";
|
|
$core->prop->untaint;
|
|
}
|
|
return $task;
|
|
}
|
|
|
|
sub add_dontjunk($self, $job, $h)
|
|
{
|
|
return if !defined $job->{builder}{dontjunk};
|
|
for my $pkgname (keys %{$job->{builder}{dontjunk}}) {
|
|
$h->{$pkgname} = 1;
|
|
}
|
|
}
|
|
|
|
sub add_live_depends($self, $h, $core)
|
|
{
|
|
for my $job ($core->same_host_jobs) {
|
|
if (defined $job->{live_depends}) {
|
|
for my $d (@{$job->{live_depends}}) {
|
|
$h->{$d} = 1;
|
|
}
|
|
}
|
|
for my $d (keys %{$job->{depends}}) {
|
|
$h->{$d} = 1;
|
|
}
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
sub run($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
my $v = $job->{v};
|
|
|
|
$self->handle_output($job);
|
|
|
|
my ($nojunk, $h) =
|
|
$job->{builder}->locker->find_dependencies($core->hostname);
|
|
if ($nojunk) {
|
|
print "Can't run junk because of lock on $nojunk\n";
|
|
exit(2);
|
|
}
|
|
if ($self->add_live_depends($h, $core)) {
|
|
$self->add_dontjunk($job, $h);
|
|
my $opt = '-aIX';
|
|
if ($core->prop->{nochecksum}) {
|
|
$opt .= 'q';
|
|
}
|
|
my @cmd = ('/usr/sbin/pkg_delete', $opt, sort keys %$h);
|
|
my $s = join(' ', @cmd)."\n";
|
|
print $s;
|
|
if (!$core->prop->{silentjunking}) {
|
|
for my $j ($core->same_host_jobs) {
|
|
next if $j eq $job;
|
|
$j->silent_log(
|
|
">> JUNKING in $job->{path}:\n",
|
|
">> $s");
|
|
|
|
}
|
|
}
|
|
$core->shell->as_root->exec(@cmd);
|
|
exit(1);
|
|
} else {
|
|
exit(2);
|
|
}
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
# did we really run ? then clean up stuff
|
|
if ($core->{status} == 0) {
|
|
my $job = $core->job;
|
|
if (!$core->prop->{silentjunking}) {
|
|
for my $j ($core->same_host_jobs) {
|
|
next if $j eq $job;
|
|
$j->silent_log(
|
|
">> JUNKING end in $job->{path}\n");
|
|
}
|
|
}
|
|
$core->prop->{last_junk} = $job->{v};
|
|
$core->prop->{junk_count} = 0;
|
|
$core->prop->{ports_count} = 0;
|
|
$core->prop->{depends_count} = 0;
|
|
}
|
|
$core->{status} = 0;
|
|
$self->SUPER::finalize($core);
|
|
return 1;
|
|
}
|
|
|
|
# there's nothing to run here, just where we get committed to affinity
|
|
package DPB::Task::Port::InBetween;
|
|
our @ISA = qw(DPB::Task::BasePort);
|
|
sub setup($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
|
|
$job->{builder}{state}{affinity}->start($job->{v}, $core);
|
|
$job->track_lock;
|
|
|
|
return $job->next_task($core);
|
|
}
|
|
|
|
# likewise, just a task that gets inserted to remind the next one
|
|
# to track "Building from scratch"
|
|
package DPB::Task::Port::Placeholder;
|
|
our @ISA = qw(DPB::Task::BasePort);
|
|
sub setup($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
$job->{pos} = tell($job->{logfh});
|
|
return $job->next_task($core);
|
|
}
|
|
|
|
package DPB::Task::Port::ShowSize;
|
|
our @ISA = qw(DPB::Task::Port);
|
|
|
|
sub want_percent($) { 0 }
|
|
|
|
sub fork($self, $core)
|
|
{
|
|
open($self->{fh}, "-|");
|
|
}
|
|
|
|
sub handle_output($, $)
|
|
{
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
my $fh = $self->{fh};
|
|
if ($core->{status} == 0) {
|
|
my $line = <$fh>;
|
|
$line = <$fh>;
|
|
if ($line =~ m/^\s*(\d+)\s+/) {
|
|
my $sz = $1;
|
|
my $job = $core->job;
|
|
my $info = DPB::Serialize::Size->write({
|
|
pkgpath => $job->{path},
|
|
pkname => $job->{v}->fullpkgname,
|
|
size => $sz });
|
|
print {$job->{builder}{logsize}} $info, "\n";
|
|
# XXX the rolling log might be shared with other dpb
|
|
# so it can be rewritten and sorted
|
|
# don't keep a handle on it, so that we always
|
|
# append new information to the correct filename
|
|
my $fh2 = $job->{builder}->logger->open('>>', $job->{builder}{state}{size_log});
|
|
print $fh2 $info."\n";
|
|
}
|
|
}
|
|
close($fh);
|
|
return 1;
|
|
}
|
|
|
|
package DPB::Task::Port::Install;
|
|
our @ISA=qw(DPB::Task::Port);
|
|
|
|
sub notime($) { 1 }
|
|
|
|
sub want_percent($) { 0 }
|
|
|
|
sub run($self, $core)
|
|
{
|
|
my $job = $core->job;
|
|
my $v = $job->{v};
|
|
|
|
$self->handle_output($job);
|
|
my @cmd = ('/usr/sbin/pkg_add', '-I');
|
|
if ($job->{builder}->{update}) {
|
|
push(@cmd, "-rqU", "-Dupdate", "-Dupdatedepends");
|
|
}
|
|
if ($job->{builder}->{forceupdate}) {
|
|
push(@cmd, "-Dinstalled");
|
|
}
|
|
if ($job->{builder}{state}{localbase} ne '/usr/local') {
|
|
push(@cmd, "-L", $job->{builder}{state}{localbase});
|
|
}
|
|
print join(' ', @cmd, $v->fullpkgname, "\n");
|
|
my $path = $job->{builder}->{fullrepo}.'/';
|
|
$core->shell->nochroot->env(TRUSTED_PKG_PATH => $path)->as_root
|
|
->exec(@cmd, $v->fullpkgname);
|
|
exit(1);
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$core->{status} = 0;
|
|
$self->SUPER::finalize($core);
|
|
return 1;
|
|
}
|
|
|
|
|
|
package DPB::Task::Port::Fetch;
|
|
our @ISA = qw(DPB::Task::Port);
|
|
|
|
sub notime($) { 1 }
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
# if there's a watch file, then we remove the current size,
|
|
# so that we DON'T take prepare into account.
|
|
my $job = $core->job;
|
|
if (defined $job->{watched}) {
|
|
$job->{watched}->reset_offset;
|
|
}
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
package DPB::Task::Port::Clean;
|
|
our @ISA = qw(DPB::Task::BasePort);
|
|
|
|
sub notime($) { 1 }
|
|
sub want_percent($) { 0 }
|
|
|
|
sub setup($task, $core)
|
|
{
|
|
my $job = $core->job;
|
|
if ($job->{builder}->should_clean($job->{v})) {
|
|
$job->{lock}->write("cleaned");
|
|
return $task;
|
|
} else {
|
|
return $job->next_task($core);
|
|
}
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$self->SUPER::finalize($core);
|
|
return 1;
|
|
}
|
|
|
|
package DPB::Task::Test;
|
|
our @ISA = qw(DPB::Task::BasePort);
|
|
|
|
# to put test results elsewhere
|
|
#sub redirect_output
|
|
#{
|
|
#}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
$self->SUPER::finalize($core);
|
|
# we always make as though we succeeded
|
|
return 1;
|
|
}
|
|
|
|
package DPB::Task::PrepareTestResults;
|
|
our @ISA = qw(DPB::Task::PrepareResults);
|
|
|
|
package DPB::Port::TaskFactory;
|
|
my $repo = {
|
|
default => 'DPB::Task::Port',
|
|
checksum => 'DPB::Task::Port::Checksum',
|
|
clean => 'DPB::Task::Port::Clean',
|
|
configure => 'DPB::Task::Port::Configure',
|
|
extract => 'DPB::Task::Port::Extract',
|
|
'show-prepare-results' => 'DPB::Task::Port::PrepareResults',
|
|
'show-prepare-test-results' => 'DPB::Task::Port::PrepareResults',
|
|
fetch => 'DPB::Task::Port::Fetch',
|
|
depends => 'DPB::Task::Port::Depends',
|
|
'show-size' => 'DPB::Task::Port::ShowSize',
|
|
junk => 'DPB::Task::Port::Uninstall',
|
|
inbetween => 'DPB::Task::Port::InBetween',
|
|
placeholder => 'DPB::Task::Port::Placeholder',
|
|
fake => 'DPB::Task::Port::Fake',
|
|
signature => 'DPB::Task::Port::Signature',
|
|
test => 'DPB::Task::Test',
|
|
};
|
|
|
|
sub create($class, $k)
|
|
{
|
|
my $fw = $repo->{$k};
|
|
$fw //= $repo->{default};
|
|
$fw->new($k);
|
|
}
|
|
|
|
# placeholder class in case we don't have a real lock
|
|
package DPB::DummyLock;
|
|
|
|
sub new($class)
|
|
{
|
|
bless {}, $class;
|
|
}
|
|
|
|
sub write($, $, $ = undef)
|
|
{
|
|
}
|
|
|
|
sub close($)
|
|
{
|
|
}
|
|
|
|
package DPB::Job::BasePort;
|
|
our @ISA = qw(DPB::Job::Watched);
|
|
|
|
use Time::HiRes;
|
|
|
|
sub new($class, %prop)
|
|
{
|
|
my $job = bless \%prop, $class;
|
|
$job->{path} = $job->{v}->fullpkgpath;
|
|
|
|
my $e = $job->{endcode};
|
|
|
|
$job->{endcode} = sub($core) {
|
|
print {$job->{logfh}} ">>> Ended at ", DPB::Util->current_ts, "\n";
|
|
print {$job->{logfh}} $job->timings, "\n";
|
|
close($job->{logfh});
|
|
&$e($core);
|
|
};
|
|
$job->{tasks} = [];
|
|
# for stuff that doesn't really lock
|
|
$job->{lock} //= DPB::DummyLock->new;
|
|
$job->set_watch($job->{builder}->logger, $job->{v});
|
|
|
|
return $job;
|
|
}
|
|
|
|
sub debug_dump($self)
|
|
{
|
|
return $self->{v}->fullpkgpath;
|
|
}
|
|
|
|
# a small wrapper that allows us to initialize things
|
|
sub next_task($self, $core)
|
|
{
|
|
my $task = shift @{$self->{tasks}};
|
|
if (defined $task) {
|
|
return $task->setup($core);
|
|
} else {
|
|
return $task;
|
|
}
|
|
}
|
|
|
|
sub save_depends($job, $l)
|
|
{
|
|
$job->{live_depends} = $l;
|
|
$job->{lock}->write("needed", join(' ', sort @$l));
|
|
}
|
|
|
|
sub save_wanted_depends($job)
|
|
{
|
|
$job->{lock}->write("wanted", join(' ', sort keys %{$job->{depends}}));
|
|
}
|
|
|
|
sub need_depends($self, $core, $with_tests)
|
|
{
|
|
my $dep = $self->{v}{info}->solve_depends($with_tests);
|
|
return 0 unless %$dep;
|
|
# XXX we are running this synchronously with other jobs on the
|
|
# same host, so we know exactly which live_depends we can reuse.
|
|
# try to see if other jobs that already have locks are enough to
|
|
# satisfy our depends, then we can completely avoid a pkg_add
|
|
my @live = ();
|
|
my %deps2 = %$dep;
|
|
for my $job ($core->same_host_jobs) {
|
|
next unless defined $job->{live_depends};
|
|
for my $d (@{$job->{live_depends}}) {
|
|
if (defined $deps2{$d}) {
|
|
delete $deps2{$d};
|
|
push(@live, $d);
|
|
}
|
|
}
|
|
}
|
|
my $c = scalar(keys %deps2);
|
|
if (!$c) {
|
|
$self->save_depends(\@live);
|
|
print {$self->{logfh}} "Avoided depends for ",
|
|
join(' ', @live), "\n";
|
|
} else {
|
|
$self->save_wanted_depends;
|
|
$self->{depends} = $dep;
|
|
}
|
|
return $c;
|
|
}
|
|
|
|
my $logsize = {};
|
|
my $times = {};
|
|
|
|
sub add_build_info($class, $pkgpath, $host, $time, $sz)
|
|
{
|
|
$logsize->{$pkgpath} = $sz;
|
|
$times->{$pkgpath} = $time;
|
|
}
|
|
|
|
|
|
sub current_task($self)
|
|
{
|
|
if (@{$self->{tasks}} > 0) {
|
|
return $self->{tasks}[0]{phase};
|
|
} else {
|
|
return "<nothing>";
|
|
}
|
|
}
|
|
|
|
sub pkgpath($self)
|
|
{
|
|
return $self->{v};
|
|
}
|
|
|
|
sub name($self)
|
|
{
|
|
return $self->{path};
|
|
}
|
|
|
|
sub description($self)
|
|
{
|
|
my $d = $self->SUPER::description;
|
|
if ($self->{nojunk}) {
|
|
$d .= '!';
|
|
}
|
|
return $d;
|
|
}
|
|
|
|
sub finished_task($self, $task)
|
|
{
|
|
push(@{$self->{done}}, $task);
|
|
}
|
|
|
|
sub finalize($self, $core)
|
|
{
|
|
if ($self->{stuck}) {
|
|
print {$self->{logfh}} $self->{stuck}, "\n";
|
|
}
|
|
$self->SUPER::finalize($core);
|
|
}
|
|
|
|
sub totaltime($self)
|
|
{
|
|
my $t = 0;
|
|
for my $plus (@{$self->{done}}) {
|
|
next if $plus->notime;
|
|
$t += $plus->elapsed;
|
|
}
|
|
$t *= $self->{parallel} if $self->{parallel};
|
|
return DPB::Util->ts2string($t);
|
|
}
|
|
|
|
sub timings($self)
|
|
{
|
|
return join('/', "max_stuck=".DPB::Util->ts2string($self->{watched}{max}), map {sprintf("%s=%.2f", $_->{phase}, $_->elapsed)} @{$self->{done}});
|
|
}
|
|
|
|
sub equates($class, $h)
|
|
{
|
|
for my $v (values %$h) {
|
|
next unless defined $logsize->{$v};
|
|
for my $w (values %$h) {
|
|
$logsize->{$w} //= $logsize->{$v};
|
|
$times->{$w} //= $logsize->{$v};
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
|
|
sub set_watch($self, $logger, $v)
|
|
{
|
|
my $expected;
|
|
for my $w ($v->build_path_list) {
|
|
if (defined $logsize->{$w}) {
|
|
$expected = $logsize->{$w};
|
|
last;
|
|
}
|
|
}
|
|
$self->{watched} = DPB::Port::Watch->new(
|
|
$logger->file($logger->log_pkgpath($v)),
|
|
$expected, $self->{offset}, $self->{started});
|
|
}
|
|
|
|
sub track_lock($self)
|
|
{
|
|
$self->{watched}->track_lock;
|
|
}
|
|
|
|
sub get_timeout($self, $core)
|
|
{
|
|
return $core->stuck_timeout;
|
|
}
|
|
|
|
sub really_watch($self, $current)
|
|
{
|
|
return "" unless defined $self->{watched};
|
|
my $diff = $self->{watched}->check_change($current);
|
|
$self->{lastdiff} //= 5;
|
|
if ($diff > $self->{lastdiff} * 2) {
|
|
$self->{lastdiff} = $diff;
|
|
return 1;
|
|
} elsif ($diff < $self->{lastdiff}) {
|
|
$self->{lastdiff} = 5;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
sub cleanup_after_fork($self)
|
|
{
|
|
$self->{lock}->close;
|
|
$self->SUPER::cleanup_after_fork;
|
|
}
|
|
|
|
sub silent_log($job, @parts)
|
|
{
|
|
my $msg = join(@parts);
|
|
my $old = $job->{logfh}->autoflush(1);
|
|
print {$job->{logfh}} $msg;
|
|
$job->{logfh}->autoflush($old);
|
|
if (defined $job->{watched}) {
|
|
$job->{watched}->adjust_by(length($msg));
|
|
}
|
|
}
|
|
|
|
package DPB::Job::Port;
|
|
our @ISA = qw(DPB::Job::BasePort);
|
|
|
|
sub new($class, @rest)
|
|
{
|
|
my $job = $class->SUPER::new(@rest);
|
|
|
|
my $v = $job->{v};
|
|
my $core = $job->{core};
|
|
my $builder = $job->{builder};
|
|
|
|
my $prop = $core->prop;
|
|
# note that lonesome *and* parallel can be specified
|
|
if ($v->{info}->has_property('lonesome')) {
|
|
$job->{lonesome} = 1;
|
|
}
|
|
if ($prop->{parallel2} && $v->{info}->has_property('parallel2')) {
|
|
$job->{parallel} = $prop->{parallel2};
|
|
} elsif ($prop->{parallel} && $v->{info}->has_property('parallel')) {
|
|
$job->{parallel} = $prop->{parallel};
|
|
}
|
|
|
|
if ($builder->checks_rebuild($v)) {
|
|
$job->add_tasks(DPB::Port::TaskFactory->create('signature'));
|
|
} else {
|
|
$job->add_normal_tasks($builder->should_clean($v), $core);
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
sub new_junk_only($class, @rest)
|
|
{
|
|
my $job = $class->SUPER::new(@rest);
|
|
my $fh2 = $job->{builder}->logger->append("junk");
|
|
print $fh2 "$$@", CORE::time(), ": ", $job->{core}->hostname,
|
|
": forced junking -> $job->{path}\n";
|
|
$job->add_tasks(DPB::Port::TaskFactory->create('junk'));
|
|
return $job;
|
|
}
|
|
|
|
|
|
sub add_normal_tasks($self, $should_clean, $core)
|
|
{
|
|
|
|
my @todo;
|
|
my $builder = $self->{builder};
|
|
my $hostprop = $core->prop;
|
|
my $small = 0;
|
|
if (defined $times->{$self->{v}} &&
|
|
$times->{$self->{v}} < $hostprop->{small_timeout}) {
|
|
$small = 1;
|
|
}
|
|
if ($builder->{clean}) {
|
|
push(@todo, 'clean');
|
|
}
|
|
$hostprop->{junk_count} //= 0;
|
|
$hostprop->{depends_count} //= 0;
|
|
$hostprop->{ports_count} //= 0;
|
|
my $c = $self->need_depends($core, 0);
|
|
$hostprop->{ports_count}++;
|
|
$hostprop->{depends_count} += $c;
|
|
my $junk = DPB::Junk->want($core, $self);
|
|
if ($junk == 2) {
|
|
push(@todo, 'junk');
|
|
my $fh = $self->{builder}->logger->append("junk");
|
|
print $fh "$$@", CORE::time(), ": ", $core->hostname,
|
|
": forced junking -> $self->{path}\n";
|
|
}
|
|
if ($c) {
|
|
$hostprop->{junk_count}++;
|
|
push(@todo, 'depends');
|
|
# depends only install dependencies, stuff that have
|
|
# an extra :patch/:configure stage need to complete prepare !
|
|
if (exists $self->{v}{info}{BEXTRA}) {
|
|
push(@todo, 'placeholder');
|
|
push(@todo, 'prepare');
|
|
}
|
|
push(@todo, 'show-prepare-results');
|
|
}
|
|
# gc stuff we will no longer need
|
|
delete $self->{v}{info}{solved};
|
|
if ($junk == 1) {
|
|
my $fh = $self->{builder}->logger->append("junk");
|
|
print $fh "$$@", CORE::time(), ": ", $core->hostname,
|
|
": depends=$hostprop->{depends_count} ",
|
|
" ports=$hostprop->{ports_count} ",
|
|
" junk=$hostprop->{junk_count} -> $self->{path}\n";
|
|
push(@todo, 'junk');
|
|
}
|
|
if ($builder->{fetch}) {
|
|
push(@todo, qw(checksum));
|
|
} else {
|
|
push(@todo, qw(fetch));
|
|
}
|
|
|
|
push(@todo, qw(inbetween));
|
|
my $nojunk = $self->{v}{info}->has_property("nojunk");
|
|
my $nojunk2 = $self->{v}{info}->has_property("noconfigurejunk");
|
|
if (!$c) {
|
|
push(@todo, 'placeholder');
|
|
}
|
|
if ($nojunk || $nojunk2) {
|
|
push(@todo, qw(extract));
|
|
}
|
|
if (!$small || $nojunk2) {
|
|
push(@todo, qw(patch configure));
|
|
}
|
|
if (!$nojunk && $nojunk2) {
|
|
$self->{noconfigurejunk} = 1;
|
|
}
|
|
push(@todo, qw(build));
|
|
|
|
if (!$small) {
|
|
push(@todo, qw(fake));
|
|
}
|
|
push(@todo, qw(package));
|
|
if ($builder->want_size($self->{v}, $core)) {
|
|
push @todo, 'show-size';
|
|
}
|
|
if ($self->{v}{info}->want_tests) {
|
|
$should_clean = 0;
|
|
}
|
|
if ($should_clean) {
|
|
push @todo, 'clean';
|
|
}
|
|
$self->add_tasks(map {DPB::Port::TaskFactory->create($_)} @todo);
|
|
}
|
|
|
|
sub wake_others($self, $core)
|
|
{
|
|
my ($minjob, $minpid);
|
|
$core->walk_same_host_jobs(
|
|
sub($pid, $job) {
|
|
return unless exists $job->{wakemeup};
|
|
if (!defined $minjob ||
|
|
$job->{wakemeup} < $minjob->{wakemeup}) {
|
|
($minjob, $minpid) = ($job, $pid);
|
|
}
|
|
});
|
|
if (defined $minjob) {
|
|
local $> = 0;
|
|
kill IO => $minpid;
|
|
print {$core->job->{logfh}} "Woken up $minjob->{path}\n";
|
|
}
|
|
}
|
|
|
|
package DPB::Job::Port::Test;
|
|
our @ISA = qw(DPB::Job::BasePort);
|
|
sub new($class, @rest)
|
|
{
|
|
my $job = $class->SUPER::new(@rest);
|
|
|
|
$job->add_test_tasks($job->{core});
|
|
|
|
return $job;
|
|
}
|
|
|
|
sub add_test_tasks($self, $core)
|
|
{
|
|
my @todo;
|
|
|
|
my $c = $self->need_depends($core, 1);
|
|
if ($c) {
|
|
push(@todo, qw(depends show-prepare-test-results));
|
|
}
|
|
delete $self->{v}{info}{solved};
|
|
push(@todo, qw(test clean));
|
|
|
|
$self->add_tasks(map {DPB::Port::TaskFactory->create($_)} @todo);
|
|
}
|
|
|
|
package DPB::Job::Port::Install;
|
|
our @ISA = qw(DPB::Job::BasePort);
|
|
|
|
sub new($class, @rest)
|
|
{
|
|
my $job = $class->SUPER::new(@rest);
|
|
|
|
push(@{$job->{tasks}},
|
|
DPB::Task::Port::Install->new('install'));
|
|
return $job;
|
|
}
|
|
|
|
package DPB::Job::Port::Wipe;
|
|
our @ISA = qw(DPB::Job::BasePort);
|
|
|
|
sub new($class, @rest)
|
|
{
|
|
my $job = $class->SUPER::new(@rest);
|
|
|
|
push(@{$job->{tasks}},
|
|
DPB::Task::Port::Clean->new('clean'));
|
|
return $job;
|
|
}
|
|
|
|
|
|
package DPB::Port::Watch;
|
|
our @ISA = qw(DPB::Watch);
|
|
|
|
# set things up so that we only track Awaiting lock once.
|
|
|
|
sub track_lock($self)
|
|
{
|
|
$self->{tracked} //= 1;
|
|
}
|
|
|
|
|
|
sub tweak_msg($self, $rmsg)
|
|
{
|
|
if (defined $self->{tracked} && $self->{tracked} == 1) {
|
|
# we tracked already, so never do it again
|
|
$self->{tracked} = 0;
|
|
# optimistic grab of last line of file
|
|
my $line = $self->peek(150);
|
|
chomp $line;
|
|
if ($line =~ m/Awaiting lock\s+(.*)/) {
|
|
$self->{override} = " stuck on $1";
|
|
}
|
|
}
|
|
if (defined $self->{override}) {
|
|
$$rmsg = $self->{override};
|
|
}
|
|
}
|
|
|
|
sub frozen_message($self, $diff)
|
|
{
|
|
my $msg = $self->SUPER::frozen_message($diff);
|
|
if ($msg eq "") {
|
|
delete $self->{override};
|
|
} else {
|
|
$self->tweak_msg(\$msg);
|
|
}
|
|
return $msg;
|
|
}
|
|
|
|
1;
|