Commit af963e1f authored by gregor herrmann's avatar gregor herrmann

dpt-salsa: merge githashes() into mrconfig().

Looping only once over all repos saves some time as the bottleneck is the
Gitlab API at salsa.debian.org, and although only mrconfig() was run from
our group's .mrconfig, de facto githashes() was required as well.

Note that githashes() now just calls mrconfig(), and that manual calls to
`dpt salsa githashes' should be removed from cronjobs or personal
~/.mrconfig files.
parent 9e17df5d
......@@ -22,7 +22,7 @@ Build-Depends-Indep: dpkg-dev (>= 1.17.0),
libmime-lite-perl,
libnet-github-perl,
libparse-debianchangelog-perl,
libparallel-forkmanager-perl,
libparallel-forkmanager-perl (>= 0.7.6),
libproc-invokeeditor-perl,
librt-client-rest-perl,
libsoap-lite-perl,
......@@ -78,7 +78,7 @@ Recommends: autodep8,
libmime-lite-perl,
libmodule-inspector-perl,
libnet-github-perl,
libparallel-forkmanager-perl,
libparallel-forkmanager-perl (>= 0.7.6),
libsoap-lite-perl,
libterm-readline-gnu-perl,
libwww-mechanize-perl,
......
......@@ -130,7 +130,9 @@ if ( $command eq 'version' ) {
} elsif ( $command eq 'mrconfig' ) {
mrconfig();
} elsif ( $command eq 'githashes' ) {
githashes();
say STDERR
"githashes() is deprecated and its functionality integrated into mrconfig(). Calling mrconfig() now.";
mrconfig();
} elsif ( $command eq 'help' ) {
pod2usage(1);
} else {
......@@ -417,75 +419,81 @@ sub changerepo {
## mrconfig
sub mrconfig {
die 'DPT_PACKAGES not set.' unless $config{packages};
my $outdir = File::Spec->catdir( $config{packages}, '..' );
die "Can't find directory '$outdir'." unless -d $outdir;
my $mrconfig = File::Spec->catfile( $outdir, '.mrconfig' );
die "Can't find '.mrconfig' in '$outdir'" unless -f $mrconfig;
my $outfile = File::Spec->catfile( $outdir, '.mrconfig.packages' );
open( my $fh, '>', "$outfile.new" )
or die "Can't open '$outfile.new': $!";
# mrconfig
my $mroutdir = File::Spec->catdir( $config{packages}, '..' );
die "Can't find directory '$mroutdir'." unless -d $mroutdir;
my $paginator = $api->paginator(
'group_projects',
$config{perl_team_modules_packages_id},
{ order_by => 'name', sort => 'asc' }
);
while ( my $repo = trypaginator($paginator) ) {
my $reponame = $repo->{name};
my $mrconfig = File::Spec->catfile( $mroutdir, '.mrconfig' );
die "Can't find '.mrconfig' in '$mroutdir'" unless -f $mrconfig;
# stdout
say "[packages/$reponame]";
say "checkout = git_checkout $reponame\n";
my $mroutfile = File::Spec->catfile( $mroutdir, '.mrconfig.packages' );
open( my $mrfh, '>', "$mroutfile.new" )
or die "Can't open '$mroutfile.new': $!";
my @mrstanzas;
# file
say $fh "[packages/$reponame]";
say $fh "checkout = git_checkout $reponame\n";
}
rename "$outfile.new", "$outfile"
or die "Can't rename '$outfile.new' to '$outfile': $!";
close $fh;
}
# githashes
my $hashesoutdir
= File::Spec->catdir( $config{packages}, '..', '.git.hashes' );
-d $hashesoutdir
or mkdir($hashesoutdir)
or die "Can't find or create directory '$hashesoutdir': $!";
## githashes
sub githashes {
die 'DPT_PACKAGES not set.' unless $config{packages};
my $outdir = File::Spec->catdir( $config{packages}, '..', '.git.hashes' );
-d $outdir
or mkdir($outdir)
or die "Can't find or create directory '$outdir': $!";
my @currenthashesfiles = glob("$hashesoutdir/*");
s{$hashesoutdir/}{}o for @currenthashesfiles;
my %currenthashesfiles = map( ( $_ => 1 ), @currenthashesfiles );
# parallel
my $jobs = $opts{parallel} || 1;
$jobs = 0 unless $jobs =~ /^\d{1,3}$/ and $jobs > 1;
my $pm;
if ($jobs) {
eval {
require Parallel::ForkManager;
Parallel::ForkManager->VERSION('0.7.6');
$pm = Parallel::ForkManager->new($jobs);
$pm->run_on_finish(
sub {
my ( $pid, $exit_code, $ident, $exit_signal, $core_dump,
$data_structure_reference )
= @_;
push @mrstanzas, ${$data_structure_reference}
if defined($data_structure_reference);
}
);
}
or warn "Parallel::ForkManager not found. "
or warn "Parallel::ForkManager >= 0.7.6 not found. "
. " apt install libparallel-forkmanager ?\n"
. "Continuing in a single process.\n";
}
my @currentfiles = glob("$outdir/*");
s{$outdir/}{}o for @currentfiles;
my %currentfiles = map( ( $_ => 1 ), @currentfiles );
my $paginator = $api->paginator(
'group_projects',
$config{perl_team_modules_packages_id},
{ order_by => 'name', sort => 'asc' }
);
while ( my $repo = trypaginator($paginator) ) {
delete $currentfiles{ $repo->{name} };
my $reponame = $repo->{name};
delete $currenthashesfiles{$reponame};
if ($pm) {
$pm->start and next;
}
my ( $repotags, $repobranches, $repohashes );
# mrconfig.
my $mrstanza
= "[packages/$reponame]\ncheckout = git_checkout $reponame\n\n";
# without Parallel::ForkManager or $jobs=0, output directly
# with Parallel::ForkManager, return $mrstanza at the end
if ( !$pm ) {
## stdout
say $mrstanza;
## file
say $mrfh $mrstanza;
}
# githashes
my ( $repotags, $repobranches, $repohashes );
# "try" api calls so we don't die in the loop
my ( $tags, $branches );
# disable tags retrieval to save time
......@@ -499,19 +507,34 @@ sub githashes {
$repohashes->{branches} = $repobranches if $repobranches;
# $repohashes->{trunk} seems to be a PET artifact and not used by compare-hashes
my $outfile = File::Spec->catfile( $outdir, $repo->{name} );
open( my $fh, '>', "$outfile.new" )
or die "Can't open '$outfile.new': $!";
print $fh prettyjson($repohashes);
close $fh;
rename "$outfile.new", "$outfile"
or die "Can't rename '$outfile.new' to '$outfile': $!";
$pm->finish if $pm;
my $hashesoutfile = File::Spec->catfile( $hashesoutdir, $reponame );
open( my $hashesfh, '>', "$hashesoutfile.new" )
or die "Can't open '$hashesoutfile.new': $!";
print $hashesfh prettyjson($repohashes);
close $hashesfh;
rename "$hashesoutfile.new", "$hashesoutfile"
or die
"Can't rename '$hashesoutfile.new' to '$hashesoutfile': $!";
$pm->finish( 0, \$mrstanza ) if $pm;
}
$pm->wait_all_children if $pm;
unlink "$outdir/$_" for keys %currentfiles;
# mrconfig
# parallel case
if ($pm) {
## stdout
say @mrstanzas;
## file
say $mrfh @mrstanzas;
}
close $mrfh;
rename "$mroutfile.new", "$mroutfile"
or die "Can't rename '$mroutfile.new' to '$mroutfile': $!";
# githashes
unlink "$hashesoutdir/$_" for keys %currenthashesfiles;
}
# helper functions
......@@ -813,21 +836,33 @@ internal use.
=head2 others
=head3 I<mrconfig>
=head3 I<mrconfig> [--parallel N] [-j N]
Helper for creating
Helper for creating a F<.mrconfig.packages> file in the local clone of
=over
=item *
a F<.mrconfig.packages> file in the local clone of
C<meta.git> for all active packages of the I<modules> subgroup of the
I<perl-team> group. Also writes to stdout which can be included from
F<.mrconfig>.
=head3 I<githashes> [--parallel N] [-j N]
=item *
Helper for creating F<.git.hashes/PKGNAME> files in the local clone of
F<.git.hashes/PKGNAME> files in the local clone of
C<meta.git> for all active packages of the I<modules> subgroup of the
I<perl-team> group which are then used by B<compare-hashes> in F<.mrconfig>.
=back
With C<--parallel> L<Parallel::ForkManager> is employed for parallelism.
=head3 I<githashes>
DEPRECATED. Functionality merged into C<mrconfig()>.
=head3 I<current_user> [--json]
Outputs information about the user whose I<GitLab> token is used.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment