benoit/helper-scripts
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Ajout de apt-mirror et des patchs appliqués

Browse Source
This commit is contained in:
root
2020-04-29 02:31:32 +02:00
parent 1a7578ef26
commit 6bf601f8f0
7 changed files with 4898 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apt-mirror/apt-mirror
Executable
+1204
View File
File diff suppressed because it is too large Load Diff
apt-mirror/apt-mirror-111
+1110
View File
File diff suppressed because it is too large Load Diff
apt-mirror/apt-mirror-131
+1158
View File
File diff suppressed because it is too large Load Diff
apt-mirror/apt-mirror.orig
Executable
+1066
View File
File diff suppressed because it is too large Load Diff
apt-mirror/patch-111
+237
View File
@@ -0,0 +1,237 @@
--- /usr/bin/apt-mirror.old 2017-05-29 14:02:33.000000000 +0200
+++ /usr/bin/apt-mirror 2020-04-11 02:09:10.913855091 +0200
@@ -8,7 +8,7 @@
=head1 SYNOPSIS
-apt-mirror [configfile]
+apt-mirror [--[no-]progress] [--verbose] [configfile]
=head1 DESCRIPTION
@@ -16,14 +16,34 @@
the whole Debian GNU/Linux distribution or any other apt sources.
Main features:
- * It uses a config similar to APT's F<sources.list>
- * It's fully pool compliant
- * It supports multithreaded downloading
- * It supports multiple architectures at the same time
- * It can automatically remove unneeded files
- * It works well on an overloaded Internet connection
- * It never produces an inconsistent mirror including while mirroring
- * It works on all POSIX compliant systems with Perl and wget
+
+=over
+
+=item *
+It uses a config similar to APT's F<sources.list>
+
+=item *
+It's fully pool compliant
+
+=item *
+It supports multithreaded downloading
+
+=item *
+It supports multiple architectures at the same time
+
+=item *
+It can automatically remove unneeded files
+
+=item *
+It works well on an overloaded Internet connection
+
+=item *
+It never produces an inconsistent mirror including while mirroring
+
+=item *
+It works on all POSIX compliant systems with Perl and wget
+
+=back
=head1 COMMENTS
@@ -94,6 +114,7 @@
use File::Path qw(make_path);
use File::Basename;
use Fcntl qw(:flock);
+use Getopt::Long;
my $config_file;
@@ -128,11 +149,17 @@
my @childrens = ();
my %skipclean = ();
my %clean_directory = ();
+my $verbose = 0;
+my $progress = 1;
######################################################################################
## Setting up $config_file variable
$config_file = "/etc/apt/mirror.list"; # Default value
+GetOptions('verbose|v+', \$verbose,
+ 'progress|p!', \$progress,
+ ) or die "Usage: apt-mirror [--verbose] [--[no-]progress] [configfile]\n";
+
if ( $_ = shift )
{
die("apt-mirror: invalid config file specified") unless -e $_;
@@ -256,7 +283,9 @@
open URLS, ">" . get_variable("var_path") . "/$stage-urls.$i" or die("apt-mirror: can't write to intermediate file ($stage-urls.$i)");
foreach (@part) { print URLS "$_\n"; }
close URLS or die("apt-mirror: can't close intermediate file ($stage-urls.$i)");
-
+ if ($verbose >= 2) {
+ print join("\n ", "Downloading batch $i:", @part), "\n";
+ }
$pid = fork();
die("apt-mirror: can't do fork in download_urls") if !defined($pid);
@@ -274,14 +303,17 @@
$nthreads--;
}
- print "Begin time: " . localtime() . "\n[" . scalar(@childrens) . "]... ";
+ print "Begin time: " . localtime() . "\n[" . scalar(@childrens) . "]... "
+ if $progress;
while ( scalar @childrens )
{
my $dead = wait();
@childrens = grep { $_ != $dead } @childrens;
- print "[" . scalar(@childrens) . "]... ";
+ print "[" . scalar(@childrens) . "]... "
+ if $progress;
}
- print "\nEnd time: " . localtime() . "\n\n";
+ print "\nEnd time: " . localtime() . "\n\n"
+ if $progress;
}
## Parse config
@@ -600,12 +632,13 @@
close STREAM;
}
-print "Processing translation indexes: [";
+print "Processing translation indexes: ["
+ if $progress;
foreach (@config_binaries)
{
my ( $arch, $uri, $distribution, @components ) = @{$_};
- print "T";
+ print "T" if $progress;
if (@components)
{
$url = $uri . "/dists/" . $distribution . "/";
@@ -618,7 +651,7 @@
}
}
-print "]\n\n";
+print "]\n\n" if $progress;
push( @index_urls, sort keys %urls_to_download );
download_urls( "translation", sort keys %urls_to_download );
@@ -691,12 +724,13 @@
}
}
-print "Processing DEP-11 indexes: [";
+print "Processing DEP-11 indexes: ["
+ if $progress;
foreach (@config_binaries)
{
my ( $arch, $uri, $distribution, @components ) = @{$_};
- print "D";
+ print "D" if $progress;
if (@components)
{
$url = $uri . "/dists/" . $distribution . "/";
@@ -709,7 +743,7 @@
}
}
-print "]\n\n";
+print "]\n\n" if $progress;
push( @index_urls, sort keys %urls_to_download );
download_urls( "dep11", sort keys %urls_to_download );
@@ -815,12 +849,16 @@
{ # Packages index
$skipclean{ remove_double_slashes( $path . "/" . $lines{"Filename:"} ) } = 1;
print FILES_ALL remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n";
+ print "ALL: " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n"
+ if $verbose >= 3;
print FILES_MD5 $lines{"MD5sum:"} . " " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n" if defined $lines{"MD5sum:"};
print FILES_SHA1 $lines{"SHA1:"} . " " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n" if defined $lines{"SHA1:"};
print FILES_SHA256 $lines{"SHA256:"} . " " . remove_double_slashes( $path . "/" . $lines{"Filename:"} ) . "\n" if defined $lines{"SHA256:"};
if ( need_update( $mirror . "/" . $lines{"Filename:"}, $lines{"Size:"} ) )
{
print FILES_NEW remove_double_slashes( $uri . "/" . $lines{"Filename:"} ) . "\n";
+ print "NEW: " . remove_double_slashes( $uri . "/" . $lines{"Filename:"} ) . "\n"
+ if $verbose >= 1;
add_url_to_download( $uri . "/" . $lines{"Filename:"}, $lines{"Size:"} );
}
}
@@ -833,10 +871,14 @@
die("apt-mirror: invalid Sources format") if @file != 3;
$skipclean{ remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) } = 1;
print FILES_ALL remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n";
+ print "ALL: " . remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n"
+ if $verbose >= 3;
print FILES_MD5 $file[0] . " " . remove_double_slashes( $path . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n";
if ( need_update( $mirror . "/" . $lines{"Directory:"} . "/" . $file[2], $file[1] ) )
{
print FILES_NEW remove_double_slashes( $uri . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n";
+ print "NEW: " . remove_double_slashes( $uri . "/" . $lines{"Directory:"} . "/" . $file[2] ) . "\n"
+ if $verbose >= 1;
add_url_to_download( $uri . "/" . $lines{"Directory:"} . "/" . $file[2], $file[1] );
}
}
@@ -846,12 +888,13 @@
close STREAM;
}
-print "Processing indexes: [";
+print "Processing indexes: ["
+ if $progress;
foreach (@config_sources)
{
my ( $uri, $distribution, @components ) = @{$_};
- print "S";
+ print "S" if $progress;
if (@components)
{
my $component;
@@ -869,7 +912,7 @@
foreach (@config_binaries)
{
my ( $arch, $uri, $distribution, @components ) = @{$_};
- print "P";
+ print "P" if $progress;
if (@components)
{
my $component;
@@ -886,7 +929,8 @@
clear_stat_cache();
-print "]\n\n";
+print "]\n\n"
+ if $progress;
close FILES_ALL;
close FILES_NEW;
@@ -1019,6 +1063,7 @@
foreach (@rm_files)
{
print CLEAN "rm -f '$_'\n";
+ print " $_\n" if $verbose >= 1;
print CLEAN "echo -n '[" . int( 100 * $i / $total ) . "\%]'\n" unless $i % 500;
print CLEAN "echo -n .\n" unless $i % 10;
$i++;
apt-mirror/patch-130
+21
View File
@@ -0,0 +1,21 @@
--- /usr/bin/apt-mirror.old 2017-05-29 14:02:33.000000000 +0200
+++ /usr/bin/apt-mirror 2020-04-11 01:42:20.959628172 +0200
@@ -520,7 +520,7 @@
if ( @parts == 3 )
{
my ( $sha1, $size, $filename ) = @parts;
- if ( $filename =~ m{^$component/i18n/Translation-[^./]*\.bz2$} )
+ if ( $filename =~ m{^$component/i18n/Translation-[^./]*\.(gz|bz2|xz)$} )
{
add_url_to_download( $dist_uri . $filename, $size );
}
@@ -826,7 +826,8 @@
}
else
{ # Sources index
- foreach ( split( /\n/, $lines{"Files:"} ) )
+ # foreach ( split( /\n/, $lines{"Files:"} ) )
+ foreach ( split( /\n/, $lines{"Checksums-Sha256:"} ) )
{
next if $_ eq '';
my @file = split;
apt-mirror/patch-131
+102
View File
@@ -0,0 +1,102 @@
--- /usr/bin/apt-mirror.old 2017-05-29 14:02:33.000000000 +0200
+++ /usr/bin/apt-mirror 2020-04-11 02:08:01.820901199 +0200
@@ -722,6 +722,99 @@
}
######################################################################################
+## by-hash SHA256 files download
+
+%urls_to_download = ();
+
+sub find_by_hash_sha256_files_in_release
+{
+ # Look in the dists/$DIST/Release file for the by-hash SHA256 files that belong
+ # to the given component and architecture.
+
+ my $dist_uri = shift;
+ my $component = shift;
+ my $arch = shift;
+ my ( $release_uri, $release_path, $line ) = '';
+
+ $release_uri = $dist_uri . "Release";
+ $release_path = get_variable("skel_path") . "/" . sanitise_uri($release_uri);
+
+ unless ( open STREAM, "<$release_path" )
+ {
+ warn( "Failed to open Release file from " . $release_uri );
+ return;
+ }
+
+ my $checksums = 0;
+ while ( $line = <STREAM> )
+ {
+ chomp $line;
+ if ($checksums)
+ {
+ if ( $line =~ /^ +(.*)$/ )
+ {
+ my @parts = split( / +/, $1 );
+ if ( @parts == 3 )
+ {
+ my ( $sha256, $size, $filename ) = @parts;
+ my $dirname = dirname($filename);
+ my $sha256_filename = '/'.$dirname.'/by-hash/SHA256/'.$sha256;
+ {
+ add_url_to_download( $dist_uri . $sha256_filename );
+ }
+ }
+ else
+ {
+ warn("Malformed checksum line \"$1\" in $release_uri");
+ }
+ }
+ else
+ {
+ $checksums = 0;
+ }
+ }
+ if ( not $checksums )
+ {
+ if ( $line eq "SHA256:" )
+ {
+ $checksums = 1;
+ }
+ }
+ }
+}
+
+print "Processing SHA256 by-hash files ["
+ if $progress;
+
+foreach (@config_binaries)
+{
+ my ( $arch, $uri, $distribution, @components ) = @{$_};
+ print "D" if $progress;
+ if (@components)
+ {
+ $url = $uri . "/dists/" . $distribution . "/";
+
+ my $component;
+ foreach $component (@components)
+ {
+ find_by_hash_sha256_files_in_release( $url, $component, $arch );
+ }
+ }
+}
+
+print "]\n\n" if $progress;
+
+push( @index_urls, sort keys %urls_to_download );
+download_urls( "by-hash-SHA256", sort keys %urls_to_download );
+
+foreach ( keys %urls_to_download )
+{
+ s[^(\w+)://][];
+ s[~][%7E]g if get_variable("_tilde");
+ $skipclean{$_} = 1;
+}
+
+######################################################################################
## Main download preparations
%urls_to_download = ();