From c6f6050ad9716117deda0285572545a2a1621d6a Mon Sep 17 00:00:00 2001 From: Jan Kratochvil Date: Thu, 16 Jul 2020 20:18:25 +0200 Subject: [PATCH] init --- debuginfosize | 59 ++++++++++++++++++++ run | 10 ++++ runone | 91 +++++++++++++++++++++++++++++++ stat | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 331 insertions(+) create mode 100755 debuginfosize create mode 100755 run create mode 100755 runone create mode 100755 stat diff --git a/debuginfosize b/debuginfosize new file mode 100755 index 0000000..7778ffb --- /dev/null +++ b/debuginfosize @@ -0,0 +1,59 @@ +#! /bin/bash +set -e +# build/znc-1.8.1-2.fc33.src.rpm.rpms +[ $# -eq 1 ] +which readelf >/dev/null +srcrpm=$1 +test -e $srcrpm.rpms +test ! -e $srcrpm.rpmsizes +test ! -e $srcrpm.rpms.d +mkdir $srcrpm.rpms.d +pushd $srcrpm.rpms.d >/dev/null + rpmsbase=../../`basename $srcrpm` + dwzrpms=$rpmsbase.rpms + dtrpms=$rpmsbase.dt.rpm.rpms + mkdir empty + cd empty + if [ -z "$(find $dwzrpms $dtrpms -maxdepth 1 -type f)" ];then + popd >/dev/null + rm -rf $srcrpm.rpms.d + echo "0 0" >$srcrpm.rpmsizes + exit 0 + fi + cd .. + function extract + { + for i in $1/*-debuginfo*.rpm;do + test ! -e $i.tgz + rpm2archive $i + test -e $i.tgz + tar xzf $i.tgz + rm -f $i.tgz + done + } + mkdir dwz;(cd dwz;extract $dwzrpms) + mkdir dt;(cd dt;extract $dtrpms) + dwzcommon="$(find dwz/usr/lib/debug/.dwz/ -type f -printf "%s" 2>/dev/null || :)" + if [ -z "$dwzcommon" ];then + dwzcommon=0 + fi + dwztot=0 + dttot=0 + pushd dwz >/dev/null + for i in `find -type f -name "*.debug*"`;do + dwz=$(find $i -maxdepth 0 -printf "%s") + dt=$(find ../dt/$i -maxdepth 0 -printf "%s") + if readelf -WS $i 2>/dev/null|fgrep -q .gnu_debugaltlink;then + if [ $dwzcommon -eq 0 ];then + echo >&2 Excessive .gnu_debugaltlink: $i + exit 1 + fi + dwz=$[$dwz+$dwzcommon] + fi + dwztot=$[$dwztot+$dwz] + dttot=$[$dttot+$dt] + done + popd >/dev/null +popd >/dev/null +rm -rf $srcrpm.rpms.d +echo "$dwztot $dttot" >$srcrpm.rpmsizes diff --git a/run b/run new file mode 100755 index 0000000..bd9718b --- /dev/null +++ b/run @@ -0,0 +1,10 @@ +#! /bin/bash +set -ex +renice 19 -p $$ +ionice -c3 -p $$ +#repos=fedora-source,updates-source +repos=rawhide-source +#all="$(dnf -C --disablerepo='*' --enablerepo=$repos repoquery -a --arch src --qf '%{name}-%{version}-%{release}'|shuf)" +#all="$(dnf -C --disablerepo='*' --enablerepo=$repos repoquery -a --arch src --qf '%{size} %{name}-%{version}-%{release}'|sort -nr|awk '{print $2}')" +all=$(cat part|shuf) +parallel -j8 -l8 ./runone $repos -- $all diff --git a/runone b/runone new file mode 100755 index 0000000..0eac9e0 --- /dev/null +++ b/runone @@ -0,0 +1,91 @@ +#! /bin/bash +set -ex +repos=$1 +shift +mock="mock -v -r fedora-rawhide-x86_64 --uniqueext=mass$$ --no-bootstrap-chroot --disable-plugin=yum_cache" +mkdir -p build +cd build +for pkg in "$@";do + #dnf --disablerepo='*' --enablerepo=$repos repoquery --requires "$pkg"|egrep '^(gcc|clang)' || continue + srcrpm="$pkg.src.rpm" + if [ -e "$srcrpm" ];then + continue + fi + rm -rf $srcrpm* + exec >>$srcrpm.log 2>&1 + dnf --disablerepo='*' --enablerepo=$repos download --source "$pkg" + if [ ! -e $srcrpm ];then + touch $srcrpm.downloadfail + continue + fi + #rpm -q --requires -p $srcrpm|egrep '^(gcc|clang)' || continue + rpm2archive $srcrpm + rm -rf $srcrpm.d + mkdir $srcrpm.d + cd $srcrpm.d + tar xzf ../$srcrpm.tgz + rm -f ../$srcrpm.tgz + name=$(echo "$pkg"|sed 's/-[^-]*-[^-]*$//') + if [ ! -e $name.spec ];then + touch ../$srcrpm.specfail + continue + fi + (echo '%global _find_debuginfo_dwz_opts %{nil}' + echo '%global optflags %(echo %{optflags}|sed "s/ -g / -g -fdebug-types-section /")' + sed 's/^%\(\(define\|global\) *_find_debuginfo_dwz_opts\)/\1/' + ) <$name.spec >$name.spec.new + cp -p $name.spec ../$srcrpm.spec + cp -p $name.spec.new ../$srcrpm.spec.new + diff -u $name.spec $name.spec.new || : + mv -f $name.spec.new $name.spec + if ! $mock --resultdir=$PWD/resultdir --buildsrpm --spec $name.spec --sources $PWD;then + touch ../$srcrpm.bsmockfail + continue + fi + if [ $[$(find resultdir -name "*.src.rpm"|wc -l)] -ne 1 ];then + touch ../$srcrpm.bsmockefail + continue + fi + # if not pkg.endswith('.rpm'): log.error("%s doesn't appear to be an rpm - skipping", pkg) + mv -f resultdir/*.src.rpm ../$srcrpm.dt.rpm + cd .. + test -e $srcrpm.dt.rpm + rm -rf $srcrpm.d + for ext in "" ".dt.rpm";do + resultdir=$PWD/$srcrpm$ext.resultdir + test ! -e $resultdir + set +e + # --chain instead of --rebuild as otherwise -a does not work + $mock --resultdir=$resultdir --nocheck -a https://people.redhat.com/jkratoch/dwzrepo/ --chain $srcrpm$ext &>$srcrpm$ext.mocklog + rc=$? + set -e + test -e $resultdir + #rm -f $srcrpm$ext + if [ $rc -ne 0 ];then + touch $srcrpm$ext.mockfail + break + fi + # --chain ignores --resultdir + resultdirsrc="$(sed -n 's/^INFO: Results out to: //p' $srcrpm$ext.mocklog)" + if [ ! -e $resultdirsrc/*/root.log ];then + touch $srcrpm$ext.mockrfail + break + fi + # With --chain $resultdir contains just some small *.log files. + rm -rf $resultdir + mv `dirname $resultdirsrc/*/root.log` $resultdir + echo >$srcrpm$ext.rpmsize $(du -sbc /dev/null $resultdir/*-debuginfo*.rpm|tail -n1|awk '{print $1}') + size=0 + for i in $resultdir/*-debuginfo*.rpm;do + if [ -e "$i" ];then + sizethis="$(rpm -q --qf '%{size}' -p $i)" + size=$[$size+$sizethis] + fi + done + echo >$srcrpm$ext.filesize $size + mkdir $srcrpm$ext.rpms + mv -f $resultdir/*-debuginfo*.rpm $srcrpm$ext.rpms || : + rm -rf $resultdir + done + touch $srcrpm.done +done diff --git a/stat b/stat new file mode 100755 index 0000000..a209c70 --- /dev/null +++ b/stat @@ -0,0 +1,171 @@ +#! /usr/bin/perl +use strict; +use warnings; + +$|=1; +sub readfile { + my($fname)=@_; + local *F; + open F,"$fname" or die $fname; + local $/=undef(); + defined(my $r=) or die $fname; + close F or die $fname; + return $r; +} +sub writefile { + my($fname,$content)=@_; + local *F; + open F,">$fname" or die $fname; + print F $content or die $fname; + close F or die $fname; +} + +my $mockfail=0; +my $dtmockfail=0; +my $emptybyfile=0; +my $dwzwinbyfile=0; +my $dtwinbyfile=0; +my $equalbyfile=0; +my $emptybyrpm=0; +my $dwzwinbyrpm=0; +my $dtwinbyrpm=0; +my $equalbyrpm=0; +my $dwzfilesizetot=0; +my $dtfilesizetot=0; +my $dwzrpmsizetot=0; +my $dtrpmsizetot=0; +my $done=0; +my %filestat; +my @srcrpms; +sub onesrcrpm($) { + my($srcrpm)=@_; +# print "$srcrpm\n"; + if (-e "$srcrpm.mockfail") { + die if -e "$srcrpm.dt.rpm.mockfail"; + warn "mockfail: $srcrpm"; + ++$mockfail; + return; + } + if (-e "$srcrpm.dt.rpm.mockfail") { + warn "dtmockfail: $srcrpm"; + ++$dtmockfail; + return; + } + my $dwzfilesize=0+readfile "$srcrpm.filesize"; + my $dwzrpmsize=0+readfile "$srcrpm.rpmsize"; + my $dtfilesize=0+readfile "$srcrpm.dt.rpm.filesize"; + my $dtrpmsize=0+readfile "$srcrpm.dt.rpm.rpmsize"; + $dwzfilesizetot+=$dwzfilesize; + $dtfilesizetot+=$dtfilesize; + $dwzrpmsizetot+=$dwzrpmsize; + $dtrpmsizetot+=$dtrpmsize; + if (($dwzfilesize>=$dtfilesize)!=($dwzrpmsize>=$dtrpmsize)) { + #warn "file vs. rpm size do not match (dwzfilesize=$dwzfilesize dtfilesize=$dtfilesize dwzrpmsize=$dwzrpmsize dtrpmsize=$dtrpmsize: $srcrpm"; + return; + } + if ($dwzfilesize==0&&$dtfilesize==0) { + ++$emptybyfile; + } elsif ($dwzfilesize<$dtfilesize) { + #warn "dwz dwzfilesize $dwzfilesize < dtfilesize $dtfilesize: $srcrpm"; + ++$dwzwinbyfile; + } elsif ($dwzfilesize>$dtfilesize) { + ++$dtwinbyfile; + } else { + ++$equalbyfile; + } + push @{$filestat{$dwzfilesize/$dtfilesize}},"$srcrpm dwzfilesize=$dwzfilesize dtfilesize=$dtfilesize" if $dtfilesize; + if ($dwzrpmsize==0&&$dtrpmsize==0) { + ++$emptybyrpm; + } elsif ($dwzrpmsize<$dtrpmsize) { + ++$dwzwinbyrpm; + } elsif ($dwzrpmsize>$dtrpmsize) { + ++$dtwinbyrpm; + } else { + ++$equalbyrpm; + } + ++$done; + push @srcrpms,$srcrpm; +} +my %rpmqa; +if (($ARGV[0]||"") eq "-r") { + shift; + local $_; + for (split /\n/,readfile($ARGV[0] eq "-"?"rpm -qa '*-debuginfo'|":$ARGV[0])) { + next if /^\s*$/s; + /^(.*)-debuginfo-[^-]*-[^-]*$/ or die; + $rpmqa{$1}=1; + } + shift; +} +for my $srcrpmdone (glob "build/*.src.rpm.done") { + (my $srcrpm=$srcrpmdone)=~s{\Q.done\E$}{} or die $srcrpmdone; + if (%rpmqa) { + $srcrpm=~m{^build/(.*)-[^-]*-[^-]*$} or die; + next if !$rpmqa{$1}; + } + onesrcrpm $srcrpm; +} +print "dwzwinbyfile=$dwzwinbyfile"." dtwinbyfile=$dtwinbyfile"." equalbyfile=$equalbyfile"." emptybyfile=$emptybyfile"."\n"; +print "dwzwinbyrpm =$dwzwinbyrpm" ." dtwinbyrpm =$dtwinbyrpm" ." equalbyrpm =$equalbyrpm" ." emptybyrpm =$emptybyrpm" ."\n"; +print "mockfail=$mockfail dtmockfail=$dtmockfail\n"; +print "dwzfilesizetot=$dwzfilesizetot"." dtfilesizetot=$dtfilesizetot"." = ".($dwzfilesizetot/$dtfilesizetot*100)."%\n"; +print " dwzrpmsizetot=$dwzrpmsizetot" ." dtrpmsizetot=$dtrpmsizetot" ." = ".( $dwzrpmsizetot/ $dtrpmsizetot*100)."%\n"; +my $fail=0; +for my $srcrpmfail (glob "build/*.src.rpm.*fail*") { + ++$fail; +} +my $allpkgs=readfile "dnf -C --disablerepo='*' --enablerepo=rawhide-source repoquery -a --arch src 2>/dev/null|"; +$allpkgs=~tr/\n//cd; +$allpkgs=length $allpkgs; +print "done=$done + fail=$fail = ".($done+$fail)." all=$allpkgs processed=".(($done+$fail)/$allpkgs*100)."%\n"; +#print "equal: $_\n" for @{$filestat{1}}; +my @sorted=map({ my $r=$_; map(($_,$r),@{$filestat{$_}}); } sort {$a<=>$b;} keys %filestat); +sub out($) { + my($ix)=@_; + my $name=$sorted[2*$ix+0]; + my $ratio=$sorted[2*$ix+1]; + print(sprintf("%.2f",$ratio*100)."%: $name\n"); +} +my $sorted=@sorted/2; +if (@ARGV) { + out $_ for 0..$sorted-1; +} else { + my $p=5; + out $_ for 0..$p-1; + out $_ for $sorted-$p..$sorted-1; +} +print "debuginfod download size: "; +my $dwzsizetot=0; +my $dtsizetot=0; +for my $srcrpm (@srcrpms) { + my $rpmsizes="$srcrpm.rpmsizes"; + -e $rpmsizes or (system "./debuginfosize $srcrpm" and die $srcrpm); + -e $rpmsizes or die $srcrpm; + my $f=readfile $rpmsizes; + $f=~/^(\d+) (\d+)\n$/s; + my $dwzsize=$1; + my $dtsize=$2; + $dwzsizetot+=$dwzsize; + $dtsizetot+=$dtsize; +} +print "dwzsizetot=$dwzsizetot dtsizetot=$dtsizetot = ".sprintf("%.2f",$dwzsizetot/$dtsizetot*100)."%\n"; +print "build time: "; +my $dwztime=0; +my $dttime=0; +for my $srcrpm (@srcrpms) { + sub onemocklog($) { + my($mocklog)=@_; + my $mocklogtime="$mocklog.time"; + return 0+readfile $mocklogtime if -e $mocklogtime; + my $f=readfile($mocklog); + # INFO: Done(firefox-78.0-2.fc33.src.rpm) Config(fedora-rawhide-x86_64) 288 minutes 35 seconds + my $t; + $t=$1*60+$2 while $f=~/\nINFO: Done[(][^\n]*[)] (\d+) minutes (\d+) seconds\n/g; + defined $t or die $mocklog; + writefile $mocklogtime,"$t\n"; + return $t; + } + $dwztime+=onemocklog "$srcrpm.mocklog"; + $dttime+=onemocklog "$srcrpm.dt.rpm.mocklog"; +} +print "dwztime=$dwztime dttime=$dttime = ".sprintf("%.2f",$dwztime/$dttime*100)."%\n"; -- 1.8.3.1