init

diff --git a/debuginfosize b/debuginfosize
new file mode 100755 (executable)
index 0000000..7778ffb
--- /dev/null
+++ b/debuginfosize
@@ -0,0 +1,59 @@
+#! /bin/bash
+set -e
+# build/znc-1.8.1-2.fc33.src.rpm.rpms
+[ $# -eq 1 ]
+which readelf >/dev/null
+srcrpm=$1
+test -e $srcrpm.rpms
+test ! -e $srcrpm.rpmsizes
+test ! -e $srcrpm.rpms.d
+mkdir $srcrpm.rpms.d
+pushd $srcrpm.rpms.d >/dev/null
+  rpmsbase=../../`basename $srcrpm`
+  dwzrpms=$rpmsbase.rpms
+  dtrpms=$rpmsbase.dt.rpm.rpms
+  mkdir empty
+  cd empty
+    if [ -z "$(find $dwzrpms $dtrpms -maxdepth 1 -type f)" ];then
+      popd >/dev/null
+      rm -rf $srcrpm.rpms.d
+      echo "0 0" >$srcrpm.rpmsizes
+      exit 0
+    fi
+  cd ..
+  function extract
+  {
+    for i in $1/*-debuginfo*.rpm;do
+      test ! -e $i.tgz
+      rpm2archive $i
+      test -e $i.tgz
+      tar xzf $i.tgz
+      rm -f $i.tgz
+    done
+  }
+  mkdir dwz;(cd dwz;extract $dwzrpms)
+  mkdir  dt;(cd  dt;extract  $dtrpms)
+  dwzcommon="$(find dwz/usr/lib/debug/.dwz/ -type f -printf "%s" 2>/dev/null || :)"
+  if [ -z "$dwzcommon" ];then
+    dwzcommon=0
+  fi
+  dwztot=0
+  dttot=0
+  pushd dwz >/dev/null
+    for i in `find -type f -name "*.debug*"`;do
+      dwz=$(find $i      -maxdepth 0 -printf "%s")
+      dt=$(find ../dt/$i -maxdepth 0 -printf "%s")
+      if readelf -WS $i 2>/dev/null|fgrep -q .gnu_debugaltlink;then
+       if [ $dwzcommon -eq 0 ];then
+         echo >&2 Excessive .gnu_debugaltlink: $i
+         exit 1
+       fi
+       dwz=$[$dwz+$dwzcommon]
+      fi
+      dwztot=$[$dwztot+$dwz]
+      dttot=$[$dttot+$dt]
+    done
+  popd >/dev/null
+popd >/dev/null
+rm -rf $srcrpm.rpms.d
+echo "$dwztot $dttot" >$srcrpm.rpmsizes

diff --git a/run b/run
new file mode 100755 (executable)
index 0000000..bd9718b
--- /dev/null
+++ b/run
@@ -0,0 +1,10 @@
+#! /bin/bash
+set -ex
+renice 19 -p $$
+ionice -c3 -p $$
+#repos=fedora-source,updates-source
+repos=rawhide-source
+#all="$(dnf -C --disablerepo='*' --enablerepo=$repos repoquery -a --arch src --qf         '%{name}-%{version}-%{release}'|shuf)"
+#all="$(dnf -C --disablerepo='*' --enablerepo=$repos repoquery -a --arch src --qf '%{size} %{name}-%{version}-%{release}'|sort -nr|awk '{print $2}')"
+all=$(cat part|shuf)
+parallel -j8 -l8 ./runone $repos -- $all

diff --git a/runone b/runone
new file mode 100755 (executable)
index 0000000..0eac9e0
--- /dev/null
+++ b/runone
@@ -0,0 +1,91 @@
+#! /bin/bash
+set -ex
+repos=$1
+shift
+mock="mock -v -r fedora-rawhide-x86_64 --uniqueext=mass$$ --no-bootstrap-chroot --disable-plugin=yum_cache"
+mkdir -p build
+cd build
+for pkg in "$@";do
+  #dnf --disablerepo='*' --enablerepo=$repos repoquery --requires "$pkg"|egrep '^(gcc|clang)' || continue
+  srcrpm="$pkg.src.rpm"
+  if [ -e "$srcrpm" ];then
+    continue
+  fi
+  rm -rf $srcrpm*
+  exec >>$srcrpm.log 2>&1
+  dnf --disablerepo='*' --enablerepo=$repos download --source "$pkg"
+  if [ ! -e $srcrpm ];then
+    touch $srcrpm.downloadfail
+    continue
+  fi
+  #rpm -q --requires -p $srcrpm|egrep '^(gcc|clang)' || continue
+  rpm2archive $srcrpm
+  rm -rf $srcrpm.d
+  mkdir $srcrpm.d
+  cd $srcrpm.d
+   tar xzf ../$srcrpm.tgz
+   rm -f ../$srcrpm.tgz
+   name=$(echo "$pkg"|sed 's/-[^-]*-[^-]*$//')
+   if [ ! -e $name.spec ];then
+     touch ../$srcrpm.specfail
+     continue
+   fi
+   (echo '%global _find_debuginfo_dwz_opts %{nil}'
+    echo '%global optflags %(echo %{optflags}|sed "s/ -g / -g -fdebug-types-section /")'
+    sed 's/^%\(\(define\|global\) *_find_debuginfo_dwz_opts\)/\1/'
+   ) <$name.spec >$name.spec.new
+   cp -p $name.spec     ../$srcrpm.spec
+   cp -p $name.spec.new ../$srcrpm.spec.new
+   diff -u $name.spec $name.spec.new || :
+   mv -f $name.spec.new $name.spec
+   if ! $mock --resultdir=$PWD/resultdir --buildsrpm --spec $name.spec --sources $PWD;then
+     touch ../$srcrpm.bsmockfail
+     continue
+   fi
+   if [ $[$(find resultdir -name "*.src.rpm"|wc -l)] -ne 1 ];then
+     touch ../$srcrpm.bsmockefail
+     continue
+   fi
+   # if not pkg.endswith('.rpm'): log.error("%s doesn't appear to be an rpm - skipping", pkg)
+   mv -f resultdir/*.src.rpm ../$srcrpm.dt.rpm
+   cd ..
+  test -e $srcrpm.dt.rpm
+  rm -rf $srcrpm.d
+  for ext in "" ".dt.rpm";do
+    resultdir=$PWD/$srcrpm$ext.resultdir
+    test ! -e $resultdir
+    set +e
+    # --chain instead of --rebuild as otherwise -a does not work
+    $mock --resultdir=$resultdir --nocheck -a https://people.redhat.com/jkratoch/dwzrepo/ --chain $srcrpm$ext &>$srcrpm$ext.mocklog
+    rc=$?
+    set -e
+    test -e $resultdir
+    #rm -f $srcrpm$ext
+    if [ $rc -ne 0 ];then
+      touch $srcrpm$ext.mockfail
+      break
+    fi
+    # --chain ignores --resultdir
+    resultdirsrc="$(sed -n 's/^INFO: Results out to: //p' $srcrpm$ext.mocklog)"
+    if [ ! -e $resultdirsrc/*/root.log ];then
+      touch $srcrpm$ext.mockrfail
+      break
+    fi
+    # With --chain $resultdir contains just some small *.log files.
+    rm -rf $resultdir
+    mv `dirname $resultdirsrc/*/root.log` $resultdir
+    echo >$srcrpm$ext.rpmsize $(du -sbc /dev/null $resultdir/*-debuginfo*.rpm|tail -n1|awk '{print $1}')
+    size=0
+    for i in $resultdir/*-debuginfo*.rpm;do
+      if [ -e "$i" ];then
+       sizethis="$(rpm -q --qf '%{size}' -p $i)"
+       size=$[$size+$sizethis]
+      fi
+    done
+    echo >$srcrpm$ext.filesize $size
+    mkdir $srcrpm$ext.rpms
+    mv -f $resultdir/*-debuginfo*.rpm $srcrpm$ext.rpms || :
+    rm -rf $resultdir
+  done
+  touch $srcrpm.done
+done

diff --git a/stat b/stat
new file mode 100755 (executable)
index 0000000..a209c70
--- /dev/null
+++ b/stat
@@ -0,0 +1,171 @@
+#! /usr/bin/perl
+use strict;
+use warnings;
+
+$|=1;
+sub readfile {
+  my($fname)=@_;
+  local *F;
+  open F,"$fname" or die $fname;
+  local $/=undef();
+  defined(my $r=<F>) or die $fname;
+  close F or die $fname;
+  return $r;
+}
+sub writefile {
+  my($fname,$content)=@_;
+  local *F;
+  open F,">$fname" or die $fname;
+  print F $content or die $fname;
+  close F or die $fname;
+}
+
+my $mockfail=0;
+my $dtmockfail=0;
+my $emptybyfile=0;
+my $dwzwinbyfile=0;
+my $dtwinbyfile=0;
+my $equalbyfile=0;
+my $emptybyrpm=0;
+my $dwzwinbyrpm=0;
+my $dtwinbyrpm=0;
+my $equalbyrpm=0;
+my $dwzfilesizetot=0;
+my $dtfilesizetot=0;
+my $dwzrpmsizetot=0;
+my $dtrpmsizetot=0;
+my $done=0;
+my %filestat;
+my @srcrpms;
+sub onesrcrpm($) {
+  my($srcrpm)=@_;
+#  print "$srcrpm\n";
+  if (-e "$srcrpm.mockfail") {
+    die if -e "$srcrpm.dt.rpm.mockfail";
+    warn "mockfail: $srcrpm";
+    ++$mockfail;
+    return;
+  }
+  if (-e "$srcrpm.dt.rpm.mockfail") {
+    warn "dtmockfail: $srcrpm";
+    ++$dtmockfail;
+    return;
+  }
+  my $dwzfilesize=0+readfile "$srcrpm.filesize";
+  my $dwzrpmsize=0+readfile "$srcrpm.rpmsize";
+  my $dtfilesize=0+readfile "$srcrpm.dt.rpm.filesize";
+  my $dtrpmsize=0+readfile "$srcrpm.dt.rpm.rpmsize";
+  $dwzfilesizetot+=$dwzfilesize;
+  $dtfilesizetot+=$dtfilesize;
+  $dwzrpmsizetot+=$dwzrpmsize;
+  $dtrpmsizetot+=$dtrpmsize;
+  if (($dwzfilesize>=$dtfilesize)!=($dwzrpmsize>=$dtrpmsize)) {
+    #warn "file vs. rpm size do not match (dwzfilesize=$dwzfilesize dtfilesize=$dtfilesize dwzrpmsize=$dwzrpmsize dtrpmsize=$dtrpmsize: $srcrpm";
+    return;
+  }
+  if ($dwzfilesize==0&&$dtfilesize==0) {
+    ++$emptybyfile;
+  } elsif ($dwzfilesize<$dtfilesize) {
+    #warn "dwz dwzfilesize $dwzfilesize < dtfilesize $dtfilesize: $srcrpm";
+    ++$dwzwinbyfile;
+  } elsif ($dwzfilesize>$dtfilesize) {
+    ++$dtwinbyfile;
+  } else {
+    ++$equalbyfile;
+  }
+  push @{$filestat{$dwzfilesize/$dtfilesize}},"$srcrpm dwzfilesize=$dwzfilesize dtfilesize=$dtfilesize" if $dtfilesize;
+  if ($dwzrpmsize==0&&$dtrpmsize==0) {
+    ++$emptybyrpm;
+  } elsif ($dwzrpmsize<$dtrpmsize) {
+    ++$dwzwinbyrpm;
+  } elsif ($dwzrpmsize>$dtrpmsize) {
+    ++$dtwinbyrpm;
+  } else {
+    ++$equalbyrpm;
+  }
+  ++$done;
+  push @srcrpms,$srcrpm;
+}
+my %rpmqa;
+if (($ARGV[0]||"") eq "-r") {
+  shift;
+  local $_;
+  for (split /\n/,readfile($ARGV[0] eq "-"?"rpm -qa '*-debuginfo'|":$ARGV[0])) {
+    next if /^\s*$/s;
+    /^(.*)-debuginfo-[^-]*-[^-]*$/ or die;
+    $rpmqa{$1}=1;
+  }
+  shift;
+}
+for my $srcrpmdone (glob "build/*.src.rpm.done") {
+  (my $srcrpm=$srcrpmdone)=~s{\Q.done\E$}{} or die $srcrpmdone;
+  if (%rpmqa) {
+    $srcrpm=~m{^build/(.*)-[^-]*-[^-]*$} or die;
+    next if !$rpmqa{$1};
+  }
+  onesrcrpm $srcrpm;
+}
+print "dwzwinbyfile=$dwzwinbyfile"." dtwinbyfile=$dtwinbyfile"." equalbyfile=$equalbyfile"." emptybyfile=$emptybyfile"."\n";
+print "dwzwinbyrpm =$dwzwinbyrpm" ." dtwinbyrpm =$dtwinbyrpm" ." equalbyrpm =$equalbyrpm" ." emptybyrpm =$emptybyrpm" ."\n";
+print "mockfail=$mockfail dtmockfail=$dtmockfail\n";
+print "dwzfilesizetot=$dwzfilesizetot"." dtfilesizetot=$dtfilesizetot"." = ".($dwzfilesizetot/$dtfilesizetot*100)."%\n";
+print " dwzrpmsizetot=$dwzrpmsizetot" ."  dtrpmsizetot=$dtrpmsizetot" ." = ".( $dwzrpmsizetot/ $dtrpmsizetot*100)."%\n";
+my $fail=0;
+for my $srcrpmfail (glob "build/*.src.rpm.*fail*") {
+  ++$fail;
+}
+my $allpkgs=readfile "dnf -C --disablerepo='*' --enablerepo=rawhide-source repoquery -a --arch src 2>/dev/null|";
+$allpkgs=~tr/\n//cd;
+$allpkgs=length $allpkgs;
+print "done=$done + fail=$fail = ".($done+$fail)." all=$allpkgs processed=".(($done+$fail)/$allpkgs*100)."%\n";
+#print "equal: $_\n" for @{$filestat{1}};
+my @sorted=map({ my $r=$_; map(($_,$r),@{$filestat{$_}}); } sort {$a<=>$b;} keys %filestat);
+sub out($) {
+  my($ix)=@_;
+  my $name=$sorted[2*$ix+0];
+  my $ratio=$sorted[2*$ix+1];
+  print(sprintf("%.2f",$ratio*100)."%: $name\n");
+}
+my $sorted=@sorted/2;
+if (@ARGV) {
+  out $_ for 0..$sorted-1;
+} else {
+  my $p=5;
+  out $_ for 0..$p-1;
+  out $_ for $sorted-$p..$sorted-1;
+}
+print "debuginfod download size: ";
+my $dwzsizetot=0;
+my $dtsizetot=0;
+for my $srcrpm (@srcrpms) {
+  my $rpmsizes="$srcrpm.rpmsizes";
+  -e $rpmsizes or (system "./debuginfosize $srcrpm" and die $srcrpm);
+  -e $rpmsizes or die $srcrpm;
+  my $f=readfile $rpmsizes;
+  $f=~/^(\d+) (\d+)\n$/s;
+  my $dwzsize=$1;
+  my $dtsize=$2;
+  $dwzsizetot+=$dwzsize;
+  $dtsizetot+=$dtsize;
+}
+print "dwzsizetot=$dwzsizetot dtsizetot=$dtsizetot = ".sprintf("%.2f",$dwzsizetot/$dtsizetot*100)."%\n";
+print "build time: ";
+my $dwztime=0;
+my $dttime=0;
+for my $srcrpm (@srcrpms) {
+  sub onemocklog($) {
+    my($mocklog)=@_;
+    my $mocklogtime="$mocklog.time";
+    return 0+readfile $mocklogtime if -e $mocklogtime;
+    my $f=readfile($mocklog);
+    # INFO: Done(firefox-78.0-2.fc33.src.rpm) Config(fedora-rawhide-x86_64) 288 minutes 35 seconds
+    my $t;
+    $t=$1*60+$2 while $f=~/\nINFO: Done[(][^\n]*[)] (\d+) minutes (\d+) seconds\n/g;
+    defined $t or die $mocklog;
+    writefile $mocklogtime,"$t\n";
+    return $t;
+  }
+  $dwztime+=onemocklog "$srcrpm.mocklog";
+  $dttime+=onemocklog "$srcrpm.dt.rpm.mocklog";
+}
+print "dwztime=$dwztime dttime=$dttime = ".sprintf("%.2f",$dwztime/$dttime*100)."%\n";