From: root Date: Wed, 25 May 2016 16:45:21 +0000 (+0200) Subject: Merge branch 'master' of ssh://vps.jankratochvil.net/var/lib/git/nethome X-Git-Url: https://git.jankratochvil.net/?p=nethome.git;a=commitdiff_plain;h=8104eca7d9cf4376bfa2eddc3c2a71b84b9d9d02;hp=e36f4e85c46f2f9eedcfc9b64dded50171364c0f Merge branch 'master' of ssh://vps.jankratochvil.net/var/lib/git/nethome --- diff --git a/.bashrc b/.bashrc index d01df08..5c9b4a0 100644 --- a/.bashrc +++ b/.bashrc @@ -112,7 +112,9 @@ unalias 2>/dev/null du # -h unalias 2>/dev/null ls # --color=auto unalias 2>/dev/null l. # ls -d .* --color=auto unalias 2>/dev/null ll # ls -l --color=auto -unalias 2>/dev/null grep # grep --color=auto +unalias 2>/dev/null grep # /etc/profile.d/colorgrep.sh +unalias 2>/dev/null egrep # /etc/profile.d/colorgrep.sh +unalias 2>/dev/null fgrep # /etc/profile.d/colorgrep.sh # Prevent processing of aliases during the parsing of this script file by # `eval': @@ -149,7 +151,7 @@ eval ' function vncreadonly { vncviewer -FullColor -Shared -ViewOnly "$@"; } function diff {(unset diff; ( diff -dup "$@"; ); );} $(: MAKEFLAGS= - "make install" does not expect -j and .spec files do not force -j1.) - function rpmbuildlocal { MAKEFLAGS= n rpmbuild --define "_topdir $PWD" --define "_builddir $PWD" --define "_rpmdir $PWD" --define "_sourcedir $PWD" --define "_specdir $PWD" --define "_srcrpmdir $PWD" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" "$@"; rmdir &>/dev/null BUILDROOT; } + function rpmbuildlocal { MAKEFLAGS= n rpmbuild --define "_topdir $PWD" --define "_builddir $PWD" --define "_rpmdir $PWD" --define "_sourcedir $PWD" --define "_specdir $PWD" --define "_srcrpmdir $PWD" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" --with buildisa "$@"; rmdir &>/dev/null BUILDROOT; } function gdbn { gdb -nx --command=~/.gdbinit "$@"; } function hd { od -Ax -tx1; } function wget {(unset wget; ( wget --no-check-certificate "$@"; ); );} @@ -192,17 +194,22 @@ if [ -n "$PS1" ];then # set only in interactive sessions # as it breaks keyboard state in X and it is not permitted for non-root. fi export MINICOM="-m -c on" # metakeys+color -export HISTSIZE=100000 +export HISTSIZE=10000000 export HISTFILESIZE="$HISTSIZE" export GDBHISTFILE="$HOME/.gdb_history" export CVS_RSH="ssh" export TZ=":/usr/share/zoneinfo/Europe/Prague" #export TZ=":/usr/share/zoneinfo/Canada/Eastern" export PYTHONUNBUFFERED=1 +export ASAN_OPTIONS=detect_leaks=0 export MAKEFLAGS= cpus="`getconf _NPROCESSORS_ONLN`" if [ -n "$cpus" ];then - MAKEFLAGS="$MAKEFLAGS -j$[$cpus*3/2]" + # *3/2 was OOM for host1 32 CPUs + 16GB RAM + # *1 was OOM for host1 ccache -C;touch gui.h;time make all + # *3/4 was also OOM with KVM running + # *1/2 was OK for host1 32 CPUs + 16GB RAM + MAKEFLAGS="$MAKEFLAGS -j$[$cpus*1]" fi unset cpus # Do not: grep -w "$TERM" /etc/termcap >/dev/null || export TERM=vt220 @@ -337,8 +344,7 @@ function sourcewarecvscheckout {(set -ex #(set +x;cvsignoresall) (set +x;ignoresall) );} -function gdbgitclone { git clone git://sourceware.org/git/gdb.git gdb-git; } -function binutilsgitclone { git clone git://sourceware.org/git/binutils.git binutils-git; } +function gdbgitclone { git clone git://sourceware.org/git/binutils-gdb.git gdb-git; } function archergitclone { git clone git://sourceware.org/git/archer.git archer-git; } function gdbcvscheckout { sourcewarecvscheckout gdb; } function binutilscvscheckout { sourcewarecvscheckout binutils; } @@ -380,7 +386,8 @@ function gpg {(unset gpg; http_proxy= all_proxy= HTTP_PROXY= ALL_PROXY= gpg "$@" ulimit -S -c 0 set +H shopt -s mailwarn -shopt -s direxpand +#RHEL6->7 breaks: echo spa\ ce ~/foo +#shopt -s direxpand unset OPTIND umask 022 diff --git a/.vimrc b/.vimrc index 00b77d7..44871db 100644 --- a/.vimrc +++ b/.vimrc @@ -99,5 +99,6 @@ noremap * :let ic_save=&ic:set noic*:let &ic=ic_save noremap # :let ic_save=&ic:set noic#:let &ic=ic_save noremap :w:!aspell --check '%':e noremap gq] gq/^. \?$ +noremap 1 :w:make -j1 endif "!exists("g:_kratochvil_vimrc") diff --git a/bin/errs12 b/bin/errs12 index 9148109..bfac6f5 100755 --- a/bin/errs12 +++ b/bin/errs12 @@ -19,8 +19,8 @@ if gcc --version|perl -ne 'exit(!(/(\d+)\.(\d+)/&&($1>4||($1==4&&$2>=7))));';the else debug="-g2" fi -#asan="" -asan="-fsanitize=address" +asan="" +#asan="-fsanitize=address -static-libasan" mcheck="" #mcheck="-lmcheck" fast=false @@ -75,7 +75,7 @@ do fi if [ "$1" = "--asan" ];then - asan="-fsanitize=address" + asan="-fsanitize=address -static-libasan" shift continue fi diff --git a/bin/mocksetup b/bin/mocksetup index da4800c..b24afbc 100755 --- a/bin/mocksetup +++ b/bin/mocksetup @@ -82,7 +82,8 @@ for r in $l;do echo $c; $c fi - cp -p /etc/resolv.conf $rpath/etc/resolv.conf + cmp -s /etc/resolv.conf $rpath/etc/resolv.conf \ + || cp -p /etc/resolv.conf $rpath/etc/resolv.conf dir="/dev/pts" c="umount $rpath$dir" @@ -96,11 +97,11 @@ for r in $l;do echo $c fi - test -e $rpath/dev/pts && rmdir $rpath/dev/pts + test -e $rpath/dev/pts && (rmdir $rpath/dev/pts || :) if ! $u;then mkdir -p $rpath/dev/pts fi - test -e $rpath/dev/shm && rmdir $rpath/dev/shm + test -e $rpath/dev/shm && (rmdir $rpath/dev/shm || :) if ! $u;then mkdir -p $rpath/dev/shm fi @@ -153,10 +154,29 @@ for r in $l;do ln -s /proc/self/fd $rpath/dev/fd fi - mkdir -p $rpath/unsafe - for dir in /home /usr/local/bin /proc /sys /root /unsafe;do + # Must be before umount of /quad + for cache in yum dnf;do + c="umount $rpath/var/cache/$cache" + if $c 2>&1|grep -v ': not \(mounted\|found\)$';then + echo $c + fi + if ! $u;then + if [ -d /var/cache/mock/$r/${cache}_cache ];then + cr=$r + else + cr=`echo $r|sed 's/-[^-]*$//'` + fi + if [ -d /var/cache/mock/$cr/${cache}_cache ];then + c="mount --bind /var/cache/mock/$cr/${cache}_cache $rpath/var/cache/${cache}" + echo $c; $c + fi + fi + done + + mkdir -p $rpath/quad + for dir in /home /usr/local/bin /proc /sys /root /quad;do c="umount $rpath$dir" - if $c 2>&1|grep -v ': not mounted$';then + if $c 2>&1|grep -v ': not \(mounted\|found\)$';then echo $c fi if ! $u;then @@ -172,20 +192,6 @@ for r in $l;do echo $c; $c fi - c="umount $rpath/var/cache/yum" - if $c 2>&1|grep -v ': not mounted$';then - echo $c - fi - if ! $u;then - if [ -d /var/cache/mock/$r/yum_cache ];then - cr=$r - else - cr=`echo $r|sed 's/-[^-]*$//'` - fi - c="mount --bind /var/cache/mock/$cr/yum_cache $rpath/var/cache/yum" - echo $c; $c - fi - rm -f $rpath/usr/local/lib/debug if ! $u;then ln -s ../../lib/debug $rpath/usr/local/lib/debug diff --git a/bin/psswap b/bin/psswap new file mode 100755 index 0000000..615a07b --- /dev/null +++ b/bin/psswap @@ -0,0 +1,46 @@ +#! /usr/bin/perl +use strict; +use warnings; +local *F; +my $pscmd="ps axwl"; +open F,"$pscmd|" or die "$pscmd: $!"; +my %pid; +my $rsstotal=0; +local $_; +while () { + /^(?:\S+\s+){2}(\S+)\s+(?:\S+\s+){4}(\S+)/ or die $_; + die if $pid{$1}; + $pid{$1}=$_; + $rsstotal+=$2 if $2 ne "RSS"; +} +close F or die "$pscmd: $!"; +my $head=delete $pid{"PID"} or die; +my %swap; +for my $pid (keys(%pid)) { + my $line=$pid{$pid}; + my $kb=0; + my $fn="/proc/$pid/smaps"; + if (!open F,$fn) { + warn "$fn: $line: $!" if $line!~/ $pscmd$/; + next; + } + while () { + $kb+=$1 if /^Swap:\s+(\d+) kB\n$/; + } + close F or die "$fn: $!"; + push @{$swap{$kb}},$line; +} +sub printfmt($$) { + my($kb,$line)=@_; + printf "%10s %s",$kb,$line; +} +printfmt "SWAP",$head; +my $swaptotal=0; +for my $kb (sort { $b<=>$a; } keys(%swap)) { + for my $line (@{$swap{$kb}}) { + printfmt $kb,$line; + $swaptotal+=$kb; + } +} +print "SWAP total: $swaptotal\n"; +print "RSS total: $rsstotal\n"; diff --git a/src/memchannel.C b/src/memchannel.C new file mode 100644 index 0000000..ac79d41 --- /dev/null +++ b/src/memchannel.C @@ -0,0 +1,280 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::set page_set; + +static size_t GB (1024*1024*1024); +static size_t GBDEC(1000*1000*1000); + +//#define SEQUENTIAL +//#define RANDOM + +//#define MAXPAGES ( 1.0*GBDEC/PAGE_SIZE) +#define MAXPAGES (60.0*GBDEC/PAGE_SIZE) +#ifndef MAXPAGES +# define MAXPAGES std::numeric_limits::max() +# define FREEPAGES (1.0*GBDEC/PAGE_SIZE) +#endif + +static void stat(const char *msg) { + printf("%s: %zu=%zuGBDEC\n",msg,page_set.size(),page_set.size()*PAGE_SIZE/GBDEC); +} + +static void eatmem() { + while (page_set.size()(page)); + *pagecast=0x55; + const auto successpair(page_set.insert(pagecast)); + assert(successpair.second); + } + stat("MAXPAGES"); +} + +static void freemem() { +#ifdef FREEPAGES + for (size_t pageno=0;pageno(*it)); + int err; + err=munmap(page,PAGE_SIZE); + assert(!err); + } + stat("freemem"); +#endif +} + +class Dimm { +public: + const uint64_t from,to_excl; + Dimm(uint64_t from_,uint64_t to_excl_):from(from_),to_excl(to_excl_) {} + bool contains(volatile uint8_t *const page) const { + const uint64_t addr(reinterpret_cast(page)); + return from<=addr&&addr dimms; + +static ssize_t dimmno_get_nocache(volatile uint8_t *const page) { + for (auto it=dimms.cbegin();it!=dimms.cend();++it) { + const auto &dimm(*it); + if (dimm.contains(page)) { +#if defined SEQUENTIAL + static size_t seq=0; + ++seq; + seq%=dimms.size(); +// return ((it-dimms.cbegin())&~3)+(seq%4); + return seq; +#elif defined RANDOM + return random()%dimms.size(); +#else + return it-dimms.cbegin(); +// return dimms.size()-1-(it-dimms.cbegin()); +// return ((it-dimms.cbegin())&~3)+(random()%4); +#endif + } + } + //printf("Cannot find a DIMM for: %p\n",page); + return -1; +} + +static ssize_t dimmno_get(volatile uint8_t *const page) { + static ssize_t cache_dimmno(0); + if (cache_dimmno==-1||!dimms[cache_dimmno].contains(page)) + cache_dimmno=dimmno_get_nocache(page); + return cache_dimmno; +} + +static size_t dimmno_to_cpuno(size_t dimmno) { + return 2*dimmno; + //return dimmno*std::thread::hardware_concurrency()/dimms.size(); +} + +static std::vector> dimmno_to_pages; + +static void readmap() { + std::vector pagemapvec; + volatile uint8_t *const page_set_front(*page_set. cbegin()); + volatile uint8_t *const page_set_back (*page_set.crbegin()); + { const int fd(open("/proc/self/pagemap",O_RDONLY)); + assert(fd!=-1); + pagemapvec.resize((page_set_back-page_set_front)/PAGE_SIZE+1); + const size_t count(pagemapvec.size()*sizeof(uint64_t)); + const ssize_t got(pread(fd,pagemapvec.data(),count,uintptr_t(page_set_front)/PAGE_SIZE*sizeof(uint64_t))); + assert(size_t(got)==count); + int err; + err=close(fd); + assert(!err); + } + dimmno_to_pages.resize(dimms.size()); + size_t bad_pfn_mask(0); + size_t bad_dimmno(0); + for (auto &it:page_set) { + volatile uint8_t *const page(it); + const uint64_t pagemapval(pagemapvec[(page-page_set_front)/PAGE_SIZE]); + //printf("%p=0x%016lx\n",page,pagemapval); + // vm/pagemap.txt + const uint64_t pfn_mask((1ULL<<55)-1); // Bits 0-54 page frame number (PFN) if present + if ((pagemapval&~pfn_mask)!=0x8600000000000000) { + ++bad_pfn_mask; + continue; + } + const uint64_t pfn(pagemapval&pfn_mask); + const auto pfn_page(reinterpret_cast(pfn*PAGE_SIZE)); + const ssize_t dimmno(dimmno_get(pfn_page)); + if (dimmno==-1) { + ++bad_dimmno; + continue; + } + auto successpair(dimmno_to_pages[dimmno].insert(page)); + assert(successpair.second); + } + printf("bad_pfn_mask: %zu\n",bad_pfn_mask); + printf("bad_dimmno: %zu\n",bad_dimmno ); + for (size_t dimmno=0;dimmno &pages(dimmno_to_pages[dimmno]); + volatile size_t sum=0; + for (volatile uint8_t *const page:pages) + for (size_t offset=0;offset threads; + for (size_t dimmno=0;dimmno(end-start).count()); +} + +static uint64_t strtouint64(const std::string &str) { + static_assert(sizeof(long)==sizeof(uint64_t),"!64-bit"); + char *end; + const long l(strtol(str.c_str(),&end,0/*base*/)); + assert(l>=0); + assert(*end==0); + return l; +} + +static void dmidecode() { + FILE *const f(popen("dmidecode -q -t 20","r")); + assert(f); + ssize_t dimmno(-1); + const auto uint64max(std::numeric_limits::max()); + uint64_t dimm_start(uint64max),dimm_end(uint64max); + char *linep(nullptr); + size_t linen(0); + for (;;) { + errno=0; + const ssize_t got(getline(&linep,&linen,f)); + if (got==-1) { + assert_perror(errno); + break; + } + assert(got>=0); + assert(size_t(got)==strlen(linep)); + std::string line(linep); + if (got==0) + break; + assert(line.back()=='\n'); + line.erase(line.length()-1); + if (line=="Memory Device Mapped Address") { + assert(dimmno==-1||dimm_end!=uint64max); + ++dimmno; + dimm_start=dimm_end=uint64max; + continue; + } + { const std::string start_str("\tStarting Address: "); + if (line.substr(0,start_str.length())==start_str) { + assert(dimmno>=0); + assert(dimm_start==uint64max); + dimm_start=strtouint64(line.substr(start_str.length())); + assert(dimm_start!=uint64max); + continue; + } + } + { const std::string end_str("\tEnding Address: "); + if (line.substr(0,end_str.length())==end_str) { + assert(dimmno>=0); + assert(dimm_end==uint64max); + dimm_end=strtouint64(line.substr(end_str.length())); + assert(dimm_end!=uint64max); + assert(dimm_start!=uint64max); + if (dimm_start>=2*GB) + dimm_start+=2*GB; + if (dimm_end>=2*GB) + dimm_end+=2*GB; + dimms.push_back(Dimm(dimm_start,dimm_end)); + continue; + } + } + } + free(linep); + int err; + err=fclose(f); + assert(!err); +} + +int main() { + setbuf(stdout,NULL); + dmidecode(); + eatmem(); + freemem(); + readmap(); + runthreads(); +}