From: Jan Kratochvil Date: Fri, 19 Apr 2024 13:39:16 +0000 (+0800) Subject: +.config/yt-dlp.conf X-Git-Url: https://git.jankratochvil.net/?p=nethome.git;a=commitdiff_plain;h=HEAD;hp=b39de4dca13f27ea78e6a36a0040def197c37757 +.config/yt-dlp.conf --- diff --git a/.bashrc b/.bashrc index 0a78130..82e5139 100644 --- a/.bashrc +++ b/.bashrc @@ -128,8 +128,13 @@ unalias 2>/dev/null fgrep # /etc/profile.d/colorgrep.sh # `eval': eval ' function grep {(unset grep; ( ulimitme; grep "$@"; ); );} - function ls {(unset ls; ( ls -bF "$@"; ); );} - function l { ls -lbF "$@"; } + if ls -d -bF / &>/dev/null;then + function ls {(unset ls; ( ls -bF "$@"; ); );} + else + # Alpine Linux + function ls {(unset ls; ( ls "$@"; ); );} + fi + function l { ls -l "$@"; } if which vim &>/dev/null;then alias vi="vim" else @@ -209,8 +214,11 @@ export HISTSIZE=10000000 export HISTFILESIZE="$HISTSIZE" export GDBHISTFILE="$HOME/.gdb_history" export CVS_RSH="ssh" -export TZ="Europe/Prague" -#export TZ="Canada/Eastern" +export TZ="Asia/Manila" +#export TZ="Europe/Prague" +if [ "$TZ" = Asia/Manila ];then + alias date='TZ=Asia/Manila date;TZ=Europe/Prague date' +fi export PYTHONUNBUFFERED=1 export ASAN_OPTIONS=detect_leaks=0 export GTK_OVERLAY_SCROLLING=0 diff --git a/.config/yt-dlp.conf b/.config/yt-dlp.conf new file mode 100644 index 0000000..ce0622d --- /dev/null +++ b/.config/yt-dlp.conf @@ -0,0 +1 @@ +--restrict-filenames diff --git a/.gdbinit b/.gdbinit index 613d134..8a01500 100644 --- a/.gdbinit +++ b/.gdbinit @@ -7,12 +7,21 @@ set history save on # https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=106814 set complaints 0 set record full insn-number-max 10000000 +define u + up +end +document u +up +end define javadump call _Jv_DeepDebug($arg0) end document javadump call _Jv_DeepDebug($arg0) end +define pjstr + p *$arg0._body@$arg0._length +end define pglist if (*("$arg1")=='@') set var $arg1=struct $arg1 diff --git a/.gitconfig b/.gitconfig index e37f907..dfa6b16 100644 --- a/.gitconfig +++ b/.gitconfig @@ -1,6 +1,7 @@ [user] name = Jan Kratochvil - email = jan@jankratochvil.net +# use: .gitconfig.local +# email = jan@jankratochvil.net [include] path = .gitconfig.local [init] diff --git a/.muttrc.addons b/.muttrc.addons index 0a795bf..7205f18 100644 --- a/.muttrc.addons +++ b/.muttrc.addons @@ -20,7 +20,7 @@ bind pager \` previous-unread unset strict_threads # tw=78 is for gq and RFC 2822 section 2.1.1. set editor="vim -c 'set noai' -c 'set ts=8' -c 'set tw=78'" -set send_charset="us-ascii:iso-2022-jp:utf-8" +set send_charset="us-ascii:utf-8" set charset="utf-8" set autoedit set auto_tag @@ -60,6 +60,7 @@ macro index m "l~N|~O\n" macro pager f ":set pager_index_lines=0\n macro pager g ":set pager_index_lines=10\n" set user_agent +set abort_noattach=ask-yes # https://unix.stackexchange.com/a/42714/296319 auto_view text/html # view html automatically @@ -131,7 +132,7 @@ color status brightyellow blue color tree brightmagenta black # the thread tree in the index menu color tilde brightmagenta black color message brightcyan black -color normal white black +color normal brightwhite black color attachment brightmagenta black color search black green # how to hilite search patterns in the pager color body brightyellow black "(ftp|http)://[^ ]+" # point out URLs diff --git a/.ssh/config b/.ssh/config index e9814b5..e7d34ff 100644 --- a/.ssh/config +++ b/.ssh/config @@ -4,6 +4,7 @@ IdentityFile ~/.ssh/id_rsa-old IdentityFile ~/.ssh/identity IdentityFile ~/.ssh/koji-id_rsa IdentityFile ~/.ssh/id_dsa-sourceware +IdentityFile ~/.ssh/id_ed25519 Host localhost #Cipher none @@ -35,7 +36,7 @@ User jankratochvil Host virt # 172.1[6-9].* 172.2[0-9].* 172.3[0-1].* -User root +#User root UserKnownHostsFile /dev/null StrictHostKeyChecking no Compression no @@ -101,7 +102,7 @@ Host download-stock HostName vps2.jankratochvil.net HostKeyAlias vps2.jankratochvil.net -Host cosmo8 cosmo8.jankratochvil.net cosmo8t.jankratochvil.net +Host cosmo8 cosmo8.jankratochvil.net cosmo8t.jankratochvil.net p20 Port 8022 # Cosmo: @@ -134,14 +135,30 @@ ControlPersist 60 # OpenWrt # https://bugzilla.redhat.com/show_bug.cgi?id=1881301#c38 Host 192.168.66.13 ap13 -PubkeyAcceptedKeyTypes ssh-rsa +#PubkeyAcceptedKeyTypes ssh-rsa User root -Host jane -HostName jane.jankratochvil.net -HostKeyAlias jane.jankratochvil.net -Host jane jane.jankratochvil.net -User root +Host dev? dev?.azulsystems.com +User jkratochvil +Host dev1 dev1.azulsystems.com dev2 dev2.azulsystems.com +#PubkeyAcceptedKeyTypes ssh-rsa +PubkeyAcceptedKeyTypes +ssh-rsa,ssh-dss +HostKeyAlgorithms=+ssh-dss +Host dev1 +HostName dev1.azulsystems.com +Host dev2 +HostName dev2.azulsystems.com +Host dev3 +HostName dev3.azulsystems.com +Host rpi4-05 +HostName rpi4-05.azulsystems.com +Host rpi4-06 +HostName rpi4-06.azulsystems.com +Host rpi4-* rpi4-*.azulsystems.com +User jenkins +Host zulu-dev +User ubuntu +HostName 10.10.164.222 Host * Compression yes diff --git a/.vimrc b/.vimrc index 15df0d7..2322696 100644 --- a/.vimrc +++ b/.vimrc @@ -99,7 +99,9 @@ endif noremap! [23~ noremap M :cnzz noremap L :cpzz -noremap f "myiw:grep-Ew'm'`git ls-files` +" dnf install vim-fugitive +noremap f "myiw:Ggrep-Ew'm' +noremap F "myiw:grep-Ew'm'`git ls-files` noremap g "myiw:grep-rEw'm'. noremap zz noremap zz @@ -113,7 +115,7 @@ noremap # :let ic_save=&ic:set noic#:let &ic=ic_save noremap :w:!aspell --check '%':e noremap gq] gq/^. \?$ noremap 1 :w:make -j1 -noremap 3 /^ddV/^k:w! 1gvxddV/^=======k:w! 2gvxddV/^k:w! 3/^dd''0:!diff -u 2 1 +noremap 3 /^kmajd$V/^k:w! 1gvxd$V/^=======k:w! 2gvxd$V/^k:w! 3gvxdd'a0:!diff -u 2 1;diff -u 2 3 noremap m :w:set makeprg=make:make noremap n :w:set makeprg=ninja:make diff --git a/bin/ccache-build b/bin/ccache-build new file mode 100755 index 0000000..df63d74 --- /dev/null +++ b/bin/ccache-build @@ -0,0 +1,12 @@ +#! /bin/bash +# openjdk: export PATH="$(echo "$PATH"|sed 's#:/usr/lib64/ccache:#:'$HOME'/ccache:#')";bash configure --disable-precompiled-headers --disable-ccache +set -ex +rm -rf ~/ccache +mkdir ~/ccache +cd ~/ccache +for i in /usr/lib64/ccache/*;do + j=`basename $i` + echo -e '#! /bin/bash\nexport PATH="$(echo "$PATH"|sed s#:$HOME/ccache:#:#)"\nexec '$i' "$@"' >$j + chmod +x $j +done +echo done diff --git a/bin/errs12 b/bin/errs12 index fd308d3..50740c4 100755 --- a/bin/errs12 +++ b/bin/errs12 @@ -26,7 +26,7 @@ mcheck="" #mcheck="-lmcheck" fast=false readline="" -disablebinutils="--disable-binutils --disable-gas --disable-gold --disable-gprof --disable-ld" +disablebinutils="--disable-binutils --disable-gas --disable-gold --disable-gprof --disable-ld --disable-gprofng" python="" while true diff --git a/bin/exx b/bin/exx index 97f021b..8994599 100755 --- a/bin/exx +++ b/bin/exx @@ -14,6 +14,8 @@ use constant FORMATS=>{ "rpm" =>'rpm2cpio $pathname|cpio -id --quiet', #-v #FIXME: --sparse doesn't work, why? "zip" =>'unzip -Lq $pathname', "jar" =>'unzip -Lq $pathname', + "xpi" =>'unzip -Lq $pathname', + "crx" =>'unzip -Lq $pathname', "a" =>'ar x $pathname', "deb" =>'ar x $pathname;' .'for i in *.tar.gz;do j=`basename $i .tar.gz`;mkdir -p $j;cd $j;tar xzf ../$i;cd ..;rm -f $i;done', diff --git a/bin/hog b/bin/hog new file mode 100755 index 0000000..4dd1017 --- /dev/null +++ b/bin/hog @@ -0,0 +1,19 @@ +#! /usr/bin/perl +use strict; +use warnings; +my $psu=system($ENV{"HOME"}."/bin/psu")==0; +local *F; +open F,"/usr/bin/top -b -n2 -d1|" or die; +my $top=0; +while () { + /^top / && $top++; + /^$/ && $top++; + next if $top!=5; + /^(?:\s*\S+){8}\s*(\S+)\s*\S+\s*([^:\s]*):/ or next; + next if $1<90; # CPU % + next if $2<($psu?30:1); # minutes it has run + tr/'//d; + system "DISPLAY=:0 xmessage -timeout 30 -default okay -center '$_' 2>/dev/null"; + exit 0; +} +close F or die; diff --git a/bin/jtreg-pass-fail b/bin/jtreg-pass-fail new file mode 100755 index 0000000..aca6978 --- /dev/null +++ b/bin/jtreg-pass-fail @@ -0,0 +1,14 @@ +#! /usr/bin/perl +use strict; +use warnings; +my $f=shift if ($ARGV[0]||"") eq "-f"; +my %p; +my %f; +while (<>) { + chomp; + $p{$_}=1 if s/^Passed: //; + $f{$_}=1 if s/^TEST: //; +} +die "No PASS" if !%p&&!$f; +my %a=(%p,%f); +print(($p{$_}?"PASS":"FAIL")." test/hotspot/jtreg/$_\n") for sort keys(%a); diff --git a/bin/jtreg-pass-fail2 b/bin/jtreg-pass-fail2 new file mode 100755 index 0000000..490d38b --- /dev/null +++ b/bin/jtreg-pass-fail2 @@ -0,0 +1,21 @@ +#! /usr/bin/perl +use strict; +use warnings; +my %t; +my $test; +while (<>) { + chomp; + if (/^TEST: (.*)$/) { + die $_ if $test; + $test=$1; + next; + } + if (/^TEST RESULT: (Passed|Failed)[.]/) { + die $_ if !$test; + $t{$test}=$1 eq "Passed"; + $test=undef; + next; + } +} +die if defined $test; +print(($t{$_}?"PASS":"FAIL")." test/hotspot/jtreg/$_\n") for sort keys(%t); diff --git a/bin/makeinfo b/bin/makeinfo deleted file mode 100755 index 61e1f05..0000000 --- a/bin/makeinfo +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/sh -if [ "$*" = "--version" ];then - /usr/bin/makeinfo "$@" | sed 's/texinfo[^0-9]*/&4.40 - orig /' -else - exec /usr/bin/makeinfo "$@" -fi diff --git a/bin/nn b/bin/nn new file mode 100755 index 0000000..d8ce7f4 --- /dev/null +++ b/bin/nn @@ -0,0 +1,9 @@ +#! /bin/sh +if [ -z "$*" ];then + echo >&2 "No PID" +elif [ "$1" = "-p" ];then + shift + exec chrt -i -p 0 "$@" +else + exec chrt -i 0 "$@" +fi diff --git a/bin/openjdk-find-ccache-bug b/bin/openjdk-find-ccache-bug new file mode 100755 index 0000000..2c62ff3 --- /dev/null +++ b/bin/openjdk-find-ccache-bug @@ -0,0 +1,8 @@ +#! /bin/bash +# 0 * * * * ~/bin/openjdk-find-ccache-bug +for i in $(find ~ 2>/dev/null -name config.status);do + if grep -qw REWRITE_PATHS_RELATIVE $(echo $i|sed 's#/build/[^/]*/configure-support/config.status$##')/make/common/NativeCompilation.gmk 2>/dev/null;then + continue + fi + grep CCACHE_BASEDIR= $i /dev/null +done diff --git a/bin/psu b/bin/psu new file mode 100755 index 0000000..c8d9697 --- /dev/null +++ b/bin/psu @@ -0,0 +1,4 @@ +#! /bin/bash +/usr/bin/acpi|&grep -Eq 'Battery 0: Not charging|Battery 0: Charging|No support for device type: power_supply' || exit 1 +/usr/bin/grpcurl -v -plaintext -d '{"get_status":{}}' 192.168.1.1:9000 SpaceX.API.Device.Device/Handle &>/dev/null || exit 1 +exit 0 diff --git a/bin/rpmmerge b/bin/rpmmerge index baec0c1..a163307 100755 --- a/bin/rpmmerge +++ b/bin/rpmmerge @@ -21,7 +21,9 @@ for base in $l;do if test ! -f $orig then - cmp $base $base.rpmnew + if [ "$base" = "${base%/java.security}" ];then + cmp $base $base.rpmnew + fi mv -f $base.rpmnew $base continue fi diff --git a/bin/taillog b/bin/taillog index b58d0e0..fa7906f 100755 --- a/bin/taillog +++ b/bin/taillog @@ -1,2 +1,3 @@ #! /bin/bash -tail -fn0 `(file $(find /var/log -name ntpstats -prune -o -type f -print)|grep -v 'data$'|sed 's/:.*//'|grep -v '^\(/var/log/squid/store.log\|/var/log/acpid\|/var/log/btmp\|/var/log/maillog\)$';echo "$*"|tr ' ' '\n')|sort -u` # |grep -v ': bio too big device md3 (248 > 240)$' +ulimit -n 65536 +tail -fn0 `(file $(find /var/log -name ntpstats -prune -o -type f -print)|grep -v 'data$'|sed 's/:.*//'|grep -v '^\(/var/log/squid/store.log\|/var/log/acpid\|/var/log/btmp\|/var/log/journal/.*\|/var/log/maillog\|/var/log/sa/\)$';echo "$*"|tr ' ' '\n')|sort -u` # |grep -v ': bio too big device md3 (248 > 240)$' diff --git a/bin/vi-git-both-modified b/bin/vi-git-both-modified new file mode 100755 index 0000000..8f47892 --- /dev/null +++ b/bin/vi-git-both-modified @@ -0,0 +1,15 @@ +#! /bin/bash +fis="" +for fi in $(git status|sed -n 's/^ both modified: *//p');do + if ! grep -q '^<<<<<<< ' $fi;then + git add $fi + continue + fi + fis="$fis $fi" +done +if [ -z "$fis" ];then + echo done + exit 0 +fi +set -x +exec vim $fis diff --git a/src/Makefile b/src/Makefile index d23df8b..5625e19 100644 --- a/src/Makefile +++ b/src/Makefile @@ -4,8 +4,8 @@ CFLAGS+=-Wall -Wstrict-prototypes -ggdb3 #CFLAGS+=-O9 -fexpensive-optimizations -finline-functions #CFLAGS+=-pg -all: yasm2gas -# unmime flock pipebuf +all: streamfer-server streamfer-client +# yasm2gas unmime flock pipebuf pipebuf: pipebuf.c sh $< @@ -13,3 +13,8 @@ pipebuf: pipebuf.c yasm2gas: yasm2gas.c gcc -o $@ -Wall -g $< +streamfer-server: streamfer-server.C streamfer.h streamfer.C safeio.C safeio.h socket.C socket.h stringf.h + clang++ -o $@ -Wall -g streamfer-server.C streamfer.C safeio.C socket.C + +streamfer-client: streamfer-client.C streamfer.h streamfer.C safeio.C safeio.h socket.C socket.h stringf.h + clang++ -o $@ -Wall -g streamfer-client.C streamfer.C safeio.C socket.C diff --git a/src/safeio.C b/src/safeio.C new file mode 100644 index 0000000..fb76743 --- /dev/null +++ b/src/safeio.C @@ -0,0 +1,87 @@ +#include "safeio.h" +#include + +const SafeIOError exception_SafeIOError; +const SafeIOEOF exception_SafeIOEOF ; + +void read_safe(int fd,void *buf,size_t size) { + assert(size>0); + while (size>0) { + const ssize_t got(read(fd,buf,size)); + assert(got>=-1); + if (got<=0) { + warning("read_safe: fd=%d size=%zu: %s",fd,size,(got==0?"EOF":strerror(errno))); + // Do not throw ?exc1:exc2 as it would get upcasted. + if (got==0||(got==-1&&errno==ECONNRESET)) + throw exception_SafeIOEOF; + throw exception_SafeIOError; + } + assert(size_t(got)<=size); + buf=static_cast(buf)+got; + size-=got; + } +} + +void read_safe(FILE *f,void *buf,size_t size) { + assert(size>0); + assert(!ferror(f)); + assert(!feof(f)); + const size_t got(fread(buf,1/*size*/,size/*nmemb*/,f)); + if (ferror(f)||feof(f)) { + warning("read_safe: fd=%d size=%zu: %s",fileno(f),size,(ferror(f)?"error":"EOF")); + // Do not throw ?exc1:exc2 as it would get upcasted. + if (ferror(f)) + throw exception_SafeIOError; + throw exception_SafeIOEOF; + } + // return [...] which is less than nitems only if a read error or end-of-file is encountered. + if (size_t(got)!=size) { + warning("read_safe: fd=%d size=%zu got=%zu",fileno(f),size,got); + throw exception_SafeIOError; + } +} + +void write_safe(int fd,const void *buf,size_t size) { + assert(size>0); + const ssize_t got(write(fd,buf,size)); + if (got==-1) { + warning("Error writing %zu to fd %d: %m",size,fd); + throw exception_SafeIOError; + } + assert(got>0); + if (size_t(got)!=size) { + warning("Wrote only %zd of %zu to fd %d",got,size,fd); + throw exception_SafeIOEOF; + } +} + +void write_safe(FILE *f,const void *buf,size_t size) { + assert(size>0); + assert(!ferror(f)); + const size_t got(fwrite(buf,1/*size*/,size/*nmemb*/,f)); + if (ferror(f)) { + warning("Error writing %zu to FILE * of fd %d",size,fileno(f)); + throw exception_SafeIOError; + } + if (got!=size) { + warning("Wrote only %zu of %zu to FILE * of fd %d",got,size,fileno(f)); + throw exception_SafeIOEOF; + } +} + +void writev_safe_(int fd,const struct iovec *iov, int iovcnt) { + size_t size=0; + for (int ix=0;ix0); + const ssize_t got(writev(fd,iov,iovcnt)); + if (got==-1) { + warning("Error writing %zu iovec to fd %d: %m",size,fd); + throw exception_SafeIOError; + } + assert(got>0); + if (size_t(got)!=size) { + warning("Wrote only %zd of %zu iovec to fd %d",got,size,fd); + throw exception_SafeIOEOF; + } +} diff --git a/src/safeio.h b/src/safeio.h new file mode 100644 index 0000000..84a6706 --- /dev/null +++ b/src/safeio.h @@ -0,0 +1,89 @@ +#ifndef LIB_SAFEIO_H +#define LIB_SAFEIO_H + +#include "streamfer.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern const class SafeIOError:public exception { + virtual const char *what() const noexcept override { return "SafeIOError"; } +} exception_SafeIOError; +extern const class SafeIOEOF:public SafeIOError { + virtual const char *what() const noexcept override { return "SafeIOEOF"; } +} exception_SafeIOEOF; + +void read_safe(int fd ,void *buf,size_t len); +void read_safe(FILE *f,void *buf,size_t len); + +template void read_safe(F f, T &obj) { read_safe(f,&obj,sizeof(obj)); } + +template void read_safe(F f,vector &vec) { + size_t size; + read_safe(f,size); + vec.resize(size); + if (size) + read_safe(f,vec.data(),vec.size()*sizeof(vec[0])); +} + +template void read_safe(F f,deque &vec) { + size_t size; + read_safe(f,size); + vec.resize(size); + for (auto &elem:vec) + read_safe(f,elem); +} + +template void read_safe(F f,string &str) { + uint8_t len8=0; // false GCC warning + read_safe(f,len8); + str.resize(len8); + if (len8) + read_safe(f,&str[0],str.length()); +} + +template string read_safe_string(F f) { string str; read_safe(f,str); return str; } + +void write_safe(int fd ,const void *buf,size_t count); +void write_safe(FILE *f,const void *buf,size_t count); +void writev_safe_(int fd,const struct iovec *iov, int iovcnt); +constexpr iovec writev_iovec(const void *base,size_t len) { return iovec{const_cast(base),len}; } +static inline const/*FIXME:constexpr c_str()*/ iovec iovec_for_string(const string &str) { return iovec{const_cast(str.c_str()),str.length()}; } +template constexpr iovec iovec_for_object(T &object) { return iovec{reinterpret_cast(const_cast::type *>(&object)),sizeof(object)}; } +template void writev_safe(int fd,std::array iov) { + writev_safe_(fd,iov.data(),iovcnt); +} + +template void write_safe(F f,const T &obj) { write_safe(f,&obj,sizeof(obj)); } + +template void write_safe(F f,const vector &vec) { + const size_t size(vec.size()); + write_safe(f,size); + if (size) + write_safe(f,vec.data(),size*sizeof(vec[0])); +} + +template void write_safe(F f,const deque &vec) { + const size_t size(vec.size()); + write_safe(f,size); + for (const auto &elem:vec) + write_safe(f,elem); +} + +template void write_safe(F f,const string &str) { + const uint8_t len8(str.length()); + assert(len8==str.length()); + write_safe(f,len8); + if (!str.empty()) + write_safe(f,str.c_str(),len8); +} + +#endif /* LIB_SAFEIO_H */ diff --git a/src/socket.C b/src/socket.C new file mode 100644 index 0000000..2e0e4b8 --- /dev/null +++ b/src/socket.C @@ -0,0 +1,120 @@ +#include "socket.h" +#include "stringf.h" +#include +#include +#include + +int socket_bind(string host_port_str) { + const char *cs(strrchr(host_port_str.c_str(),':')); + const string node(!cs?"":host_port_str.substr(0,cs-host_port_str.c_str())); + const string service(!cs?host_port_str:cs+1); + struct addrinfo hints={}; // designated initializer: error: missing initializer for member ... [-Werror=missing-field-initializers] + hints.ai_family=AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype=SOCK_STREAM; + hints.ai_flags=AI_PASSIVE|AI_ADDRCONFIG; /* For wildcard IP address */ + struct addrinfo *result; + int err(getaddrinfo(node.empty()?NULL:node.c_str(),service.c_str(),&hints,&result)); + if (err) + fatal("<%s>:<%s>: %s",node.c_str(),service.c_str(),gai_strerror(err)); + int fd=-1; + struct addrinfo *rp; + for (rp=result;rp;rp=rp->ai_next) { + fd=socket(rp->ai_family,rp->ai_socktype,rp->ai_protocol); + if (fd==-1) + continue; + static const int int1(1); + err=setsockopt(fd,SOL_SOCKET,SO_REUSEADDR,&int1,sizeof(int1)); + assert(!err); + if (bind(fd,rp->ai_addr,rp->ai_addrlen)==0) + break; + err=close(fd); + assert(!err); + } + if (rp==NULL) + fatal("Cannot bind(): <%s>:<%s>: %m",node.c_str(),service.c_str()); + assert(fd!=-1); + freeaddrinfo(result); + err=listen(fd,SOMAXCONN); + assert(!err); + return fd; +} + +string sockaddr_string(const struct sockaddr *sockaddrp,socklen_t socklen) { + char hostname[NI_MAXHOST]; + char servname[NI_MAXSERV]; + const int err=getnameinfo(sockaddrp,socklen,hostname,sizeof(hostname),servname,sizeof(servname),NI_NUMERICSERV/*flags*/); + assert(!err); + return stringf("%s:%s",hostname,servname); +} + +string socket_name(int socket_fd) { + struct sockaddr sockaddr; + socklen_t socklen(sizeof(sockaddr)); + const int err(getsockname(socket_fd,&sockaddr,&socklen)); + assert(!err); + return sockaddr_string(&sockaddr,socklen); +} + +static void socket_setopt(int fd) { + static const int int1(1); + int err; + err=setsockopt(fd,SOL_SOCKET,SO_KEEPALIVE,&int1,sizeof(int1)); + assert(!err); + static const uint8_t tos(IPTOS_LOWDELAY); + err=setsockopt(fd,IPPROTO_IP,IP_TOS,&tos,sizeof(tos)); + assert(!err); + err=setsockopt(fd,IPPROTO_IP,TCP_NODELAY,&int1,sizeof(int1)); + assert(!err); +} + +int socket_accept(int listen_fd,function msgfunc) { + struct sockaddr sockaddr; + socklen_t socklen(sizeof(sockaddr)); + const int client_fd(accept(listen_fd,&sockaddr,&socklen)); + assert(client_fd>=0); + msgfunc(client_fd,sockaddr_string(&sockaddr,socklen)); + socket_setopt(client_fd); + return client_fd; +} + +int socket_connect(const string &host_port_str,unsigned retries) { + const char *cs(strrchr(host_port_str.c_str(),':')); + if (!cs) + fatal("Error parsing :: %s",host_port_str.c_str()); + const string node(host_port_str.substr(0,cs-host_port_str.c_str())); + const string service(cs+1); + struct addrinfo hints={}; // designated initializer: error: missing initializer for member ... [-Werror=missing-field-initializers] + hints.ai_family=AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype=SOCK_STREAM; + struct addrinfo *result; + int err=getaddrinfo(node.c_str(),service.c_str(),&hints,&result); + if (err) + fatal("Error parsing node+service: <%s>:<%s>: %s",node.c_str(),service.c_str(),gai_strerror(err)); + int fd; + struct addrinfo *rp; + for (unsigned retryno=0;retryno<1+retries;++retryno) { + if (retryno) { + warning("Sleeping 1 second for connect retry #%u/%u",retryno,retries); + err=sleep(1); + assert(!err); + } + for (rp=result;rp;rp=rp->ai_next) { + fd=socket(rp->ai_family,rp->ai_socktype,rp->ai_protocol); + if (fd==-1) + continue; + if (connect(fd,rp->ai_addr,rp->ai_addrlen)==0) + break; + err=close(fd); + assert(!err); + } + if (rp==NULL) + warning("Could not connect(): <%s>:<%s>: %m",node.c_str(),service.c_str()); + else + break; + } + freeaddrinfo(result); + if (rp==NULL) + fatal("Could not connect(), giving up"); + socket_setopt(fd); + return fd; +} diff --git a/src/socket.h b/src/socket.h new file mode 100644 index 0000000..19570ef --- /dev/null +++ b/src/socket.h @@ -0,0 +1,17 @@ +#ifndef LIB_SOCKET_H +#define LIB_SOCKET_H + +#include "streamfer.h" +#include +#include +#include +#include +#include + +int socket_bind(string host_port_str); +string sockaddr_string(const struct sockaddr *sockaddrp,socklen_t socklen); +string socket_name(int socket_fd); +int socket_accept(int listen_fd,function msgfunc); +int socket_connect(const string &host_port_str,unsigned retries); + +#endif /* LIB_SOCKET_H */ diff --git a/src/streamfer-client.C b/src/streamfer-client.C new file mode 100644 index 0000000..b2b9a61 --- /dev/null +++ b/src/streamfer-client.C @@ -0,0 +1,128 @@ +#include "safeio.h" +#include "socket.h" +#include "stringf.h" +#include +#include +#include + +static string get_string(FILE *f,const char *fn,const char *what) { + char buf[PATH_MAX]; + char *got=fgets(buf,sizeof(buf),f); + if (got!=buf) + fatal("Error reading %s from %s: %m",what,fn); + char *s(strchr(buf,'\n')); + if (!s) + fatal("Stored %s in %s is not newline-terminated",what,fn); + *s=0; + assert(!s[1]); + return buf; +} + +static void last_stored_write(const char *last_stored_fn,const string &filename,uint64_t offset) { + std::string last_tmp_fn(std::string(last_stored_fn)+".new"); + int last_tmp_fd(open(last_tmp_fn.c_str(),O_WRONLY|O_CREAT|O_TRUNC,0644)); + if (last_tmp_fd==-1) + fatal("Error storing filename to %s: %m",last_tmp_fn.c_str()); + write_safe(last_tmp_fd,filename.c_str(),filename.length()); + write_safe(last_tmp_fd,'\n'); + string offsetstr(stringf("%zu",(size_t)offset)); + write_safe(last_tmp_fd,offsetstr.c_str(),offsetstr.length()); + write_safe(last_tmp_fd,'\n'); + if (close(last_tmp_fd)) + fatal("Error closing %s: %m",last_tmp_fn.c_str()); + if (rename(last_tmp_fn.c_str(),last_stored_fn)) + fatal("Error renaming %s->%s: %m",last_tmp_fn.c_str(),last_stored_fn); +} + +static string my_basename(const string &path) { + size_t slash(path.find_last_of('/')); + if (slash!=string::npos) + return path.substr(slash+1); + return path; +} + +int main(int argc,char **argv) { + if (argc!=1+3) + fatal("streamfer-client : "); + int server_fd(socket_connect(string(argv[1]),0/*retries*/)); + string pattern(argv[2]); + write_safe(server_fd,pattern); + string last_found; + const char *last_stored_fn(argv[3]); + FILE *last_stored_f(fopen(last_stored_fn,"r")); + uint64_t offset(0); + if (!last_stored_f) { + if (errno!=ENOENT) + fatal("Error opening filename from %s: %m",last_stored_fn); + } else { + last_found= get_string(last_stored_f,last_stored_fn,"filename") ; + string offsetstr(get_string(last_stored_f,last_stored_fn,"offset" )); + char *end; + errno=0; + unsigned long ul(strtoul(offsetstr.c_str(),&end,0)); + if (errno||(end&&*end)) + fatal("Error converting offset from %s: %s",last_stored_fn,offsetstr.c_str()); + offset=ul; + int gotint(fgetc(last_stored_f)); + if (gotint!=EOF) + fatal("Stored filename in %s has excessive data after filename %s and offset %s",last_stored_fn,last_found.c_str(),offsetstr.c_str()); + int err(fclose(last_stored_f)); + assert(!err); + } + write_safe(server_fd,last_found); + write_safe(server_fd,offset); + { struct stat statbuf; + string last_found_basename(my_basename(last_found)); + if (stat(last_found_basename.c_str(),&statbuf)) { + static const struct timespec mtim_zero{}; + write_safe(server_fd,mtim_zero); + } else + write_safe(server_fd,statbuf.st_mtim); + } + string last_got; + try { + read_safe(server_fd,last_got); + } catch (SafeIOError) { + fatal("SafeIOError reading from the server %s",argv[1]); + } + if (last_got.empty()) { + warning("No more files to transfer"); + exit(EXIT_SUCCESS); + } + struct timespec mtim; + read_safe(server_fd,mtim); + int file_fd; + string file_name(my_basename(last_got));; + if (last_found==last_got) { + file_fd=open(file_name.c_str(),O_WRONLY); + if (file_fd==-1) + fatal("Error opening for write %s: %m",file_name.c_str()); + off_t got(lseek(file_fd,offset,SEEK_SET)); + if ((uint64_t)got!=offset) + fatal("Error seeking in %s to %zu, got: %zu",file_name.c_str(),(size_t)offset,(size_t)got); + } else { + offset=0; + file_fd=open(file_name.c_str(),O_WRONLY|O_CREAT,0644); + if (file_fd==-1) + fatal("Error creating %s: %m",file_name.c_str()); + } + last_stored_write(last_stored_fn,last_got,offset); + uint64_t transferred(transfer(server_fd,"server fd",file_fd,file_name.c_str())); + if (!transferred) { + struct stat statbuf; + int err(fstat(file_fd,&statbuf)); + assert(!err); + if (memcmp(&mtim,&statbuf.st_mtim,sizeof(mtim))==0) + return EXIT_FAILURE; + } + offset+=transferred; + struct timespec mtim2[2]; + mtim2[0]=mtim; // atime + mtim2[1]=mtim; // mtime + if (futimens(file_fd,mtim2)) + fatal("Error setting timestamp of %s: %m",file_name.c_str()); + if (close(file_fd)) + fatal("Error closing %s: %m",file_name.c_str()); + last_stored_write(last_stored_fn,last_got,offset); + return EXIT_SUCCESS; +} diff --git a/src/streamfer-server.C b/src/streamfer-server.C new file mode 100644 index 0000000..5a0bd8a --- /dev/null +++ b/src/streamfer-server.C @@ -0,0 +1,212 @@ +#include "safeio.h" +#include "socket.h" +#include "stringf.h" +#include +#include +#include +#include +#include +#include +#include + +// https://stackoverflow.com/a/8615450/2995591 +#include // glob(),globfree() +#include // memset() +#include +#include + +static std::vector cxxglob(const std::string pattern) { + glob_t glob_result; + memset(&glob_result,0,sizeof(glob_result)); + int return_value=glob(pattern.c_str(),GLOB_TILDE,NULL,&glob_result); + if (return_value) + fatal("glob() failed with return_value %s",return_value); + vector filenames; + filenames.reserve(glob_result.gl_pathc); + for (size_t i = 0; i < glob_result.gl_pathc; ++i) + filenames.push_back(string(glob_result.gl_pathv[i])); + globfree(&glob_result); + return filenames; +} + +// FIXME: Use C++17 +static bool fd_is_open(const char *execname,const char *fn) { + const char slashproc[]("/proc"); + DIR *dir(opendir(slashproc)); + if (!dir) + fatal("Cannot opendir %s: %m",slashproc); + bool retval(false); + for (;;) { + errno=0; + const struct dirent *de=readdir(dir); + if (!de) { + if (errno) + fatal("Cannot readdir %s: %m",slashproc); + break; + } + if (!isdigit(de->d_name[0])) + continue; + + char buf[PATH_MAX]; + ssize_t got(readlinkat(dirfd(dir),stringf("%s/exe",de->d_name).c_str(),buf,sizeof(buf))); + if (got==-1||got==sizeof(buf)) + continue; + buf[got]=0; + char *s=strrchr(buf,'/'); + if (!s) + continue; + if (strcmp(s+1,execname)!=0) + continue; + + string procpidfd(stringf("/proc/%s/fd",de->d_name)); + DIR *fddir(opendir(procpidfd.c_str())); + if (!fddir) + fatal("Cannot opendir %s: %m",procpidfd.c_str()); + for (;;) { + errno=0; + const struct dirent *de=readdir(fddir); + if (!de) { + if (errno) + fatal("Cannot readdir %s: %m",procpidfd.c_str()); + break; + } + if (!isdigit(de->d_name[0])) + continue; + char buf[PATH_MAX]; + ssize_t got(readlinkat(dirfd(fddir),de->d_name,buf,sizeof(buf))); + if (got==-1||got==sizeof(buf)) + continue; + buf[got]=0; + if (strcmp(buf,fn)==0) { + retval=true; + break; + } + } + if (closedir(fddir)) + fatal("Cannot closedir %s: %m",procpidfd.c_str()); + if (retval) + break; + } + if (closedir(dir)) + fatal("Cannot closedir %s: %m",slashproc); + return retval; +} + +int main(int argc,char **argv) { + static struct sigaction sigchld; + sigchld.sa_handler=SIG_DFL; + sigchld.sa_flags=SA_NOCLDWAIT; + int err(sigaction(SIGCHLD,&sigchld,nullptr)); + assert(!err); + + if (argc!=1+2&&argc!=1+3) + fatal("streamfer-server [:] [follow-fd-of-executable-basename]"); + string prefix; + if (argc>=1+2&&*argv[2]) + prefix=argv[2]; + const char *execname(nullptr); + if (argc>=1+3) + execname=argv[3]; + int listen_fd(socket_bind(argv[1])); + int client_fd; + for (;;) { + client_fd=socket_accept(listen_fd,[&](int client_fd,string addr) { + warning("%d:%s",client_fd,addr.c_str()); + }); + int child(fork()); + assert(child!=-1); + if (!child) + break; + int err(close(client_fd)); + assert(!err); + } + err=close(listen_fd); + assert(!err); + + string pattern(read_safe_string(client_fd)); + std::vector matched(cxxglob(pattern)); + for (size_t ix=0;ix=0) + fatal("glob: strcmp(\"%s\",\"%s\")=%d",a.c_str(),b.c_str(),err); + } + string last(read_safe_string(client_fd)); + size_t lastix(SIZE_MAX); + for (size_t ix=0;ix0) + assert(lastix==SIZE_MAX); + else if (lastix==SIZE_MAX) + lastix=ix; + } + if (lastix==SIZE_MAX) + fatal("Requested too new file"); + uint64_t offset; + read_safe(client_fd,offset); + struct timespec mtim; + read_safe(client_fd,mtim); + const string *fnp; + int file_fd=-1; + struct stat statbuf; + for (;lastix(uint64_t)statbuf.st_size) + warning("File %s has transferred %zu < %zu which is its size",fn.c_str(),(size_t)offset,(size_t)statbuf.st_size); + if (lastix==matched.size()-1&&execname) + break; + err=close(file_fd); + assert(!err); + offset=0; + } + if (file_fd==-1) { + string empty(""); + write_safe(client_fd,empty); + fatal("No more files to transfer"); + } + const string &fn(*fnp); + const char *fn_canon(nullptr); + if (!prefix.empty()||execname) { + fn_canon=realpath(fn.c_str(),nullptr); + if (!fn_canon) + fatal("realpath %s: %m",fn.c_str()); + } + if (fn!=last) + offset=0; + if (!prefix.empty()&&strncmp(prefix.c_str(),fn_canon,prefix.length())!=0) + fatal("prefix=\"%s\" realpath=\"%s\"",prefix.c_str(),fn_canon); + warning("%s @%zu",fn.c_str(),(size_t)offset); + write_safe(client_fd,fn); + write_safe(client_fd,statbuf.st_mtim); + off_t got(lseek(file_fd,offset,SEEK_SET)); + assert((uint64_t)got==offset); + struct pollfd fds; + fds.fd=client_fd; + fds.events=POLLIN|POLLPRI|POLLRDHUP; + for (;;) { + transfer(file_fd,fn.c_str(),client_fd,"client fd"); + if (!fn_canon||!fd_is_open(execname,fn_canon)) + break; + int err(poll(&fds,1,1000/*ms*/)); + if (err==-1) + fatal("poll client fd: %m"); + if (err==1) + fatal("poll client fd: revents=0x%x",fds.revents); + assert(err==0); + } +} diff --git a/src/streamfer.C b/src/streamfer.C new file mode 100644 index 0000000..7f2e432 --- /dev/null +++ b/src/streamfer.C @@ -0,0 +1,51 @@ +#include "streamfer.h" +#include + +static void vwarning(const char *msg,va_list ap) { + int err=vfprintf(stderr,msg,ap); + assert(err>0); + fputc('\n',stderr); +} + +void warning(const char *msg,...) { + va_list ap; + va_start(ap,msg); + vwarning(msg,ap); + va_end(ap); +} + +void fatal(const char *msg,...) { + va_list ap; + va_start(ap,msg); + vwarning(msg,ap); + va_end(ap); + exit(EXIT_FAILURE); +} + +uint64_t transfer(int from_fd,const char *from_fn,int to_fd,const char *to_fn) { + size_t total(0); + uint8_t buffer[0x10000]; + size_t buffer_filled(0); + for (;;) { + if (buffer_filled==0) { + ssize_t got(read(from_fd,buffer+buffer_filled,sizeof(buffer)-buffer_filled)); + if (got==-1) + fatal("Error reading %s: %m",from_fn); + if (got==0) + return total; + buffer_filled=got; + assert(buffer_filled<=sizeof(buffer)); + } + size_t buffer_written(0); + while (buffer_written0); + buffer_written+=got; + assert(buffer_written<=buffer_filled); + } + total+=buffer_filled; + buffer_filled=0; + } +} diff --git a/src/streamfer.h b/src/streamfer.h new file mode 100644 index 0000000..d692137 --- /dev/null +++ b/src/streamfer.h @@ -0,0 +1,19 @@ +#ifndef STREAMFER_H +#define STREAMFER_H 1 + +#include +#include +#include +#include +#include +using namespace std; + +#define PRINTF(f,a) __attribute__((format(printf,f,a))) +#define UNUSED __attribute__((unused)) + +void warning(const char *msg,...); +void fatal(const char *msg,...); + +uint64_t transfer(int from_fd,const char *from_fn,int to_fd,const char *to_fn); + +#endif // STREAMFER_H diff --git a/src/stringf.h b/src/stringf.h new file mode 100644 index 0000000..2e194fc --- /dev/null +++ b/src/stringf.h @@ -0,0 +1,45 @@ +#ifndef LIB_STRINGF_H +#define LIB_STRINGF_H + +#include "streamfer.h" + +// template: format string is not a string literal (potentially insecure) [-Werror,-Wformat-security] + +static inline PRINTF(2,0) string vstringf_const(size_t sizemax,const char *fmt,va_list ap) { + string str(sizemax+1,0); + const int got=vsnprintf(&str[0],str.length(),fmt,ap); + if (got!=ssize_t(sizemax)) + fatal("stringf*(): expected=%zu got=%d fmt=\"%s\" str=\"%s\"",sizemax,got,fmt,str.c_str()); + str.resize(sizemax); + return str; +} + +static inline PRINTF(2,3) string stringf_const(size_t sizemax,const char *fmt,...) { + va_list ap; + va_start(ap,fmt); + const string str(vstringf_const(sizemax,fmt,ap)); + va_end(ap); + return str; +} + +static inline PRINTF(1,2) string stringf(const char *fmt,...) { + // 189: for i in activetick.data/;do find $i -name "*.xz";done|xargs xz -dc|perl -lne 'BEGIN{$m=0;}if (length($_)>$m){$m=length $_;print $m;}' + static const int sizetry(192-1/*+1->malloc()*/); + string str(sizetry,0); + va_list ap; + va_start(ap,fmt); + const int got(vsnprintf(&str[0],sizetry,fmt,ap)); + va_end(ap); + assert(got>=0); + if (got>=sizetry) { + str.resize(got+1); + va_start(ap,fmt); + const int got2(vsnprintf(&str[0],str.length(),fmt,ap)); + va_end(ap); + assert(got2==got); + } + str.resize(got); + return str; +} + +#endif /* LIB_STRINGF_H */