pax_global_header00006660000000000000000000000064131776672000014523gustar00rootroot0000000000000052 comment=f233e02dea581746ff8301b46f1f040a99fe0879 xe-0.11/000077500000000000000000000000001317766720000120605ustar00rootroot00000000000000xe-0.11/Makefile000066400000000000000000000010561317766720000135220ustar00rootroot00000000000000ALL=xe ZSHCOMP=_xe CFLAGS=-g -O2 -Wall -Wno-switch -Wextra -Wwrite-strings DESTDIR= PREFIX=/usr/local BINDIR=$(PREFIX)/bin MANDIR=$(PREFIX)/share/man ZSHCOMPDIR=$(PREFIX)/share/zsh/site-functions all: $(ALL) clean: FRC rm -f $(ALL) check: FRC all prove -v install: FRC all mkdir -p $(DESTDIR)$(BINDIR) $(DESTDIR)$(MANDIR)/man1 $(DESTDIR)$(ZSHCOMPDIR) install -m0755 $(ALL) $(DESTDIR)$(BINDIR) install -m0644 $(ALL:=.1) $(DESTDIR)$(MANDIR)/man1 install -m0644 $(ZSHCOMP) $(DESTDIR)$(ZSHCOMPDIR) README: xe.1 mandoc -Tutf8 $< | col -bx >$@ FRC: xe-0.11/NEWS.md000066400000000000000000000020311317766720000131520ustar00rootroot00000000000000## HEAD ## 0.11 (2017-11-05) * Add zsh completion. * Fix `-F` to exit as soon as possible when a command failed. * Fix when using `-s` and `-p` together. * Better error handling for some rare situations. * Made test suite more robust. ## 0.10 (2017-10-30) * New flag `-p` to enable "percent rules" ala make(1), see the man page for details. * New flag `-q` to run commands, hiding their output. * Fix issues launching commands on FreeBSD and DragonFlyBSD. ## 0.9 (2017-07-17) * New flag `-L` to enable line buffering on output. * New flag `-LL` to prefix output lines with `$ITER`. * `-j` now can scale with the available CPU cores (e.g. `-j0.5x` to use half the cores). * `-vv` will trace `$ITER` and `exit status`. * Fix issues related to having more children than we knew of. * Documentation fixes and improvement by Larry Hynes. ## 0.8 (2017-06-02) * Only read new arguments when we immediately need them ## 0.7 (2017-04-13) * Error when the `-A` separator does not appear * Bug: fix quoting of empty strings * Documentation fixes xe-0.11/README000066400000000000000000000157401317766720000127470ustar00rootroot00000000000000XE(1) General Commands Manual XE(1) NAME xe – execute a command for every argument SYNOPSIS xe [-0FLRnqv] [-I replace-arg] [-N maxargs] [-j maxjobs] command ... xe [flags ...] -p pattern command ... [+ pattern command ...]... xe [flags ...] -f argfile command ... xe [flags ...] -s shellscript xe [flags ...] -a command ... -- args ... xe [flags ...] -A argsep command ... argsep args ... DESCRIPTION The xe utility constructs command lines from specified arguments, combining some of the best features of xargs(1) and apply(1). xe means “execute for every ...”. xe supports different methods to specify arguments to commands: command ... By default, arguments - separated by newlines - are read from the standard input. The resulting command is constructed from the command line parameters, replacing replace-arg with the read argument, and is executed with execvp(3). In this mode, no shell is involved and replace-arg must appear as a word on its own, i.e. ‘foo {} bar’ will work, but ‘foo{} bar’ will not, where {} is the default value for replace-arg. If no argument is specified, the default is ‘printf %s\n’. -f argfile Read arguments from argfile, instead of the standard input. This does not close the standard input for execution, it is passed to the forked process. -s shellscript In this mode, the single parameter shellscript is executed using sh -c. In the script, the specified arguments can be accessed using $1, $2, ... For example: echo 'a\nb' | xe -N2 -s 'echo $2 $1' -a command ... -- args ... In this mode, everything after -- is passed as args to command. -A argsep command ... argsep args ... Same as -a, but the custom argument separator argsep is used to distinguish between command and its args. The options are as follows: -0 Input filenames are separated by NUL bytes (instead of newlines, which is the default) -F Fatal: stop and exit when a command execution fails. -L Run the resulting commands with line-buffered output; lines from two jobs will not interleave. When used twice, or with -vv, also prefix each line with the number of the job (see ENVIRONMENT) in such a manner that the output can be piped to ‘sort -snk1’ to group it. -R Return with status 122 when no arguments have been specified (instead of 0, the default). xe never executes a command when no arguments are specified. -n Dry run: don't run the resulting commands, just print them. -q Quiet mode: redirect standard output and standard error of commands to /dev/null. -v Verbose: print commands to standard error before running them. When used twice, also print job id and exit status for each command. -p Enable make(1)-style percent rules. The first argument of command ... is regarded as a pattern, see PERCENT RULES below. Patterns without a slash (or ‘**’) are matched against the basenames only. Multiple runs of patterns and commands are separated by ‘+’. Only the first matching percent rule is executed; in case no pattern matches, no command is run. -I replace-arg Replace first occurrence of replace-arg (default: {}) in the resulting command with the argument(s). Pass an empty replace-arg to disable the replace function. Contrary to xargs(1) this will expand into multiple arguments when needed. -N maxargs Pass up to maxargs arguments to each command (default: 1). Using -N0 will pass as many arguments as possible. -j maxjobs Run up to maxjobs processes concurrently. Using -j0 will run as many processes as there are CPU cores running. If maxjobs ends with an ‘x’, it is regarded as a multiplier of the number of running CPU cores (rounded down, but using at least one core). PERCENT RULES The percent rules of xe are similar to the globs of sh(1) or fnmatch(3): ‘?’ matches a single character that is not ‘/’. ‘/’ matches one or multiple ‘/’ in the string. ‘*’ matches zero or more characters, but never ‘/’. ‘**’ matches zero or more characters, including ‘/’. Note that all of these also match leading dots in file names. ‘{a,b,c}’ matches either a, b or c. ‘[abc]’ matches one of the characters abc (but never ‘/’). ‘[!abc]’ matches all characters but abc. Alternatively, ‘[^abc]’ can be used too. ‘[a-c]’ matches any character in the range between a and c inclusive. In character ranges, characters can be escaped using a backslash. In the pattern, a single occurrence of ‘%’ matches one or more characters, and replaces the first occurrence of ‘%’ with the matched string in the remaining arguments, which are then used as the command to be executed. ENVIRONMENT The environment variable ITER is passed to the child process and incremented on each command execution. EXIT STATUS xe follows the convention of GNU and OpenBSD xargs: 0 on success 123 if any invocation of the command exited with status 1 to 125. 124 if the command exited with status 255 125 if the command was killed by a signal 126 if the command cannot be run 127 if the command was not found 1 if some other error occurred Additionally, 122 is returned when -R was passed and the command was never executed. EXAMPLES Compress all .c files in the current directory, using all CPU cores: xe -a -j0 gzip -- *.c Remove all empty files, using lr(1): lr -U -t 'size == 0' | xe -N0 rm Convert .mp3 to .ogg, using all CPU cores: xe -a -j0 -s 'ffmpeg -i "${1}" "${1%.mp3}.ogg"' -- *.mp3 Same, using percent rules: xe -a -j0 -p %.mp3 ffmpeg -i %.mp3 %.ogg -- *.mp3 Similar, but hiding output of ffmpeg, instead showing spawned jobs: xe -ap -j0 -vvq '%.{m4a,ogg,opus}' ffmpeg -y -i {} out/%.mp3 -- * SEE ALSO apply(1), parallel(1), xapply(1), xargs(1) AUTHORS Leah Neukirchen LICENSE xe is in the public domain. To the extent possible under law, the creator of this work has waived all copyright and related or neighboring rights to this work. http://creativecommons.org/publicdomain/zero/1.0/ Void Linux November 3, 2017 Void Linux xe-0.11/README.md000066400000000000000000000022211317766720000133340ustar00rootroot00000000000000## xe: simple xargs and apply replacement `xe` is a new tool for constructing command lines from file listings or arguments, which includes the best features of `xargs(1)` and `apply(1)`. `xe` means "execute for every ...". ## Benefits Over xargs: * Sane defaults (behaves like `xargs -d'\n' -I{} -n1 -r`). * No weird parsing, arguments are separated linewise or by NUL byte. * Can also take arguments from command-line. * No shell involved unless `-s` is used. * `{}` replacing possible with multiple arguments. * Support for patterns to run different commands depending on the argument. Over apply: * Parallel mode. * Sane argument splitting. * Can use shell-syntax instead of escape characters. ## [Man page](README) ## Installation Use `make all` to build, `make install` to install relative to `PREFIX` (`/usr/local` by default). The `DESTDIR` convention is respected. You can also just copy the binary into your `PATH`. ## Copyright xe is in the public domain. To the extent possible under law, Leah Neukirchen has waived all copyright and related or neighboring rights to this work. http://creativecommons.org/publicdomain/zero/1.0/ xe-0.11/_xe000066400000000000000000000013411317766720000125550ustar00rootroot00000000000000#compdef xe _arguments -s -S -A '-*' : \ '-f[read argument from file]:file:_files' \ '-s[use shell]:script: ' \ '-a[use arguments after --]' \ '-A[use arguments after separator]:separator: ' \ '-0[read arguments NUL-separated]' \ '-F[fatal: stop when a command fails]' \ '*-L[line-buffer output]' \ '-R[return 122 when no arguments are specified]' \ '-n[dry run (do not run commands)]' \ '-q[quiet mode]' \ '*-v[verbose mode]' \ '-I[argument replacement string]:string: ' \ '-j[maximum number of parallel command executions]:number: ' \ '(-p)-N[maximum number of arguments per command]:number: ' \ '(-N)-p[enable percent rules]:percent pattern: ' \ '(-):command name: _command_names -e' \ '*::arguments:_normal' xe-0.11/t/000077500000000000000000000000001317766720000123235ustar00rootroot00000000000000xe-0.11/t/errors.t000077500000000000000000000026621317766720000140350ustar00rootroot00000000000000#!/bin/sh export "PATH=.:$PATH" printf '1..13\n' printf '# error handling\n' tap3 'exit code on success' <<'EOF' xe >>>= 0 EOF tap3 'exit code on other error' <<'EOF' true | xe -j NaN >>>= 1 EOF tap3 'exit code on when command fails with 1-125' <<'EOF' xe -s 'exit 42' <<< a >>>= 123 EOF tap3 'exit code on when command fails with 255' <<'EOF' xe -s 'exit 255' <<< a >>>= 124 EOF tap3 'exit code when process was killed' <<'EOF' xe perl -e 'kill "KILL", $$' <<< a >>>= 125 EOF # possible false positive result when exec returns ENOENT instead of ENOTDIR here tap3 'exit code when command cannot be run' <<'EOF' xe /dev/null/calc.exe <<< a >>>= 126 EOF tap3 'exit code when command was not found' <<'EOF' xe /bin/calc.exe <<< a >>>= 127 EOF tap3 'exit code on empty input when run with -R' <<'EOF' xe -R echo a >>>= 122 EOF tap3 'doesn'\''t stop on errors by default' <<'EOF' xe -s 'if [ b = $1 ]; then false; else echo $1; fi' <<< a b c >>> a c >>>= 123 EOF tap3 'stops on first error with -F' <<'EOF' xe -F -s 'if [ b = $1 ]; then false; else echo $1; fi' <<< a b c >>> a >>>= 123 EOF tap3 'should close stdin when arguments were read from it' <<'EOF' xe -s 'sed q' <<< a b c >>> EOF tap3 'should not close stdin when arguments were read from command line' <<'EOF' yes | xe -a -s "sed q" -- 1 2 3 >>> y y y EOF tap3 'should not close stdin when arguments were read from file' <<'EOF' yes | xe -f NEWS.md -s 'sed q' 2>&1 | sed 3q >>> y y y EOF xe-0.11/t/limits.t000077500000000000000000000005571317766720000140230ustar00rootroot00000000000000#!/bin/sh export "PATH=.:$PATH" printf '1..2\n' printf '# limit checks, expecting maximal POSIX limits available\n' tap3 'argscap check' <<'EOF' dd if=/dev/zero bs=1 count=17711 2>/dev/null | tr "\0" "\012" | xe -N0 -s 'echo $#' >>> 8187 8187 1337 EOF tap3 'argslen check' <<'EOF' perl -e 'print "x"x8000, "\n" for 1..42' | xe -N0 -s 'echo $#' >>> 16 16 10 EOF xe-0.11/t/percent.t000077500000000000000000000027241317766720000141600ustar00rootroot00000000000000#!/bin/sh export "PATH=.:$PATH" printf '1..13\n' printf '# percent rules\n' tap3 'literal matches' <<'EOF' xe -ap bcd echo found -- abc bcd defg >>> found EOF tap3 'multiple patterns' <<'EOF' xe -ap one echo 1 + two echo 2 + three echo 3 -- zero one two three four five >>> 1 2 3 EOF tap3 '{} expansion' <<'EOF' xe -ap bcd echo {} -- abc bcd defg >>> bcd EOF tap3 '% expansion' <<'EOF' xe -ap bcd echo % -- abc bcd defg >>> bcd EOF tap3 'dirnames' <<'EOF' xe -ap bcd echo % -- abc bcd /tmp/bcd /tmp/abc >>> bcd /tmp/bcd EOF tap3 '? glob' <<'EOF' xe -ap "b?d" echo % -- abc bcd b3d defg >>> bcd b3d EOF tap3 '* glob' <<'EOF' xe -ap "b*d" echo % -- bd bed bad bugged bx zbd b/d >>> bd bed bad bugged EOF tap3 'multiple * glob' <<'EOF' xe -ap "b*g*d" echo % -- bd bed bugged bx zbd bagdad badger ba/gd/ad >>> bugged bagdad EOF tap3 'multiple ** glob' <<'EOF' xe -ap "b**g**d" echo % -- bd bed bugged bx zbd bagdad badger ba/gd/ad >>> bugged bagdad ba/gd/ad EOF tap3 '/ slash' <<'EOF' xe -ap a/b echo 1 + c///d echo 2 + "*" echo 3 -- a/b a//b a/ b c/d /c////d >>> 1 1 3 3 2 3 EOF tap3 '[] ranges' <<'EOF' xe -ap "[abc]" echo "1%" + "[d-g]" echo "2%" + "[^xyz-]" echo "3%" + "[!-vw]" echo "4%" + % echo "5%" -- a c d e g h w x - >>> 1a 1c 2d 2e 2g 3h 3w 4x 5- EOF tap3 '{} alternation' <<'EOF' xe -ap "{a,bc,def*}" echo % -- x a abc bc bcd def defx xdef >>> a bc def defx EOF tap3 '% match' <<'EOF' xe -ap %.c echo obj/%.o -- foo.c bar.cc meh/quux.c >>> obj/foo.o meh/obj/quux.o EOF xe-0.11/t/regressions.t000077500000000000000000000002501317766720000150530ustar00rootroot00000000000000#!/bin/sh export "PATH=.:$PATH" printf '1..1\n' printf '# regressions\n' tap3 '0fb64a4 quoting of empty strings' <<'EOF' xe -N2 -v true <<< foo >>>2 true foo '' EOF xe-0.11/t/simple.t000077500000000000000000000043051317766720000140060ustar00rootroot00000000000000#!/bin/sh export "PATH=.:$PATH" printf '1..27\n' printf '# simple tests\n' tap3 'single argument run' <<'EOF' xe echo <<< 1 2 3 >>> 1 2 3 EOF tap3 'dual argument run' <<'EOF' xe -N2 echo <<< 1 2 3 4 5 >>> 1 2 3 4 5 EOF tap3 'unlimited argument run' <<'EOF' xe -N0 echo <<< 1 2 3 4 5 >>> 1 2 3 4 5 EOF tap3 'empty input run' <<'EOF' true | xe echo a >>> EOF tap3 'dry run' <<'EOF' xe -n echo x <<< a b c >>>2 echo x a echo x b echo x c EOF tap3 'dry run quoting' <<'EOF' xe -n echo x <<< a b b c >>>2 echo x a echo x 'b b' echo x c EOF tap3 'verbose run' <<'EOF' xe -v echo x <<< a b c >>> x a x b x c >>>2 echo x a echo x b echo x c EOF tap3 'with no command' <<'EOF' xe -N2 <<< 1 2 3 >>> 1 2 3 EOF tap3 'using {}' <<'EOF' xe echo a {} x <<< 1 2 3 >>> a 1 x a 2 x a 3 x EOF tap3 'using {} twice' <<'EOF' xe echo {} x {} <<< 1 2 3 >>> 1 x {} 2 x {} 3 x {} EOF tap3 'using -I%' <<'EOF' xe -I% echo {} x % <<< 1 2 3 >>> {} x 1 {} x 2 {} x 3 EOF tap3 'using -I "" to disable' <<'EOF' xe -I "" echo {} x % <<< 1 2 3 >>> {} x % 1 {} x % 2 {} x % 3 EOF tap3 'using {} with multiple arguments' <<'EOF' xe -N2 echo a {} x {} <<< 1 2 3 >>> a 1 2 x {} a 3 x {} EOF tap3 'using -0' <<'EOF' printf "foo\0bar\0quux" | xe -0 echo >>> foo bar quux EOF tap3 'using -a' <<'EOF' xe -a echo -- 1 2 3 >>> 1 2 3 EOF tap3 'using -a with no arguments' <<'EOF' xe -a echo >>> EOF tap3 'using -a with no command' <<'EOF' xe -N2 -a -- 1 2 3 >>> 1 2 3 EOF tap3 'using -A%' <<'EOF' xe -A% echo -- % 1 2 3 >>> -- 1 -- 2 -- 3 EOF tap3 'using -A% with no arguments' <<'EOF' xe -A% echo >>>2 xe: '-A %' used but no separator '%' found in command line. EOF tap3 'using -A% with no command' <<'EOF' xe -N2 -A% % 1 2 3 >>> 1 2 3 EOF tap3 'using -f' <<'EOF' echo notme | xe -f Makefile echo >>> /DESTDIR/ EOF tap3 'using -s' <<'EOF' xe -s 'echo x$1' <<< 1 2 3 >>> x1 x2 x3 EOF tap3 'using -s with -N0' <<'EOF' xe -N0 -s 'echo x$@' <<< 1 2 3 >>> x1 2 3 EOF tap3 'using -s with -a' <<'EOF' xe -s 'echo x$@' -a 1 2 3 >>> x1 x2 x3 EOF tap3 'using -s with -a' <<'EOF' xe -a -s 'echo x$@' 1 2 3 >>> x1 x2 x3 EOF tap3 'using -s with -a' <<'EOF' xe -a -s 'echo x$@' -- 1 2 3 >>> x1 x2 x3 EOF tap3 'with ITER' <<'EOF' xe -a -s 'echo $ITER' -- a b c >>> 1 2 3 EOF xe-0.11/t/slow.t000077500000000000000000000011701317766720000134760ustar00rootroot00000000000000#!/bin/sh export "PATH=.:$PATH" printf '1..4\n' printf '# slow tests\n' tap3 'is eager' <<'EOF' { { echo 1; sleep 1; echo 11 >/dev/stderr; echo 2; } | xe echo; } 2>&1 >>> 1 11 2 EOF tap3 'using -L' <<'EOF' { echo 1; sleep 1; echo 2; sleep 1; echo 3; } | xe -j2 -L -s 'printf $1; sleep 1; echo $1' >>> 11 22 33 EOF tap3 'using -LL' <<'EOF' { echo 1; sleep 1; echo 2; sleep 1; echo 3; } | xe -j2 -LL -s 'printf $1; sleep 1; echo $1' >>> 0001= 11 0002= 22 0003= 33 EOF tap3 'using -vvL' <<'EOF' { echo 1; sleep 1; echo 2; sleep 1; echo 3; } | xe -j2 -vvL -s 'printf %s $1; sleep 1; echo $1' | wc -l >>> /\s*9$/ EOF xe-0.11/tap3000077500000000000000000000055261317766720000126650ustar00rootroot00000000000000#!/usr/bin/env perl # tap3 [DESC] - check output/error/status of a command against a specification # # A tiny variant of shelltestrunner (format v1), just takes one test # case and outputs a TAP line. # # Input format: # # CMD # <<< # INPUT # >>> # OUTPUT # >>> /OUTPUT REGEX/ # >>>2 # STDERR # >>>2 /STDERR REGEX/ # >>>= STATUS # >>>= !STATUS # # All but CMD are optional and can be put in any order, # Regex variants can be repeated, all patterns must match. # By default, STATUS is set to 0 and STDERR assumed empty. # # To the extent possible under law, the creator of this work has waived # all copyright and related or neighboring rights to this work. # http://creativecommons.org/publicdomain/zero/1.0/ use strict; use warnings; use Symbol 'gensym'; use IPC::Open3; my $cmd = ""; my ($input, $output, @output_rx, $stderr, @stderr_rx, $status, $status_not); my $ignored = ""; my $var = \$cmd; while () { if (/^#!? /) { next; } if (/^<<<$/) { $var = \$input; $input = ""; next; } if (/^>>>$/) { $var = \$output; $output = ""; next; } if (/^>>>2$/) { $var = \$stderr; $stderr = ""; next; } if (/^>>>\s*\/(.*)\/$/) { push @output_rx, $1; next; } if (/^>>>2\s*\/(.*)\/$/) { push @stderr_rx, $1; next; } if (/^>>>=\s+(\d+)$/) { $var = \$ignored; $status = $1; next; } if (/^>>>=\s+!(\d+)$/) { $var = \$ignored; $status_not = $1; next; } $$var .= $_; } chomp($cmd); die "No command to check given\n" if !$cmd; my ($wtr, $rdr); my $err = gensym; my $pid = open3($wtr, $rdr, $err, "/bin/sh", "-c", $cmd); my $desc = shift || $cmd; $desc =~ s/\n.*//; print $wtr $input if (defined($input)); close $wtr; my $real_output = do { local $/; <$rdr>; }; my $real_stderr = do { local $/; <$err>; }; waitpid($pid, 0); my $real_status = $? >> 8; my $r = 0; sub not_ok { print "not ok - $desc\n" if (!$r); $r = 1; $_[0] =~ s/^/# /mg; print $_[0]; } if (defined($output) && $real_output ne $output) { not_ok("wrong output:\n$real_output"); } for my $rx (@output_rx) { if ($real_output !~ $rx) { not_ok("output doesn't match /$rx/:\n$real_output\n"); } } if (defined($stderr) && $real_stderr ne $stderr) { not_ok("wrong stderr:\n$real_stderr"); } for my $rx (@stderr_rx) { if ($real_stderr !~ $rx) { not_ok("stderr doesn't match /$rx/:\n$real_stderr\n"); } } if (!defined($stderr) && !@stderr_rx && !defined($status) && !defined($status_not) && $real_stderr) { not_ok("output to stderr:\n$real_stderr\n"); } if (defined($status) && $real_status != $status) { not_ok("wrong status: $real_status (expected $status)\n"); } if (defined($status_not) && $real_status == $status_not) { not_ok("wrong status: $real_status (expected anything else)\n"); } if (!defined($status) && !defined($status_not) && !defined($stderr) && !@stderr_rx && $real_status != 0) { not_ok("wrong status: $real_status (command failed)\n"); } print "ok - $desc\n" if (!$r); exit $r; xe-0.11/xe.1000066400000000000000000000152411317766720000125610ustar00rootroot00000000000000.Dd November 3, 2017 .Dt XE 1 .Os .Sh NAME .Nm xe .Nd execute a command for every argument .Sh SYNOPSIS .Nm .Op Fl 0FLRnqv .Oo Fl I Ar replace-arg Oc .Op Fl N Ar maxargs .Op Fl j Ar maxjobs .Ar command\ ... .Nm .Op Ar flags\ ... .Fl p Ar pattern Ar command\ ... .Oo Cm \&+ Ar pattern Ar command\ ... Oc Ns ... .Nm .Op Ar flags\ ... .Fl f Ar argfile Ar command\ ... .Nm .Op Ar flags\ ... .Fl s Ar shellscript .Nm .Op Ar flags\ ... .Fl a Ar command\ ... Cm -- Ar args\ ... .Nm .Op Ar flags\ ... .Fl A Ar argsep Ar command\ ... Ar argsep Ar args\ ... .Sh DESCRIPTION The .Nm utility constructs command lines from specified arguments, combining some of the best features of .Xr xargs 1 and .Xr apply 1 . .Pp .Nm means .Dq execute for every ... . .Pp .Nm supports different methods to specify arguments to commands: .Bl -tag -width Ds .It Ar command\ ... By default, arguments - separated by newlines - are read from the standard input. The resulting command is constructed from the command line parameters, replacing .Ar replace-arg with the read argument, and is executed with .Xr execvp 3 . .Pp In this mode, no shell is involved and .Ar replace-arg must appear as a word on its own, i.e. .Sq foo {} bar will work, but .Sq foo{} bar will not, where {} is the default value for .Ar replace-arg . .Pp If no argument is specified, the default is .Sq Ic printf %s\en . .It Fl f Ar argfile Read arguments from .Ar argfile , instead of the standard input. .Pp This does not close the standard input for execution, it is passed to the forked process. .It Fl s Ar shellscript In this mode, the single parameter .Ar shellscript is executed using .Ic sh -c . In the script, the specified arguments can be accessed using $1, $2, ... .Pp For example: .Dl echo \(aqa\enb\(aq | xe -N2 \-s \(aqecho $2 $1\(aq .It Fl a Ar command\ ... Cm -- Ar args\ ... In this mode, everything after .Cm -- is passed as .Ar args to .Ar command . .It Fl A Ar argsep Ar command\ ... Ar argsep Ar args\ ... Same as .Fl a , but the custom argument separator .Ar argsep is used to distinguish between .Ar command and its .Ar args . .El .Pp The options are as follows: .Bl -tag -width Ds .It Fl 0 Input filenames are separated by NUL bytes (instead of newlines, which is the default) .It Fl F Fatal: stop and exit when a command execution fails. .It Fl L Run the resulting commands with line-buffered output; lines from two jobs will not interleave. When used twice, or with .Fl vv , also prefix each line with the number of the job (see .Sx ENVIRONMENT ) in such a manner that the output can be piped to .Sq Li sort -snk1 to group it. .It Fl R Return with status 122 when no arguments have been specified (instead of 0, the default). .Nm never executes a command when no arguments are specified. .It Fl n Dry run: don't run the resulting commands, just print them. .It Fl q Quiet mode: redirect standard output and standard error of commands to /dev/null. .It Fl v Verbose: print commands to standard error before running them. When used twice, also print job id and exit status for each command. .It Fl p Enable .Xr make 1 Ns \&- Ns style percent rules. The first argument of .Ar command\ ... is regarded as a pattern, see .Sx PERCENT RULES below. Patterns without a slash (or .Sq Li \&*\&* ) are matched against the basenames only. .Pp Multiple runs of patterns and commands are separated by .Sq Li \&+ . Only the first matching percent rule is executed; in case no pattern matches, no command is run. .It Fl I Ar replace-arg Replace first occurrence of .Ar replace-arg (default: .Cm {} ) in the resulting command with the argument(s). Pass an empty .Ar replace-arg to disable the replace function. Contrary to .Xr xargs 1 this will expand into multiple arguments when needed. .It Fl N Ar maxargs Pass up to .Ar maxargs arguments to each command (default: 1). .br Using .Fl N0 will pass as many arguments as possible. .It Fl j Ar maxjobs Run up to .Ar maxjobs processes concurrently. Using .Fl j0 will run as many processes as there are CPU cores running. If .Ar maxjobs ends with an .Sq Ic x , it is regarded as a multiplier of the number of running CPU cores (rounded down, but using at least one core). .El .Sh PERCENT RULES The percent rules of .Nm are similar to the globs of .Xr sh 1 or .Xr fnmatch 3 : .Sq Li \&? matches a single character that is not .Sq Li \&/ . .Sq Li \&/ matches one or multiple .Sq Li \&/ in the string. .Sq Li \&* matches zero or more characters, but never .Sq Li \&/ . .Sq Li \&*\&* matches zero or more characters, including .Sq Li \&/ . Note that all of these also match leading dots in file names. .Pp .Sq Li \&{ Ns Va a Ns \&, Ns Va b Ns \&, Ns Va c Ns \&} matches either .Va a , b or .Va c . .Sq Li \&[ Ns Va abc Ns \&] matches one of the characters .Va abc (but never .Sq Li \&/ ) . .Sq Li \&[ Ns \&! Ns Va abc Ns \&] matches all characters but .Va abc . Alternatively, .Sq Li \&[ Ns \&^ Ns Va abc Ns \&] can be used too. .Sq Li \&[ Ns Va a Ns \&- Ns Va c Ns \&] matches any character in the range between .Va a and .Va c inclusive. In character ranges, characters can be escaped using a backslash. .Pp In the pattern, a single occurrence of .Sq Li \&% matches one or more characters, and replaces the first occurrence of .Sq Li \&% with the matched string in the remaining arguments, which are then used as the command to be executed. .Sh ENVIRONMENT The environment variable .Ev ITER is passed to the child process and incremented on each command execution. .Sh EXIT STATUS .Nm follows the convention of GNU and OpenBSD xargs: .Bl -tag -compact -width Ds .It 0 on success .It 123 if any invocation of the command exited with status 1 to 125. .It 124 if the command exited with status 255 .It 125 if the command was killed by a signal .It 126 if the command cannot be run .It 127 if the command was not found .It 1 if some other error occurred .El .Pp Additionally, 122 is returned when .Fl R was passed and the command was never executed. .Sh EXAMPLES Compress all .c files in the current directory, using all CPU cores: .Dl xe -a -j0 gzip -- *.c Remove all empty files, using .Xr lr 1 : .Dl lr -U -t 'size == 0' | xe -N0 rm Convert .mp3 to .ogg, using all CPU cores: .Dl xe -a -j0 -s 'ffmpeg -i \&"${1}\&" \&"${1%.mp3}.ogg\&"' -- *.mp3 Same, using percent rules: .Dl xe -a -j0 -p %.mp3 ffmpeg -i %.mp3 %.ogg -- *.mp3 Similar, but hiding output of ffmpeg, instead showing spawned jobs: .Dl xe -ap -j0 -vvq '%.{m4a,ogg,opus}' ffmpeg -y -i {} out/%.mp3 -- * .Sh SEE ALSO .Xr apply 1 , .Xr parallel 1 , .Xr xapply 1 , .Xr xargs 1 .Sh AUTHORS .An Leah Neukirchen Aq Mt leah@vuxu.org .Sh LICENSE .Nm is in the public domain. .Pp To the extent possible under law, the creator of this work has waived all copyright and related or neighboring rights to this work. .Pp .Lk http://creativecommons.org/publicdomain/zero/1.0/ xe-0.11/xe.c000066400000000000000000000354341317766720000126510ustar00rootroot00000000000000/* * xe - simple xargs and apply replacement * * To the extent possible under law, Leah Neukirchen * has waived all copyright and related or neighboring rights to this work. * http://creativecommons.org/publicdomain/zero/1.0/ */ #include #include #include #include #include #include #include #include #include #include #include static char delim = '\n'; static char default_replace[] = "{}"; static char *replace = default_replace; static char *argsep; static char *fflag; static char *sflag; static int maxatonce = 1; static int maxjobs = 1; static int runjobs = 0; static int failed = 0; static int Aflag, Fflag, Lflag, Rflag, aflag, nflag, pflag, qflag, vflag; static long iterations = 0; static FILE *traceout; static FILE *input; static size_t argmax; static char *buf; static size_t buflen; static size_t bufcap; static char **args; static size_t argslen; static size_t argscap; static size_t argsresv; struct child { pid_t pid; // 0 if empty slot long iter; }; static struct child *children; static char **inargs; static char *line = 0; static size_t linelen = 0; static char * getarg() { if (aflag || Aflag) { if (inargs && *inargs) return *inargs++; else return 0; } int read = getdelim(&line, &linelen, delim, input); if (read == -1) { if (feof(input)) return 0; else exit(1); } if (line[read-1] == delim) // strip delimiter line[read-1] = 0; return line; } static int mywait() { int status; pid_t pid; pid = wait(&status); if (pid < 0) return !(errno == ECHILD); int i; for (i = 0; i < maxjobs; i++) if (children[i].pid == pid) goto my_child; return 1; my_child: if (WIFEXITED(status)) { if (WEXITSTATUS(status) >= 1 && WEXITSTATUS(status) <= 125) { if (Fflag) { fprintf(stderr, "xe: job %ld [pid %ld] exited with status %d\n", children[i].iter, (long)pid, WEXITSTATUS(status)); exit(123); } failed = 1; } else if (WEXITSTATUS(status) == 255) { fprintf(stderr, "xe: job %ld [pid %ld] exited with status 255\n", children[i].iter, (long)pid); exit(124); } else if (WEXITSTATUS(status) > 125) { exit(WEXITSTATUS(status)); } } else if (WIFSIGNALED(status)) { fprintf(stderr, "xe: job %ld [pid %ld] terminated by signal %d\n", children[i].iter, (long)pid, WTERMSIG(status)); exit(125); } if (vflag > 1) { fprintf(traceout, "%04ld< status %d\n", children[i].iter, WEXITSTATUS(status)); fflush(traceout); } children[i].pid = 0; runjobs--; return 1; } static void shquote(const char *s) { if (*s && !strpbrk(s, "\001\002\003\004\005\006\007\010" "\011\012\013\014\015\016\017\020" "\021\022\023\024\025\026\027\030" "\031\032\033\034\035\036\037\040" "`^#*[]=|\\?${}()'\"<>&;\177")) { fprintf(traceout, "%s", s); return; } fprintf(traceout, "\'"); for (; *s; s++) if (*s == '\'') fprintf(traceout, "'\\''"); else fprintf(traceout, "%c", *s); fprintf(traceout, "\'"); } static int trace() { size_t i; for (i = 0; i < argslen; i++) { if (i > 0) fprintf(traceout, " "); shquote(args[i]); } fprintf(traceout, "\n"); fflush(traceout); return 0; } static void scanargs() { char *s = buf; size_t i; for (i = 0; i < argslen; i++) { args[i] = s; s += strlen(s) + 1; } args[i] = 0; } static int pusharg(const char *a) { size_t l = strlen(a) + 1; // including nul if (buflen >= argmax - l - argsresv || argslen + 1 >= argscap) return 0; if (buflen + l > bufcap) { while (buflen + l > bufcap) bufcap *= 2; buf = realloc(buf, bufcap); if (!args) exit(1); } memcpy(buf + buflen, a, l); buflen += l; argslen++; return 1; } static int run() { pid_t pid, lpid; while (runjobs >= maxjobs) mywait(); runjobs++; iterations++; scanargs(); if (!args[0]) { fprintf(stderr, "xe: no command\n"); exit(126); } if (nflag) { trace(); runjobs--; return 0; } int pipefd[2]; if (Lflag) { if (pipe(pipefd) < 0) exit(126); } pid = fork(); if (pid == 0) { // in child char iter[32]; snprintf(iter, sizeof iter, "%ld", iterations); setenv("ITER", iter, 1); // redirect stdin to /dev/null when we read arguments from it // just close it if this fails if (input == stdin) { int fd = open("/dev/null", O_RDONLY); if (fd >= 0) { if (dup2(fd, 0) != 0) close(0); close(fd); } else { close(0); } } if (Lflag) { if (dup2(pipefd[1], 1) < 0) exit(126); close(pipefd[0]); close(pipefd[1]); } if (qflag) { int fd = open("/dev/null", O_RDONLY); if (fd >= 0) { dup2(fd, 2); dup2(fd, 1); close(fd); } } execvp(args[0], args); int status = (errno == ENOENT ? 127 : 126); fprintf(stderr, "xe: %s: %s\n", args[0], strerror(errno)); exit(status); } else if (pid < 0) { // fork failed fprintf(stderr, "xe: %s: %s\n", args[0], strerror(errno)); exit(126); } if (Lflag) { long iter = iterations; lpid = fork(); if (lpid == 0) { // in line-logging child char *line = 0; size_t linelen = 0; close(0); close(pipefd[1]); FILE *input = fdopen(pipefd[0], "r"); if (!input) exit(126); setvbuf(stdout, 0, _IOLBF, 0); while (1) { int rd = getdelim(&line, &linelen, '\n', input); if (rd == -1) exit(0); if (vflag > 1 || Lflag > 1) printf("%04ld= ", iter); fwrite(line, 1, rd, stdout); }; } if (lpid < 0) exit(126); close(pipefd[0]); close(pipefd[1]); } if (vflag) { if (vflag > 1) fprintf(traceout, "%04ld> ", iterations); trace(); } int i; for (i = 0; i < maxjobs; i++) if (!children[i].pid) { children[i].pid = pid; children[i].iter = iterations; break; } return 0; } void toolong() { fprintf(stderr, "xe: fixed argument list too long\n"); exit(1); } int parse_jobs(char *s) { char *e; int n; if (*s && s[strlen(s) - 1] == 'x') { #ifdef _SC_NPROCESSORS_ONLN n = (int)sysconf(_SC_NPROCESSORS_ONLN); #else n = 1; #endif double d = 0.0; errno = 0; d = strtod(s, &e); if (errno != 0 || *e != 'x' || d <= 0) { fprintf(stderr, "xe: can't parse multiplier '%s'.\n", s); exit(1); } n = (int)(d * n); if (n < 1) n = 1; } else if (strcmp(s, "j") == 0) { n = -1; } else { errno = 0; n = strtol(s, &e, 10); if (errno != 0 || *e) { fprintf(stderr, "xe: can't parse number '%s'.\n", s); exit(1); } } #ifdef _SC_NPROCESSORS_ONLN if (n <= 0) n = (int)sysconf(_SC_NPROCESSORS_ONLN); #endif if (n <= 0) n = 1; return n; } // perc is a variant of fnmatch, implemented here using left derivatives: // // differences to fnmatch: // ** matches any substring // % matches any nonempty substring (longest first), is kept for substitution // [!...] and [^...] are supported // [] matches nothing // [...] never matches / // {a,b,c} alternation // {} matches empty string // no special prefix . handling: *le matches .profile // no collations in [] size_t perc_len; char *perc_str; char * perc(char *pat, char *str, int lvl) { char *s, *e; int neg, matched; size_t l; switch (*pat) { case '\0': if (lvl > 0) { fprintf(stderr, "xe: unmatched { in glob\n"); exit(1); } return *str ? 0 : str; case '?': if (!*str || *str == '/') return 0; return perc(pat+1, str+1, lvl); case '/': if (*str != '/') return 0; while (*pat == '/') pat++; while (*str == '/') str++; return perc(pat, str, lvl); case '*': pat++; if (*pat == '*') { // any substring while (*pat == '*') pat++; l = strlen(str) + 1; } else { // any substring but no more than the next / if ((e = strchr(str, '/'))) l = e - str + 1; else l = strlen(str) + 1; } while (l--) if ((s = perc(pat, str+l, lvl))) return s; return 0; case '%': // any nonempty substring for (l = strlen(str); l >= 1; l--) if ((s = perc(pat+1, str+l, lvl))) { perc_str = str; perc_len = l; return s; } return 0; case '[': if (!*str || *str == '/') return 0; pat++; neg = 0; if (*pat == '^' || *pat == '!') { pat++; neg = 1; } for (matched = 0; *pat && *pat != ']'; pat++) { if (*pat == '\\') pat++; if (pat[1] == '-' && pat[2] != ']') { if (pat[0] <= *str && *str <= pat[2]) matched = 1; pat += 2; } else if (*str == *pat) { matched = 1; } } if (*pat != ']') { fprintf(stderr, "xe: unmatched [ in glob\n"); exit(1); } return (neg ^ matched) ? perc(pat+1, str+1, lvl) : 0; case '{': e = 0; while (*pat++ != '}') { if (!e) e = perc(pat, str, lvl+1); // skip to next , or } for (l = 0; *pat && !(l == 0 && (*pat == ',' || *pat == '}')); pat++) if (*pat == '{') l++; else if (*pat == '}') l--; else if (*pat == '[') while (*pat && *pat != ']') { if (*pat == '\\' && pat[1]) pat++; pat++; } } return e ? perc(pat, e, lvl) : 0; case ',': case '}': if (lvl > 0) return str; /* FALL THROUGH */ default: return (*pat == *str) ? perc(pat+1, str+1, lvl) : 0; } } int perc_match(char *pat, char *arg) { if (!strchr(pat, '/') && !strstr(pat, "**")) { char *d = strrchr(arg, '/'); if (d) arg = d + 1; } perc_len = 0; perc_str = 0; return perc(pat, arg, 0) != 0; } char * perc_subst(char *pat, char *base, char *arg) { static char buf[2048]; size_t l; char *dir = base; if (strcmp(arg, replace) == 0) return base; if (!strchr(pat, '/') && !strstr(pat, "**")) { char *d = strrchr(base, '/'); if (d) base = d + 1; } char *t = strchr(arg, '%'); if (!t) return arg; if (perc_len) l = snprintf(buf, sizeof buf, "%.*s%.*s%.*s%.*s", (int)(base - dir), dir, (int)(t - arg), arg, (int)perc_len, perc_str, (int)(arg + strlen(arg) - t), t+1); else l = snprintf(buf, sizeof buf, "%.*s%.*s%s%.*s", (int)(base - dir), dir, (int)(t - arg), arg, base, (int)(arg + strlen(arg) - t), t+1); if (l >= sizeof buf) { fprintf(stderr, "xe: result of percent substitution too long\n"); exit(1); } return buf; } static void trace_to_stderr() { FILE *linestderr = fdopen(2, "w"); if (linestderr) { setvbuf(linestderr, 0, _IOLBF, 0); traceout = linestderr; } else { traceout = stderr; } } int main(int argc, char *argv[], char *envp[]) { int c, i, j, cmdend; char *arg; bufcap = 4096; buf = malloc(bufcap); argscap = 8192; args = malloc(sizeof args[0] * argscap); if (!buf || !args) exit(1); argmax = sysconf(_SC_ARG_MAX); while (*envp) // subtract size of environment argmax -= strlen(*envp++) + 1 + sizeof *envp; argmax -= 4 * 1024; // subtract 4k for safety if (argmax > 128 * 1024) // upper bound argmax = 128 * 1024; if (argmax <= 0) // lower bound argmax = _POSIX_ARG_MAX; traceout = stdout; while ((c = getopt(argc, argv, "+0A:FI:LN:Raf:j:npqs:v")) != -1) switch (c) { case '0': delim = '\0'; break; case 'A': argsep = optarg; Aflag++; break; case 'I': replace = optarg; break; case 'L': Lflag++; break; case 'N': maxatonce = atoi(optarg); break; case 'F': Fflag++; break; case 'R': Rflag++; break; case 'a': aflag++; break; case 'f': fflag = optarg; break; case 'j': maxjobs = parse_jobs(optarg); break; case 'n': nflag++; break; case 'p': pflag++; break; case 'q': qflag++; break; case 's': sflag = optarg; break; case 'v': vflag++; break; default: fprintf(stderr, "Usage: %s [-0FLRnqv] [-I arg] [-N maxargs] [-j maxjobs] COMMAND...\n" " | -p PATTERN COMMAND... [+ PATTERN COMMAND...]...\n" " | -f ARGFILE COMMAND...\n" " | -s SHELLSCRIPT\n" " | -a COMMAND... -- ARGS...\n" " | -A ARGSEP COMMAND... ARGSEP ARGS...\n", argv[0]); exit(1); } children = calloc(sizeof (struct child), maxjobs); if (!children) exit(1); if (!Lflag || vflag <= 1) trace_to_stderr(); if (aflag || Aflag) { input = 0; } else if (!fflag || strcmp("-", fflag) == 0) { input = stdin; } else { input = fopen(fflag, "rb"); if (!input) { fprintf(stderr, "xe: opening %s: %s\n", fflag, strerror(errno)); exit(1); } fcntl(fileno(input), F_SETFD, FD_CLOEXEC); } cmdend = argc; if (aflag) { // find first -- in argv for (i = 1; i < argc; i++) if (strcmp(argv[i], "--") == 0) { inargs = argv + i+1; cmdend = i; break; } if (sflag) { // e.g. on xe -s 'echo $1' -a 1 2 3 cmdend = optind; inargs = argv + cmdend; } } else if (Aflag) { // find first argsep after optind for (i = optind; i < argc; i++) { if (strcmp(argv[i], argsep) == 0) { inargs = argv + i+1; cmdend = i; break; } } if (i >= argc) { fprintf(stderr, "xe: '-A %s' used but no separator " "'%s' found in command line.\n", argsep, argsep); exit(1); } } int keeparg = 0; if (pflag) { if (maxatonce != 1) { fprintf(stderr, "xe: -p cannot be used together with -N.\n"); exit(1); } while (1) { buflen = 0; argslen = 0; while (runjobs >= maxjobs) mywait(); if (!(arg = getarg())) break; int n; for (n = optind, i = n + 1; n < cmdend; n = i + 1) { char *pat = argv[n]; int matched = perc_match(pat, arg); if (sflag && matched) { pusharg("/bin/sh"); pusharg("-c"); pusharg(sflag); pusharg("/bin/sh"); } for (i = n + 1; i < cmdend; i++) { if (argv[i][0] == '+' && argv[i][1] == '\0') break; if (matched && !pusharg(perc_subst(pat, arg, argv[i]))) toolong(); } if (matched) { run(); break; } } } } else { while (1) { // check if there is an arg from a previous iteration if (!keeparg) { while (runjobs >= maxjobs) mywait(); arg = getarg(); if (!arg) break; } keeparg = 0; buflen = 0; argslen = 0; if (sflag) { pusharg("/bin/sh"); pusharg("-c"); pusharg(sflag); pusharg("/bin/sh"); } else if (optind >= cmdend) { pusharg("printf"); pusharg("%s\\n"); } for (i = optind; i < cmdend; i++) { if (*replace && strcmp(argv[i], replace) == 0) break; if (!pusharg(argv[i])) toolong(); } if (!pusharg(arg)) toolong(); i++; // reserve space for final arguments for (argsresv = 0, j = i; j < cmdend; j++) argsresv += 1 + strlen(argv[j]); // fill with up to maxatonce arguments for (j = 0; maxatonce < 1 || j < maxatonce-1; j++) { arg = getarg(); if (!arg) break; if (!pusharg(arg)) { // we are out of space, // deal with this arg in the next iter keeparg = 1; break; } } for (argsresv = 0, j = i; j < cmdend; j++) if (!pusharg(argv[j])) toolong(); run(); } } while (mywait()) ; free(buf); free(args); free(line); if (Rflag && iterations == 0) return 122; if (failed) return 123; return 0; }