pax_global_header00006660000000000000000000000064123700510120014501gustar00rootroot0000000000000052 comment=2e5e768af3000f48363de65b107e17cfaa6c593c beanstalkd-1.10/000077500000000000000000000000001237005101200135325ustar00rootroot00000000000000beanstalkd-1.10/.gitignore000066400000000000000000000000361237005101200155210ustar00rootroot00000000000000*.o /vers.c /beanstalkd /News beanstalkd-1.10/LICENSE000066400000000000000000000020601237005101200145350ustar00rootroot00000000000000Copyright (c) 2007-2011 The Beanstalkd Authors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. beanstalkd-1.10/Makefile000066400000000000000000000025041237005101200151730ustar00rootroot00000000000000DESTDIR= PREFIX=/usr/local BINDIR=$(DESTDIR)$(PREFIX)/bin CFLAGS=-Wall -Werror\ -Wformat=2\ -g\ LDFLAGS= OS=$(shell uname|tr A-Z a-z) INSTALL=install VERS=$(shell ./vers.sh) TARG=beanstalkd MOFILE=main.o OFILES=\ $(OS).o\ conn.o\ file.o\ heap.o\ job.o\ ms.o\ net.o\ primes.o\ prot.o\ sd-daemon.o\ serv.o\ time.o\ tube.o\ util.o\ vers.o\ walg.o\ TOFILES=\ testheap.o\ testjobs.o\ testserv.o\ testutil.o\ HFILES=\ dat.h\ sd-daemon.h\ CLEANFILES=\ vers.c\ .PHONY: all all: $(TARG) $(TARG): $(OFILES) $(MOFILE) $(LINK.o) -o $@ $^ $(LDLIBS) .PHONY: install install: $(BINDIR) $(BINDIR)/$(TARG) $(BINDIR): $(INSTALL) -d $@ $(BINDIR)/%: % $(INSTALL) $< $@ CLEANFILES:=$(CLEANFILES) $(TARG) $(OFILES) $(MOFILE): $(HFILES) .PHONY: clean clean: rm -f *.o $(CLEANFILES) .PHONY: check check: ct/_ctcheck ct/_ctcheck .PHONY: bench bench: ct/_ctcheck ct/_ctcheck -b ct/_ctcheck: ct/_ctcheck.o ct/ct.o $(OFILES) $(TOFILES) ct/_ctcheck.c: $(TOFILES) ct/gen ct/gen $(TOFILES) >$@.part mv $@.part $@ ct/ct.o ct/_ctcheck.o: ct/ct.h ct/internal.h $(TOFILES): $(HFILES) ct/ct.h CLEANFILES:=$(CLEANFILES) ct/_* ct/*.o ifneq ($(shell ./verc.sh),$(shell cat vers.c 2>/dev/null)) .PHONY: vers.c endif vers.c: ./verc.sh >vers.c doc/beanstalkd.1 doc/beanstalkd.1.html: doc/beanstalkd.ronn ronn $< freebsd.o: darwin.c beanstalkd-1.10/News000066400000000000000000000015441237005101200143750ustar00rootroot00000000000000This is beanstalkd version 1.10, containing bug fixes. As always, there will be no incompatible protocol changes until version 2.0. A client written for version 1.10 will work unmodified with any later 1.x release of beanstalkd. News ---- - fix crash on suspend or other EINTR (#220) - document touch command's TTR reset (#188) - add some basic benchmark tests - add DESTDIR support to Makefile Full list of changes (includes authorship information): Our Urls -------- Download the 1.10 tarball directly: Learn all about beanstalk: Talk about beanstalk development or use at: Bugs ---- Please report any bugs to: beanstalkd-1.10/README000066400000000000000000000017121237005101200144130ustar00rootroot00000000000000This is beanstalkd, a fast, general-purpose work queue. See http://kr.github.io/beanstalkd/ for general info. QUICK START $ make $ ./beanstalkd also try, $ ./beanstalkd -h $ ./beanstalkd -VVV $ make CFLAGS=-O2 $ make CC=clang $ make check $ make install $ make install PREFIX=/usr Requires Linux (2.6.17 or later), Mac OS X, or FreeBSD. See doc/protocol.txt for details of the network protocol. Uses ronn to generate the manual. See http://github.com/rtomayko/ronn. SUBDIRECTORIES adm files useful for system administrators ct testing tool; see https://github.com/kr/ct doc documentation pkg miscelaneous files for packagers TESTS Unit tests are in test*.c. See https://github.com/kr/ct for information on how to write them. Copyright © 2007-2013 the authors of beanstalkd. Copyright in contributions to beanstalkd is retained by the original copyright holder of each contribution. See file LICENSE for terms of use. beanstalkd-1.10/adm/000077500000000000000000000000001237005101200142735ustar00rootroot00000000000000beanstalkd-1.10/adm/Readme000066400000000000000000000030101237005101200154050ustar00rootroot00000000000000The usual way to run beanstalkd is to type its name in a Unix shell prompt, like this: $ beanstalkd This will start up the process and give you control over it. You can control its output (by default output is printed to the screen; you can arrange to have output go into file b.log by typing ">b.log" at the end of the command line), pause and restart the process (by pressing Control-Z and typing "fg"), and kill it (by pressing Control-C). This is most convenient while writing programs that use beanstalkd (or when working on beanstalkd itself), since you might want to start and stop it many times and regularly inspect its output. If you want beanstalkd to start when your operating system boots, the mechanism varies. Traditionally, you must add a command line to the shell script in /etc/rc (which is read by init when the system boots), using the "&" notation to run beanstalkd in the background. This would suffice for most situations, but it isn't always possible. These days, many popular operating systems have a replacement init program with its own configuration language. Example configuration files for several of these are included in subdirectories here, but the most common is probably "System V init", which reads /etc/inittab for lines describing commands to run at various times. If this file exists, you can add a line something like bean:345:respawn:su nobody -c 'exec /usr/bin/beanstalkd' and type "telinit q" to tell init to reread its configuration. Type "man 5 inittab" for details of this notation. beanstalkd-1.10/adm/launchd/000077500000000000000000000000001237005101200157115ustar00rootroot00000000000000beanstalkd-1.10/adm/launchd/beanstalkd.plist000066400000000000000000000007621237005101200211030ustar00rootroot00000000000000 Label beanstalkd UserName nobody ProgramArguments /usr/local/bin/beanstalkd KeepAlive beanstalkd-1.10/adm/systemd/000077500000000000000000000000001237005101200157635ustar00rootroot00000000000000beanstalkd-1.10/adm/systemd/beanstalkd.service000066400000000000000000000001601237005101200214520ustar00rootroot00000000000000[Unit] Description=Beanstalkd is a simple, fast work queue [Service] User=nobody ExecStart=/usr/bin/beanstalkd beanstalkd-1.10/adm/systemd/beanstalkd.socket000066400000000000000000000002041237005101200213010ustar00rootroot00000000000000[Unit] Description=Socket for beanstalkd, a simple, fast work queue [Socket] ListenStream=11300 [Install] WantedBy=sockets.target beanstalkd-1.10/adm/upstart/000077500000000000000000000000001237005101200157755ustar00rootroot00000000000000beanstalkd-1.10/adm/upstart/beanstalkd.conf000066400000000000000000000002351237005101200207540ustar00rootroot00000000000000description "simple, fast work queue" start on filesystem stop on runlevel [!2345] respawn respawn limit 5 2 exec su nobody -c 'exec /usr/bin/beanstalkd' beanstalkd-1.10/beanstalkd.spec000066400000000000000000000047321237005101200165240ustar00rootroot00000000000000%define beanstalkd_user beanstalkd %define beanstalkd_group %{beanstalkd_user} %define beanstalkd_home %{_localstatedir}/lib/beanstalkd %define beanstalkd_logdir %{_localstatedir}/log/beanstalkd Name: beanstalkd Version: 1.10 Release: 0%{?dist} Summary: A simple, fast workqueue service Group: System Environment/Daemons License: GPLv3+ URL: http://xph.us/software/%{name}/ Source0: http://xph.us/dist/%{name}/rel/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires(pre): %{_sbindir}/useradd Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig, /sbin/service Requires(postun): /sbin/service %description Beanstalk is a simple, fast workqueue service. Its interface is generic, but was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. %prep %setup -q if [ ! -e configure ]; then sh buildconf.sh fi %build %configure --disable-rpath --docdir=%{_defaultdocdir}/%{name}-%{version} make %{?_smp_mflags} %install rm -rf $RPM_BUILD_ROOT make install DESTDIR=$RPM_BUILD_ROOT %{__install} -p -D -m 0644 doc/%{name}.1 %{buildroot}%{_mandir}/man1/%{name}.1 %{__install} -p -D -m 0755 scripts/%{name}.init %{buildroot}%{_initrddir}/%{name} %{__install} -p -D -m 0644 scripts/%{name}.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} %clean rm -rf $RPM_BUILD_ROOT %pre %{_sbindir}/useradd -c "beanstalkd user" -s /bin/false -r -m -d %{beanstalkd_home} %{beanstalkd_user} 2>/dev/null || : %post /sbin/chkconfig --add %{name} %preun if [ $1 = 0 ]; then /sbin/service %{name} stop >/dev/null 2>&1 /sbin/chkconfig --del %{name} fi %postun if [ $1 -ge 1 ]; then /sbin/service %{name} condrestart > /dev/null 2>&1 || : fi %files %defattr(-,root,root,-) %doc %{_defaultdocdir}/%{name}-%{version}/protocol.txt %doc README COPYING doc/protocol.txt %{_initrddir}/%{name} %{_bindir}/%{name} %{_mandir}/man1/%{name}.1.gz %config(noreplace) %{_sysconfdir}/sysconfig/%{name} %changelog * Thu Oct 1 2009 Keith Rarick - 1.4-0 - Convert this file to an autoconf template - Tweak the summary and description * Sun Jan 4 2009 Ask Bjørn Hansen - 1.2-0 - 1.2-tobe - Use man page and .init/sysconfig scripts from .tar.gz * Sat Nov 22 2008 Jeremy Hinegardner - 1.1-1 - initial spec creation beanstalkd-1.10/conn.c000066400000000000000000000102201237005101200146260ustar00rootroot00000000000000#include #include #include #include #include #include #include #include "dat.h" #define SAFETY_MARGIN (1000000000) /* 1 second */ static int cur_conn_ct = 0, cur_worker_ct = 0, cur_producer_ct = 0; static uint tot_conn_ct = 0; int verbose = 0; static void on_watch(ms a, tube t, size_t i) { tube_iref(t); t->watching_ct++; } static void on_ignore(ms a, tube t, size_t i) { t->watching_ct--; tube_dref(t); } Conn * make_conn(int fd, char start_state, tube use, tube watch) { job j; Conn *c; c = new(Conn); if (!c) return twarn("OOM"), NULL; ms_init(&c->watch, (ms_event_fn) on_watch, (ms_event_fn) on_ignore); if (!ms_append(&c->watch, watch)) { free(c); return twarn("OOM"), NULL; } TUBE_ASSIGN(c->use, use); use->using_ct++; c->sock.fd = fd; c->state = start_state; c->pending_timeout = -1; c->tickpos = -1; j = &c->reserved_jobs; j->prev = j->next = j; /* stats */ cur_conn_ct++; tot_conn_ct++; return c; } void connsetproducer(Conn *c) { if (c->type & CONN_TYPE_PRODUCER) return; c->type |= CONN_TYPE_PRODUCER; cur_producer_ct++; /* stats */ } void connsetworker(Conn *c) { if (c->type & CONN_TYPE_WORKER) return; c->type |= CONN_TYPE_WORKER; cur_worker_ct++; /* stats */ } int count_cur_conns() { return cur_conn_ct; } uint count_tot_conns() { return tot_conn_ct; } int count_cur_producers() { return cur_producer_ct; } int count_cur_workers() { return cur_worker_ct; } static int has_reserved_job(Conn *c) { return job_list_any_p(&c->reserved_jobs); } static int64 conntickat(Conn *c) { int margin = 0, should_timeout = 0; int64 t = INT64_MAX; if (conn_waiting(c)) { margin = SAFETY_MARGIN; } if (has_reserved_job(c)) { t = connsoonestjob(c)->r.deadline_at - nanoseconds() - margin; should_timeout = 1; } if (c->pending_timeout >= 0) { t = min(t, ((int64)c->pending_timeout) * 1000000000); should_timeout = 1; } if (should_timeout) { return nanoseconds() + t; } return 0; } void connwant(Conn *c, int rw) { c->rw = rw; connsched(c); } void connsched(Conn *c) { if (c->tickpos > -1) { heapremove(&c->srv->conns, c->tickpos); } c->tickat = conntickat(c); if (c->tickat) { heapinsert(&c->srv->conns, c); } } /* return the reserved job with the earliest deadline, * or NULL if there's no reserved job */ job connsoonestjob(Conn *c) { job j = NULL; job soonest = c->soonest_job; if (soonest == NULL) { for (j = c->reserved_jobs.next; j != &c->reserved_jobs; j = j->next) { if (j->r.deadline_at <= (soonest ? : j)->r.deadline_at) soonest = j; } } c->soonest_job = soonest; return soonest; } /* return true if c has a reserved job with less than one second until its * deadline */ int conndeadlinesoon(Conn *c) { int64 t = nanoseconds(); job j = connsoonestjob(c); return j && t >= j->r.deadline_at - SAFETY_MARGIN; } int conn_ready(Conn *c) { size_t i; for (i = 0; i < c->watch.used; i++) { if (((tube) c->watch.items[i])->ready.len) return 1; } return 0; } int connless(Conn *a, Conn *b) { return a->tickat < b->tickat; } void connrec(Conn *c, int i) { c->tickpos = i; } void connclose(Conn *c) { sockwant(&c->sock, 0); close(c->sock.fd); if (verbose) { printf("close %d\n", c->sock.fd); } job_free(c->in_job); /* was this a peek or stats command? */ if (c->out_job && !c->out_job->r.id) job_free(c->out_job); c->in_job = c->out_job = NULL; c->in_job_read = 0; if (c->type & CONN_TYPE_PRODUCER) cur_producer_ct--; /* stats */ if (c->type & CONN_TYPE_WORKER) cur_worker_ct--; /* stats */ cur_conn_ct--; /* stats */ remove_waiting_conn(c); if (has_reserved_job(c)) enqueue_reserved_jobs(c); ms_clear(&c->watch); c->use->using_ct--; TUBE_ASSIGN(c->use, NULL); if (c->tickpos > -1) { heapremove(&c->srv->conns, c->tickpos); } free(c); } beanstalkd-1.10/ct/000077500000000000000000000000001237005101200141405ustar00rootroot00000000000000beanstalkd-1.10/ct/.gitignore000066400000000000000000000000111237005101200161200ustar00rootroot00000000000000/*.o /_* beanstalkd-1.10/ct/License000066400000000000000000000020471237005101200154500ustar00rootroot00000000000000Copyright © 2010–2013 Keith Rarick Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. beanstalkd-1.10/ct/ct.c000066400000000000000000000242131237005101200147140ustar00rootroot00000000000000// CT - simple-minded unit testing for C #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "internal.h" #include "ct.h" static char *curdir; static int rjobfd = -1, wjobfd = -1; static int64 bstart, bdur; static int btiming; // bool static int64 bbytes; static const int64 Second = 1000 * 1000 * 1000; static const int64 BenchTime = Second; static const int MaxN = 1000 * 1000 * 1000; #ifdef __MACH__ # include static int64 nstime() { return (int64)mach_absolute_time(); } #else static int64 nstime() { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return (int64)(t.tv_sec)*Second + t.tv_nsec; } #endif void ctlogpn(char *p, int n, char *fmt, ...) { va_list arg; printf("%s:%d: ", p, n); va_start(arg, fmt); vprintf(fmt, arg); va_end(arg); putchar('\n'); } void ctfail(void) { fflush(stdout); fflush(stderr); abort(); } char * ctdir(void) { mkdir(curdir, 0700); return curdir; } void ctresettimer(void) { bdur = 0; bstart = nstime(); } void ctstarttimer(void) { if (!btiming) { bstart = nstime(); btiming = 1; } } void ctstoptimer(void) { if (btiming) { bdur += nstime() - bstart; btiming = 0; } } void ctsetbytes(int n) { bbytes = (int64)n; } static void die(int code, int err, char *msg) { putc('\n', stderr); if (msg && *msg) { fputs(msg, stderr); fputs(": ", stderr); } fputs(strerror(err), stderr); putc('\n', stderr); exit(code); } static int tmpfd(void) { FILE *f = tmpfile(); if (!f) { die(1, errno, "tmpfile"); } return fileno(f); } static int failed(int s) { return WIFSIGNALED(s) && (WTERMSIG(s) == SIGABRT); } static void waittest(Test *ts) { Test *t; int pid, stat; pid = wait3(&stat, 0, 0); if (pid == -1) { die(3, errno, "wait"); } killpg(pid, 9); for (t=ts; t->f; t++) { if (t->pid == pid) { t->status = stat; if (!t->status) { putchar('.'); } else if (failed(t->status)) { putchar('F'); } else { putchar('E'); } fflush(stdout); } } } static void start(Test *t) { t->fd = tmpfd(); strcpy(t->dir, TmpDirPat); mktemp(t->dir); t->pid = fork(); if (t->pid < 0) { die(1, errno, "fork"); } else if (!t->pid) { setpgid(0, 0); if (dup2(t->fd, 1) == -1) { die(3, errno, "dup2"); } if (close(t->fd) == -1) { die(3, errno, "fclose"); } if (dup2(1, 2) == -1) { die(3, errno, "dup2"); } curdir = t->dir; t->f(); _exit(0); } setpgid(t->pid, t->pid); } static void runalltest(Test *ts, int limit) { int nrun = 0; Test *t; for (t=ts; t->f; t++) { if (nrun >= limit) { waittest(ts); nrun--; } start(t); nrun++; } for (; nrun; nrun--) { waittest(ts); } } static void copyfd(FILE *out, int in) { ssize_t n; char buf[1024]; // arbitrary size while ((n = read(in, buf, sizeof(buf))) != 0) { if (fwrite(buf, 1, n, out) != (size_t)n) { die(3, errno, "fwrite"); } } } // Removes path and all of its children. // Writes errors to stderr and keeps going. // If path doesn't exist, rmtree returns silently. static void rmtree(char *path) { int r = unlink(path); if (r == 0 || errno == ENOENT) { return; // success } int unlinkerr = errno; DIR *d = opendir(path); if (!d) { if (errno == ENOTDIR) { fprintf(stderr, "ct: unlink: %s\n", strerror(unlinkerr)); } else { perror("ct: opendir"); } fprintf(stderr, "ct: path %s\n", path); return; } struct dirent *ent; while ((ent = readdir(d))) { if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { continue; } int n = strlen(path) + 1 + strlen(ent->d_name); char s[n+1]; sprintf(s, "%s/%s", path, ent->d_name); rmtree(s); } closedir(d); r = rmdir(path); if (r == -1) { perror("ct: rmdir"); fprintf(stderr, "ct: path %s\n", path); } } static void runbenchn(Benchmark *b, int n) { int outfd = tmpfd(); int durfd = tmpfd(); strcpy(b->dir, TmpDirPat); mktemp(b->dir); int pid = fork(); if (pid < 0) { die(1, errno, "fork"); } else if (!pid) { setpgid(0, 0); if (dup2(outfd, 1) == -1) { die(3, errno, "dup2"); } if (close(outfd) == -1) { die(3, errno, "fclose"); } if (dup2(1, 2) == -1) { die(3, errno, "dup2"); } curdir = b->dir; ctstarttimer(); b->f(n); ctstoptimer(); write(durfd, &bdur, sizeof bdur); write(durfd, &bbytes, sizeof bbytes); _exit(0); } setpgid(pid, pid); pid = waitpid(pid, &b->status, 0); if (pid == -1) { die(3, errno, "wait"); } killpg(pid, 9); rmtree(b->dir); if (b->status != 0) { putchar('\n'); lseek(outfd, 0, SEEK_SET); copyfd(stdout, outfd); return; } lseek(durfd, 0, SEEK_SET); int r = read(durfd, &b->dur, sizeof b->dur); if (r != sizeof b->dur) { perror("read"); b->status = 1; } r = read(durfd, &b->bytes, sizeof b->bytes); if (r != sizeof b->bytes) { perror("read"); b->status = 1; } } // rounddown10 rounds a number down to the nearest power of 10. static int rounddown10(int n) { int tens = 0; // tens = floor(log_10(n)) while (n >= 10) { n = n / 10; tens++; } // result = 10**tens int i, result = 1; for (i = 0; i < tens; i++) { result *= 10; } return result; } // roundup rounds n up to a number of the form [1eX, 2eX, 5eX]. static int roundup(int n) { int base = rounddown10(n); if (n == base) return n; if (n <= 2*base) return 2 * base; if (n <= 5*base) return 5 * base; return 10 * base; } static int min(int a, int b) { if (a < b) { return a; } return b; } static int max(int a, int b) { if (a > b) { return a; } return b; } static void runbench(Benchmark *b) { printf("%s\t", b->name); fflush(stdout); int n = 1; runbenchn(b, n); while (b->status == 0 && b->dur < BenchTime && n < MaxN) { int last = n; // Predict iterations/sec. int nsop = b->dur / n; if (nsop == 0) { n = MaxN; } else { n = BenchTime / nsop; } // Run more iterations than we think we'll need for a second (1.5x). // Don't grow too fast in case we had timing errors previously. // Be sure to run at least one more than last time. n = max(min(n+n/2, 100*last), last+1); // Round up to something easy to read. n = roundup(n); runbenchn(b, n); } if (b->status == 0) { printf("%8d\t%10lld ns/op", n, b->dur/n); if (b->bytes > 0) { double mbs = 0; if (b->dur > 0) { int64 sec = b->dur / 1000L / 1000L / 1000L; int64 nsec = b->dur % 1000000000L; double dur = (double)sec + (double)nsec*.0000000001; mbs = ((double)b->bytes * (double)n / 1000000) / dur; } printf("\t%7.2f MB/s", mbs); } putchar('\n'); } else { if (failed(b->status)) { printf("failure"); } else { printf("error"); if (WIFEXITED(b->status)) { printf(" (exit status %d)", WEXITSTATUS(b->status)); } if (WIFSIGNALED(b->status)) { printf(" (signal %d)", WTERMSIG(b->status)); } } putchar('\n'); } } static void runallbench(Benchmark *b) { for (; b->f; b++) { runbench(b); } } static int report(Test *t) { int nfail = 0, nerr = 0; putchar('\n'); for (; t->f; t++) { rmtree(t->dir); if (!t->status) { continue; } printf("\n%s: ", t->name); if (failed(t->status)) { nfail++; printf("failure"); } else { nerr++; printf("error"); if (WIFEXITED(t->status)) { printf(" (exit status %d)", WEXITSTATUS(t->status)); } if (WIFSIGNALED(t->status)) { printf(" (signal %d)", WTERMSIG(t->status)); } } putchar('\n'); lseek(t->fd, 0, SEEK_SET); copyfd(stdout, t->fd); } if (nfail || nerr) { printf("\n%d failures; %d errors.\n", nfail, nerr); } else { printf("\nPASS\n"); } return nfail || nerr; } int readtokens() { int n = 1; char c, *s; if ((s = strstr(getenv("MAKEFLAGS"), " --jobserver-fds="))) { rjobfd = (int)strtol(s+17, &s, 10); // skip " --jobserver-fds=" wjobfd = (int)strtol(s+1, NULL, 10); // skip comma } if (rjobfd >= 0) { fcntl(rjobfd, F_SETFL, fcntl(rjobfd, F_GETFL)|O_NONBLOCK); while (read(rjobfd, &c, 1) > 0) { n++; } } return n; } void writetokens(int n) { char c = '+'; if (wjobfd >= 0) { fcntl(wjobfd, F_SETFL, fcntl(wjobfd, F_GETFL)|O_NONBLOCK); for (; n>1; n--) { write(wjobfd, &c, 1); // ignore error; nothing we can do anyway } } } int main(int argc, char **argv) { int n = readtokens(); runalltest(ctmaintest, n); writetokens(n); int code = report(ctmaintest); if (code != 0) { return code; } if (argc == 2 && strcmp(argv[1], "-b") == 0) { runallbench(ctmainbench); } return 0; } beanstalkd-1.10/ct/ct.h000066400000000000000000000007341237005101200147230ustar00rootroot00000000000000char *ctdir(void); void ctfail(void); void ctresettimer(void); void ctstarttimer(void); void ctstoptimer(void); void ctsetbytes(int); void ctlogpn(char*, int, char*, ...) __attribute__((format(printf, 3, 4))); #define ctlog(...) ctlogpn(__FILE__, __LINE__, __VA_ARGS__) #define assert(x) do if (!(x)) {\ ctlog("%s", "test: " #x);\ ctfail();\ } while (0) #define assertf(x, ...) do if (!(x)) {\ ctlog("%s", "test: " #x);\ ctlog(__VA_ARGS__);\ ctfail();\ } while (0) beanstalkd-1.10/ct/gen000077500000000000000000000013471237005101200146440ustar00rootroot00000000000000#!/bin/sh set -e fixsyms() { if test "`uname -s|tr A-Z a-z`" = darwin then egrep -v [.] | egrep ^_ | sed s/^_// else cat fi } syms() { prefix=$1 shift for f in "$@" do nm $f done | cut -d ' ' -f 3 | fixsyms | egrep ^$prefix } ts=`syms cttest "$@" || true` bs=`syms ctbench "$@" || true` printf '#include \n' printf '#include "internal.h"\n' for t in $ts do printf 'void %s(void);\n' $t done for b in $bs do printf 'void %s(int);\n' $b done printf 'Test ctmaintest[] = {\n' for t in $ts do printf ' {%s, "%s"},\n' $t $t done printf ' {0},\n' printf '};\n' printf 'Benchmark ctmainbench[] = {\n' for b in $bs do printf ' {%s, "%s"},\n' $b $b done printf ' {0},\n' printf '};\n' beanstalkd-1.10/ct/internal.h000066400000000000000000000007361237005101200161330ustar00rootroot00000000000000// include #define TmpDirPat "/tmp/ct.XXXXXX" typedef int64_t int64; typedef struct Test Test; typedef struct Benchmark Benchmark; struct Test { void (*f)(void); char *name; int status; int fd; int pid; char dir[sizeof TmpDirPat]; }; struct Benchmark { void (*f)(int); char *name; int status; int64 dur; int64 bytes; char dir[sizeof TmpDirPat]; }; extern Test ctmaintest[]; extern Benchmark ctmainbench[]; beanstalkd-1.10/darwin.c000066400000000000000000000042231237005101200151630ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include "dat.h" enum { Infinity = 1 << 30 }; static int kq; static char buf0[512]; /* buffer of zeros */ /* Allocate disk space. * Expects fd's offset to be 0; may also reset fd's offset to 0. * Returns 0 on success, and a positive errno otherwise. */ int rawfalloc(int fd, int len) { int i, w; for (i = 0; i < len; i += w) { w = write(fd, buf0, sizeof buf0); if (w == -1) return errno; } lseek(fd, 0, 0); /* do not care if this fails */ return 0; } int sockinit(void) { kq = kqueue(); if (kq == -1) { twarn("kqueue"); return -1; } return 0; } int sockwant(Socket *s, int rw) { int n = 0; struct kevent evs[2] = {}, *ev = evs; struct timespec ts = {}; if (s->added) { ev->ident = s->fd; ev->filter = s->added; ev->flags = EV_DELETE; ev++; n++; } if (rw) { ev->ident = s->fd; switch (rw) { case 'r': ev->filter = EVFILT_READ; break; case 'w': ev->filter = EVFILT_WRITE; break; default: // check only for hangup ev->filter = EVFILT_READ; ev->fflags = NOTE_LOWAT; ev->data = Infinity; } ev->flags = EV_ADD; ev->udata = s; s->added = ev->filter; ev++; n++; } return kevent(kq, evs, n, NULL, 0, &ts); } int socknext(Socket **s, int64 timeout) { int r; struct kevent ev; static struct timespec ts; ts.tv_sec = timeout / 1000000000; ts.tv_nsec = timeout % 1000000000; r = kevent(kq, NULL, 0, &ev, 1, &ts); if (r == -1 && errno != EINTR) { twarn("kevent"); return -1; } if (r > 0) { *s = ev.udata; if (ev.flags & EV_EOF) { return 'h'; } switch (ev.filter) { case EVFILT_READ: return 'r'; case EVFILT_WRITE: return 'w'; } } return 0; } beanstalkd-1.10/dat.h000066400000000000000000000207741237005101200144650ustar00rootroot00000000000000// Requirements: // #include // #include typedef unsigned char uchar; typedef uchar byte; typedef unsigned int uint; typedef int32_t int32; typedef uint32_t uint32; typedef int64_t int64; typedef uint64_t uint64; #define int8_t do_not_use_int8_t #define uint8_t do_not_use_uint8_t #define int32_t do_not_use_int32_t #define uint32_t do_not_use_uint32_t #define int64_t do_not_use_int64_t #define uint64_t do_not_use_uint64_t typedef struct ms *ms; typedef struct job *job; typedef struct tube *tube; typedef struct Conn Conn; typedef struct Heap Heap; typedef struct Jobrec Jobrec; typedef struct File File; typedef struct Socket Socket; typedef struct Server Server; typedef struct Wal Wal; typedef void(*ms_event_fn)(ms a, void *item, size_t i); typedef void(*Handle)(void*, int rw); typedef int(*Less)(void*, void*); typedef void(*Record)(void*, int); typedef int(FAlloc)(int, int); #if _LP64 #define NUM_PRIMES 48 #else #define NUM_PRIMES 19 #endif #define MAX_TUBE_NAME_LEN 201 /* A command can be at most LINE_BUF_SIZE chars, including "\r\n". This value * MUST be enough to hold the longest possible command or reply line, which is * currently "USING a{200}\r\n". */ #define LINE_BUF_SIZE 208 /* CONN_TYPE_* are bit masks */ #define CONN_TYPE_PRODUCER 1 #define CONN_TYPE_WORKER 2 #define CONN_TYPE_WAITING 4 #define min(a,b) ((a)<(b)?(a):(b)) #define URGENT_THRESHOLD 1024 #define JOB_DATA_SIZE_LIMIT_DEFAULT ((1 << 16) - 1) extern const char version[]; extern int verbose; extern struct Server srv; // Replaced by tests to simulate failures. extern FAlloc *falloc; struct stats { uint urgent_ct; uint waiting_ct; uint buried_ct; uint reserved_ct; uint pause_ct; uint64 total_delete_ct; uint64 total_jobs_ct; }; struct Heap { int cap; int len; void **data; Less less; Record rec; }; int heapinsert(Heap *h, void *x); void* heapremove(Heap *h, int k); struct Socket { int fd; Handle f; void *x; int added; }; int sockinit(void); int sockwant(Socket*, int); int socknext(Socket**, int64); struct ms { size_t used, cap, last; void **items; ms_event_fn oninsert, onremove; }; enum { Walver = 7 }; enum // Jobrec.state { Invalid, Ready, Reserved, Buried, Delayed, Copy }; // if you modify this struct, you must increment Walver above struct Jobrec { uint64 id; uint32 pri; int64 delay; int64 ttr; int32 body_size; int64 created_at; int64 deadline_at; uint32 reserve_ct; uint32 timeout_ct; uint32 release_ct; uint32 bury_ct; uint32 kick_ct; byte state; }; struct job { Jobrec r; // persistent fields; these get written to the wal /* bookeeping fields; these are in-memory only */ char pad[6]; tube tube; job prev, next; /* linked list of jobs */ job ht_next; /* Next job in a hash table list */ size_t heap_index; /* where is this job in its current heap */ File *file; job fnext; job fprev; void *reserver; int walresv; int walused; char body[]; // written separately to the wal }; struct tube { uint refs; char name[MAX_TUBE_NAME_LEN]; Heap ready; Heap delay; struct ms waiting; /* set of conns */ struct stats stat; uint using_ct; uint watching_ct; int64 pause; int64 deadline_at; struct job buried; }; #define twarn(fmt, args...) warn("%s:%d in %s: " fmt, \ __FILE__, __LINE__, __func__, ##args) #define twarnx(fmt, args...) warnx("%s:%d in %s: " fmt, \ __FILE__, __LINE__, __func__, ##args) void warn(const char *fmt, ...) __attribute__((format(printf, 1, 2))); void warnx(const char *fmt, ...) __attribute__((format(printf, 1, 2))); char* fmtalloc(char *fmt, ...) __attribute__((format(printf, 1, 2))); void* zalloc(int n); #define new(T) zalloc(sizeof(T)) void optparse(Server*, char**); extern const char *progname; int64 nanoseconds(void); int rawfalloc(int fd, int len); void ms_init(ms a, ms_event_fn oninsert, ms_event_fn onremove); void ms_clear(ms a); int ms_append(ms a, void *item); int ms_remove(ms a, void *item); int ms_contains(ms a, void *item); void *ms_take(ms a); #define make_job(pri,delay,ttr,body_size,tube) make_job_with_id(pri,delay,ttr,body_size,tube,0) job allocate_job(int body_size); job make_job_with_id(uint pri, int64 delay, int64 ttr, int body_size, tube tube, uint64 id); void job_free(job j); /* Lookup a job by job ID */ job job_find(uint64 job_id); /* the void* parameters are really job pointers */ void job_setheappos(void*, int); int job_pri_less(void*, void*); int job_delay_less(void*, void*); job job_copy(job j); const char * job_state(job j); int job_list_any_p(job head); job job_remove(job j); void job_insert(job head, job j); uint64 total_jobs(void); /* for unit tests */ size_t get_all_jobs_used(void); extern struct ms tubes; tube make_tube(const char *name); void tube_dref(tube t); void tube_iref(tube t); tube tube_find(const char *name); tube tube_find_or_make(const char *name); #define TUBE_ASSIGN(a,b) (tube_dref(a), (a) = (b), tube_iref(a)) Conn *make_conn(int fd, char start_state, tube use, tube watch); int count_cur_conns(void); uint count_tot_conns(void); int count_cur_producers(void); int count_cur_workers(void); extern size_t primes[]; extern size_t job_data_size_limit; void prot_init(void); int64 prottick(Server *s); Conn *remove_waiting_conn(Conn *c); void enqueue_reserved_jobs(Conn *c); void enter_drain_mode(int sig); void h_accept(const int fd, const short which, Server* srv); void prot_remove_tube(tube t); int prot_replay(Server *s, job list); int make_server_socket(char *host_addr, char *port); struct Conn { Server *srv; Socket sock; char state; char type; Conn *next; tube use; int64 tickat; // time at which to do more work int tickpos; // position in srv->conns job soonest_job; // memoization of the soonest job int rw; // currently want: 'r', 'w', or 'h' int pending_timeout; char halfclosed; char cmd[LINE_BUF_SIZE]; // this string is NOT NUL-terminated int cmd_len; int cmd_read; char *reply; int reply_len; int reply_sent; char reply_buf[LINE_BUF_SIZE]; // this string IS NUL-terminated // How many bytes of in_job->body have been read so far. If in_job is NULL // while in_job_read is nonzero, we are in bit bucket mode and // in_job_read's meaning is inverted -- then it counts the bytes that // remain to be thrown away. int in_job_read; job in_job; // a job to be read from the client job out_job; int out_job_sent; struct ms watch; struct job reserved_jobs; // linked list header }; int connless(Conn *a, Conn *b); void connrec(Conn *c, int i); void connwant(Conn *c, int rw); void connsched(Conn *c); void connclose(Conn *c); void connsetproducer(Conn *c); void connsetworker(Conn *c); job connsoonestjob(Conn *c); int conndeadlinesoon(Conn *c); int conn_ready(Conn *c); #define conn_waiting(c) ((c)->type & CONN_TYPE_WAITING) enum { Filesizedef = (10 << 20) }; struct Wal { int filesize; int use; char *dir; File *head; File *cur; File *tail; int nfile; int next; int resv; // bytes reserved int alive; // bytes in use int64 nmig; // migrations int64 nrec; // records written ever int wantsync; int64 syncrate; int64 lastsync; int nocomp; // disable binlog compaction? }; int waldirlock(Wal*); void walinit(Wal*, job list); int walwrite(Wal*, job); void walmaint(Wal*); int walresvput(Wal*, job); int walresvupdate(Wal*, job); void walgc(Wal*); struct File { File *next; uint refs; int seq; int iswopen; // is open for writing int fd; int free; int resv; char *path; Wal *w; struct job jlist; // jobs written in this file }; int fileinit(File*, Wal*, int); Wal* fileadd(File*, Wal*); void fileincref(File*); void filedecref(File*); void fileaddjob(File*, job); void filermjob(File*, job); int fileread(File*, job list); void filewopen(File*); void filewclose(File*); int filewrjobshort(File*, job); int filewrjobfull(File*, job); #define Portdef "11300" struct Server { char *port; char *addr; char *user; Wal wal; Socket sock; Heap conns; }; void srvserve(Server *srv); void srvaccept(Server *s, int ev); beanstalkd-1.10/doc/000077500000000000000000000000001237005101200142775ustar00rootroot00000000000000beanstalkd-1.10/doc/beanstalkd.1000066400000000000000000000070711237005101200164760ustar00rootroot00000000000000.\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . .TH "BEANSTALKD" "1" "April 2012" "" "" . .SH "NAME" \fBbeanstalkd\fR \- simple, fast work queue . .SH "SYNOPSIS" \fBbeanstalkd\fR [\fIoptions\fR] . .SH "DESCRIPTION" \fBBeanstalkd\fR is a simple work\-queue service\. Its interface is generic, though it was originally designed for reducing the latency of page views in high\-volume web applications by running time\-consuming tasks asynchronously\. . .P When started, \fBbeanstalkd\fR opens a socket (or uses a file descriptor provided by the init(1) system, see \fIENVIRONMENT\fR) and listens for incoming connections\. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done\. See file \fBdoc/protocol\.txt\fR in the \fBbeanstalkd\fR distribution for a thorough description of the meaning and format of the \fBbeanstalkd\fR protocol\. . .SH "OPTIONS" . .TP \fB\-b\fR \fIpath\fR Use a binlog to keep jobs on persistent storage in directory \fIpath\fR\. Upon startup, \fBbeanstalkd\fR will recover any binlog that is present in \fIpath\fR, then, during normal operation, append new jobs and changes in state to the binlog\. . .TP \fB\-c\fR Perform online, incremental compaction of binlog files\. Negates \fB\-n\fR\. This is the default behavior\. . .IP (Do not use this option, except to negate \fB\-n\fR\. Both \fB\-c\fR and \fB\-n\fR will likely be removed in a future \fBbeanstalkd\fR release\.) . .TP \fB\-f\fR \fIms\fR Call fsync(2) at most once every \fIms\fR milliseconds\. Larger values for \fIms\fR reduce disk activity and improve speed at the cost of safety\. A power failure could result in the loss of up to \fIms\fR milliseconds of history\. . .IP A \fIms\fR value of 0 will cause \fBbeanstalkd\fR to call fsync every time it writes to the binlog\. . .IP (This option has no effect without \fB\-b\fR\.) . .TP \fB\-F\fR Never call fsync(2)\. Equivalent to \fB\-f\fR with an infinite \fIms\fR value\. . .IP This is the default behavior\. . .IP (This option has no effect without \fB\-b\fR\.) . .TP \fB\-h\fR Show a brief help message and exit\. . .TP \fB\-l\fR \fIaddr\fR Listen on address \fIaddr\fR (default is 0\.0\.0\.0)\. . .IP (Option \fB\-l\fR has no effect if sd\-daemon(5) socket activation is being used\. See also \fIENVIRONMENT\fR\.) . .TP \fB\-n\fR Turn off binlog compaction, negating \fB\-c\fR\. . .IP (Do not use this option\. Both \fB\-c\fR and \fB\-n\fR will likely be removed in a future \fBbeanstalkd\fR release\.) . .TP \fB\-p\fR \fIport\fR Listen on TCP port \fIport\fR (default is 11300)\. . .IP (Option \fB\-p\fR has no effect if sd\-daemon(5) socket activation is being used\. See also \fIENVIRONMENT\fR\.) . .TP \fB\-s\fR \fIbytes\fR The size in bytes of each binlog file\. . .IP (This option has no effect without \fB\-b\fR\.) . .TP \fB\-u\fR \fIuser\fR Become the user \fIuser\fR and its primary group\. . .TP \fB\-V\fR Increase verbosity\. May be used more than once to produce more verbose output\. The output format is subject to change\. . .TP \fB\-v\fR Print the version string and exit\. . .TP \fB\-z\fR \fIbytes\fR The maximum size in bytes of a job\. . .SH "ENVIRONMENT" . .TP \fBLISTEN_PID\fR, \fBLISTEN_FDS\fR These variables can be set by init(1)\. See sd_listen_fds(3) for details\. . .SH "SEE ALSO" sd\-daemon(5), sd_listen_fds(5) . .P Files \fBREADME\fR and \fBdoc/protocol\.txt\fR in the \fBbeanstalkd\fR distribution\. . .P \fIhttp://kr\.github\.com/beanstalkd/\fR . .SH "AUTHOR" \fBBeanstalkd\fR is written and maintained by Keith Rarick with the help of many others\. beanstalkd-1.10/doc/beanstalkd.1.html000066400000000000000000000200071237005101200174330ustar00rootroot00000000000000 beanstalkd(1) - simple, fast work queue
  1. beanstalkd(1)
  3. beanstalkd(1)

NAME

beanstalkd - simple, fast work queue

SYNOPSIS

beanstalkd [options]

DESCRIPTION

Beanstalkd is a simple work-queue service. Its interface is generic, though it was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously.

When started, beanstalkd opens a socket (or uses a file descriptor provided by the init(1) system, see ENVIRONMENT) and listens for incoming connections. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done. See file doc/protocol.txt in the beanstalkd distribution for a thorough description of the meaning and format of the beanstalkd protocol.

OPTIONS

-b path

Use a binlog to keep jobs on persistent storage in directory path. Upon startup, beanstalkd will recover any binlog that is present in path, then, during normal operation, append new jobs and changes in state to the binlog.

-c

Perform online, incremental compaction of binlog files. Negates -n. This is the default behavior.

(Do not use this option, except to negate -n. Both -c and -n will likely be removed in a future beanstalkd release.)

-f ms

Call fsync(2) at most once every ms milliseconds. Larger values for ms reduce disk activity and improve speed at the cost of safety. A power failure could result in the loss of up to ms milliseconds of history.

A ms value of 0 will cause beanstalkd to call fsync every time it writes to the binlog.

(This option has no effect without -b.)

-F

Never call fsync(2). Equivalent to -f with an infinite ms value.

This is the default behavior.

(This option has no effect without -b.)

-h

Show a brief help message and exit.

-l addr

Listen on address addr (default is 0.0.0.0).

(Option -l has no effect if sd-daemon(5) socket activation is being used. See also ENVIRONMENT.)

-n

Turn off binlog compaction, negating -c.

(Do not use this option. Both -c and -n will likely be removed in a future beanstalkd release.)

-p port

Listen on TCP port port (default is 11300).

(Option -p has no effect if sd-daemon(5) socket activation is being used. See also ENVIRONMENT.)

-s bytes

The size in bytes of each binlog file.

(This option has no effect without -b.)

-u user

Become the user user and its primary group.

-V

Increase verbosity. May be used more than once to produce more verbose output. The output format is subject to change.

-v

Print the version string and exit.

-z bytes

The maximum size in bytes of a job.

ENVIRONMENT

LISTEN_PID, LISTEN_FDS
These variables can be set by init(1). See sd_listen_fds(3) for details.

SEE ALSO

sd-daemon(5), sd_listen_fds(5)

Files README and doc/protocol.txt in the beanstalkd distribution.

http://kr.github.com/beanstalkd/

AUTHOR

Beanstalkd is written and maintained by Keith Rarick with the help of many others.

  2. April 2012
  3. beanstalkd(1)
beanstalkd-1.10/doc/beanstalkd.ronn000066400000000000000000000061571237005101200173160ustar00rootroot00000000000000beanstalkd(1) -- simple, fast work queue ======================================== ## SYNOPSIS `beanstalkd` [] ## DESCRIPTION `Beanstalkd` is a simple work-queue service. Its interface is generic, though it was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. When started, `beanstalkd` opens a socket (or uses a file descriptor provided by the init(1) system, see [ENVIRONMENT][]) and listens for incoming connections. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done. See file `doc/protocol.txt` in the `beanstalkd` distribution for a thorough description of the meaning and format of the `beanstalkd` protocol. ## OPTIONS * `-b` : Use a binlog to keep jobs on persistent storage in directory . Upon startup, `beanstalkd` will recover any binlog that is present in , then, during normal operation, append new jobs and changes in state to the binlog. * `-c`: Perform online, incremental compaction of binlog files. Negates `-n`. This is the default behavior. (Do not use this option, except to negate `-n`. Both `-c` and `-n` will likely be removed in a future `beanstalkd` release.) * `-f` : Call fsync(2) at most once every milliseconds. Larger values for reduce disk activity and improve speed at the cost of safety. A power failure could result in the loss of up to milliseconds of history. A value of 0 will cause `beanstalkd` to call fsync every time it writes to the binlog. (This option has no effect without `-b`.) * `-F`: Never call fsync(2). Equivalent to `-f` with an infinite value. This is the default behavior. (This option has no effect without `-b`.) * `-h`: Show a brief help message and exit. * `-l` : Listen on address (default is 0.0.0.0). (Option `-l` has no effect if sd-daemon(5) socket activation is being used. See also [ENVIRONMENT][].) * `-n`: Turn off binlog compaction, negating `-c`. (Do not use this option. Both `-c` and `-n` will likely be removed in a future `beanstalkd` release.) * `-p` : Listen on TCP port (default is 11300). (Option `-p` has no effect if sd-daemon(5) socket activation is being used. See also [ENVIRONMENT][].) * `-s` : The size in bytes of each binlog file. (This option has no effect without `-b`.) * `-u` : Become the user and its primary group. * `-V`: Increase verbosity. May be used more than once to produce more verbose output. The output format is subject to change. * `-v`: Print the version string and exit. * `-z` : The maximum size in bytes of a job. ## ENVIRONMENT * `LISTEN_PID`, `LISTEN_FDS`: These variables can be set by init(1). See sd_listen_fds(3) for details. ## SEE ALSO sd-daemon(5), sd_listen_fds(5) Files `README` and `doc/protocol.txt` in the `beanstalkd` distribution. ## AUTHOR `Beanstalkd` is written and maintained by Keith Rarick with the help of many others. beanstalkd-1.10/doc/protocol.md000066400000000000000000000604301237005101200164650ustar00rootroot00000000000000# Beanstalkd ## Protocol ### Description The beanstalk protocol runs over TCP using ASCII encoding. Clients connect, send commands and data, wait for responses, and close the connection. For each connection, the server processes commands serially in the order in which they were received and sends responses in the same order. All integers in the protocol are formatted in decimal and (unless otherwise indicated) nonnegative. ### Name convention Names only supports ASCII strings. #### Characters Allowed * **letters** (A-Z and a-z) * **numerals** (0-9) * **hyphen** ("-") * **plus** ("+") * **slash** ("/") * **semicolon** (";") * **dot** (".") * **dollar-sign** ("$") * **underscore** ("_") * **parentheses** ("*(*" and "*)*") **Notice:** They may not begin with a hyphen and they are terminated by white space (either a space char or end of line). Each name must be at least one character long. ### Errors | Errors | Description | | --------------------| ------------- | | `OUT_OF_MEMORY\r\n` | The server cannot allocate enough memory for the job. The client should try again later.| | `INTERNAL_ERROR\r\n` | This indicates a bug in the server. It should never happen. If it does happen, please report it at http://groups.google.com/group/beanstalk-talk. | | `BAD_FORMAT\r\n` | The client sent a command line that was not well-formed. This can happen if the line does not end with \r\n, if non-numeric characters occur where an integer is expected, if the wrong number of arguments are present, or if the command line is mal-formed in any other way. | | `UNKNOWN_COMMAND\r\n` | The client sent a command that the server does not know. | ### Job Lifecycle A job in beanstalk gets created by a client with the `put` command. During its life it can be in one of four states: `ready`, `reserved`, `delayed`, or `buried`. After the `put` command, a job typically starts out ready. It waits in the ready queue until a worker comes along and runs the "reserve" command. If this job is next in the queue, it will be reserved for the worker. The worker will execute the job; when it is finished the worker will send a `delete` command to delete the job. | Status | Description | | --------------------| ------------- | | `ready` | waiting to be reserved and processed after being put onto a tubed | | `reserved` | if this job is next in the queue, it will be reserved for the worker. The worker will execute the job | | `delayed` | waiting to become ready after the specified delay. | | `buried` | waiting to be kicked, usually after job fails to process | Here is a picture of the typical job lifecycle: ``` put reserve delete -----> [READY] ---------> [RESERVED] --------> *poof* ``` Here is a picture with more possibilities: ``` put with delay release with delay ----------------> [DELAYED] <------------. | | kick | (time passes) | | | put v reserve | delete -----------------> [READY] ---------> [RESERVED] --------> *poof* ^ ^ | | | \ release | | | `-------------' | | | | kick | | | | bury | [BURIED] <---------------' | | delete `--------> *poof* ``` ### Tubes The system has one or more tubes. Each tube consists of a ready queue and a delay queue. Each job spends its entire life in one tube. Consumers can show interest in tubes by sending the `watch` command; they can show disinterest by sending the `ignore` command. This set of interesting tubes is said to be a consumer's `watch list`. When a client reserves a job, it may come from any of the tubes in its watch list. When a client connects, its watch list is initially just the tube named `default`. If it submits jobs without having sent a `use` command, they will live in the tube named `default`. Tubes are created on demand whenever they are referenced. If a tube is empty (that is, it contains no `ready`, `delayed`, or `buried` jobs) and no client refers to it, it will be deleted. ## Commands ### Producer Commands #### `put` command The `put` command is for any process that wants to insert a job into the queue. It comprises a command line followed by the job body: ``` put \r\n \r\n ``` #####`put` options It inserts a job into the client's currently used tube (see the `use` command below). * `` is an integer < 2**32. Jobs with smaller priority values will be scheduled before jobs with larger priorities. The most urgent priority is 0;the least urgent priority is 4,294,967,295. * `` is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. * `` -- time to run -- is an integer number of seconds to allow a worker to run this job. This time is counted from the moment a worker reserves this job. If the worker does not delete, release, or bury the job within `` seconds, the job will time out and the server will release the job. The minimum ttr is 1. If the client sends 0, the server will silently increase the ttr to 1. * `` is an integer indicating the size of the job body, not including the trailing "\r\n". This value must be less than max-job-size (default: 2**16). * `` is the job body -- a sequence of bytes of length from the previous line. ##### `put` responses After sending the command line and body, the client waits for a reply, which may be: * `INSERTED \r\n` to indicate success. `` is the integer id of the new job * `BURIED \r\n` if the server ran out of memory trying to grow the priority queue data structure. `` is the integer id of the new job * `EXPECTED_CRLF\r\n` The job body must be followed by a CR-LF pair, that is, `"\r\n"`. These two bytes are not counted in the job size given by the client in the put command line. * `JOB_TOO_BIG\r\n` The client has requested to put a job with a body larger than max-job-size bytes. * `DRAINING\r\n` This means that the server has been put into "drain mode" and is no longer accepting new jobs. The client should try another server or disconnect and try again later. #### `use` command The `use` command is for producers. Subsequent put commands will put jobs into the tube specified by this command. If no use command has been issued, jobs will be put into the tube named `default`. ``` use \r\n ``` ##### `use` options * `` is a name at most 200 bytes. It specifies the tube to use. If the tube does not exist, it will be created. ##### `use` responses * `USING \r\n` -- `` is the name of the tube now being used. ### Worker Commands A process that wants to consume jobs from the queue uses those commands: * `reserve` * `delete` * `release` * `bury` #### `reserve` command ``` reserve\r\n ``` Alternatively, you can specify a timeout as follows: ``` reserve-with-timeout \r\n ``` This will return a newly-reserved job. If no job is available to be reserved, beanstalkd will wait to send a response until one becomes available. Once a job is reserved for the client, the client has limited time to run (TTR) the job before the job times out. When the job times out, the server will put the job back into the ready queue. Both the TTR and the actual time left can be found in response to the `stats-job` command. If more than one job is ready, beanstalkd will choose the one with the smallest priority value. Within each priority, it will choose the one that was received first. A timeout value of `0` will cause the server to immediately return either a response or `TIMED_OUT`. A positive value of timeout will limit the amount of time the client will block on the reserve request until a job becomes available. ##### `reserve` responses ###### Non-succesful responses * `DEADLINE_SOON\r\n` During the TTR of a reserved job, the last second is kept by the server as a safety margin, during which the client will not be made to wait for another job. If the client issues a reserve command during the safety margin, or if the safety margin arrives while the client is waiting on a reserve command. * `TIMED_OUT\r\n` If a non-negative timeout was specified and the timeout exceeded before a job became available, or if the client's connection is half-closed, the server will respond with TIMED_OUT. Otherwise, the only other response to this command is a successful reservation in the form of a text line followed by the job body: ####### Succesful response ``` RESERVED \r\n \r\n ``` * `` is the job id -- an integer unique to this job in this instance of beanstalkd. * `` is an integer indicating the size of the job body, not including the trailing `\r\n"`. * `` is the job body -- a sequence of bytes of length from the previous line. This is a verbatim copy of the bytes that were originally sent to the server in the put command for this job. #### `delete` command The delete command removes a job from the server entirely. It is normally used by the client when the job has successfully run to completion. A client can delete jobs that it has `reserved`, `ready` jobs, `delayed` jobs, and jobs that are `buried`. The delete command looks like this: ``` delete \r\n ``` ##### `delete` options * `` is the job id to delete. ##### `delete` responses The client then waits for one line of response, which may be: * `DELETED\r\n` to indicate success. * `NOT_FOUND\r\n` if the job does not exist or is not either reserved by the client, ready, or buried. This could happen if the job timed out before the client sent the delete command. #### `release` command The release command puts a `reserved` job back into the ready queue (and marks its state as `ready`) to be run by any client. It is normally used when the job fails because of a transitory error. It looks like this: ``` release \r\n ``` ##### `release` options * `` is the job id to release. * `` is a new priority to assign to the job. * `` is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. ##### `release` responses The client expects one line of response, which may be: * `RELEASED\r\n` to indicate success. * `BURIED\r\n` if the server ran out of memory trying to grow the priority queue data structure. * `NOT_FOUND\r\n` if the job does not exist or is not reserved by the client. #### `bury` command The bury command puts a job into the "buried" state. Buried jobs are put into a FIFO linked list and will not be touched by the server again until a client kicks them with the `kick`" command. The bury command looks like this: ``` bury \r\n ``` ##### `bury` options * `` is the job id to release. * `` is a new priority to assign to the job. ##### `bury` responses There are two possible responses: * `BURIED\r\n` to indicate success. * `NOT_FOUND\r\n` if the job does not exist or is not reserved by the client. #### `touch` command The `touch` command allows a worker to request more time to work on a job. This is useful for jobs that potentially take a long time, but you still want the benefits of a TTR pulling a job away from an unresponsive worker. A worker may periodically tell the server that it's still alive and processing a job (e.g. it may do this on `DEADLINE_SOON`). The command postpones the auto release of a reserved job until TTR seconds from when the command is issued. The touch command looks like this: ``` touch \r\n ``` ##### `touch` options * `` is the ID of a job reserved by the current connection. ##### `touch` responses There are two possible responses: * `TOUCHED\r\n` to indicate success. * `NOT_FOUND\r\n` if the job does not exist or is not reserved by the client. #### `watch` command The `watch` command adds the named tube to the watch list for the current connection. A reserve command will take a job from any of the tubes in the watch list. For each new connection, the watch list initially consists of one tube, named `default`. ``` watch \r\n ``` ##### `watch` options * `` is a name at most 200 bytes. It specifies a tube to add to the watch list. If the tube doesn't exist, it will be created. ##### `watch` responses The reply is: * `WATCHING \r\n` `` is the integer number of tubes currently in the watch list. ##### `ignore` command The `ignore` command is for consumers. It removes the named tube from the watch list for the current connection. ``` ignore \r\n ``` ##### `ignore` options * `` is a name at most 200 bytes. It specifies a tube to add to the watch list. If the tube doesn't exist, it will be created. ##### `ignore` command The reply is one of: * `WATCHING \r\n` to indicate success. `` is the integer number of tubes currently in the watch list. * `NOT_IGNORED\r\n` if the client attempts to ignore the only tube in its watch list. ### Other Commands #### `peek` command The peek commands let the client inspect a job in the system. There are four variations. All but the first operate only on the currently used tube. * `peek \r\n` - return `job `. * `peek-ready\r\n` - return the next ready job. * `peek-delayed\r\n` - return the delayed job with the shortest delay left. * `peek-buried\r\n` - return the next job in the list of buried jobs. ##### `peek` responses There are two possible responses, either a single line: * `NOT_FOUND\r\n` if the requested job doesn't exist or there are no jobs in the requested state. * `FOUND \r\n \r\n` * `` is the job id. * `` is an integer indicating the size of the job body, not including the trailing `\r\n`. * `` is the job body -- a sequence of bytes of length from the previous line. #### `kick` command The kick command applies only to the currently used tube. It moves jobs into the ready queue. If there are any buried jobs, it will only kick buried jobs. Otherwise it will kick delayed jobs. It looks like: ``` kick \r\n ``` ##### `kick` options * `` is an integer upper bound on the number of jobs to kick. The server will kick no more than jobs. ##### `kick` responses The response is of the form: * `KICKED \r\n` * is an integer indicating the number of jobs actually kicked. #### `kick-job` command The kick-job command is a variant of kick that operates with a single job identified by its job id. If the given job id exists and is in a buried or delayed state, it will be moved to the ready queue of the the same tube where it currently belongs. The syntax is: ``` kick-job \r\n ``` ##### `kick-job` options * is the job id to kick. ##### `kick-job` responses The response is one of: * `NOT_FOUND\r\n` if the job does not exist or is not in a kickable state. This can also happen upon internal errors. * `KICKED\r\n` when the operation succeeded. #### `stats-job` command The stats-job command gives statistical information about the specified job if it exists. Its form is: ``` stats-job \r\n ``` ##### `stats-job` options * `` is a job id. ##### `stats-job` responses The response is one of: * `NOT_FOUND\r\n` if the job does not exist. * `OK \r\n\r\n` * `` is the size of the following data section in bytes. * `` is a sequence of bytes of length `` from the previous line. It is a YAML file with statistical information represented a dictionary. The `stats-job` data is a YAML file representing a single dictionary of strings to scalars. It contains these keys: * `id` is the job id * `tube` is the name of the tube that contains this job * `state` is `ready` or `delayed` or `reserved` or `buried` * `pri` is the priority value set by the put, release, or bury commands. * `age` is the time in seconds since the put command that created this job. * `time-left` is the number of seconds left until the server puts this job into the ready queue. This number is only meaningful if the job is reserved or delayed. If the job is reserved and this amount of time elapses before its state changes, it is considered to have timed out. * `file` is the number of the earliest binlog file containing this job. If -b wasn't used, this will be 0. * `reserves` is the number of times this job has been reserved. * `timeouts` is the number of times this job has timed out during a reservation. * `releases` is the number of times a client has released this job from a reservation. * `buries` is the number of times this job has been buried. * `kicks` is the number of times this job has been kicked. #### `stats-tube` command The stats-tube command gives statistical information about the specified tube if it exists. Its form is: ``` stats-tube \r\n ``` ##### `stats-tube` options * `` is a name at most 200 bytes. Stats will be returned for this tube. ##### `stats-tube` responses The response is one of: * `NOT_FOUND\r\n` if the tube does not exist. * `OK \r\n\r\n` * `` is the size of the following data section in bytes. * `` is a sequence of bytes of length `` from the previous line. It is a YAML file with statistical information represented a dictionary. The stats-tube data is a YAML file representing a single dictionary of strings to scalars. It contains these keys: * `name` is the tube's name. * `current-jobs-urgent` is the number of ready jobs with priority < 1024 in this tube. * `current-jobs-ready` is the number of jobs in the ready queue in this tube. * `current-jobs-reserved` is the number of jobs reserved by all clients in this tube. * `current-jobs-delayed` is the number of delayed jobs in this tube. * `current-jobs-buried` is the number of buried jobs in this tube. * `total-jobs` is the cumulative count of jobs created in this tube in the current beanstalkd process. * `current-using` is the number of open connections that are currently using this tube. * `current-waiting` is the number of open connections that have issued a reserve command while watching this tube but not yet received a response. * `current-watching` is the number of open connections that are currently watching this tube. * `pause` is the number of seconds the tube has been paused for. * `cmd-delete` is the cumulative number of delete commands for this tube * `cmd-pause-tube` is the cumulative number of pause-tube commands for this tube. * `pause-time-left` is the number of seconds until the tube is un-paused. #### `stats` command The stats command gives statistical information about the system as a whole. Its form is: ``` stats\r\n ``` ##### `stats` responses The server will respond: ``` OK \r\n \r\n ``` * `` is the size of the following data section in bytes. * `` is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented a dictionary. The stats data for the system is a YAML file representing a single dictionary of strings to scalars. Entries described as "cumulative" are reset when the beanstalkd process starts; they are not stored on disk with the -b flag. * `current-jobs-urgent` is the number of ready jobs with priority < 1024. * `current-jobs-ready` is the number of jobs in the ready queue. * `current-jobs-reserved` is the number of jobs reserved by all clients. * `current-jobs-delayed` is the number of delayed jobs. * `current-jobs-buried` is the number of buried jobs. * `cmd-put` is the cumulative number of put commands. * `cmd-peek` is the cumulative number of peek commands. * `cmd-peek-ready` is the cumulative number of peek-ready commands. * `cmd-peek-delayed` is the cumulative number of peek-delayed commands. * `cmd-peek-buried` is the cumulative number of peek-buried commands. * `cmd-reserve` is the cumulative number of reserve commands. * `cmd-use` is the cumulative number of use commands. * `cmd-watch` is the cumulative number of watch commands. * `cmd-ignore` is the cumulative number of ignore commands. * `cmd-delete` is the cumulative number of delete commands. * `cmd-release` is the cumulative number of release commands. * `cmd-bury` is the cumulative number of bury commands. * `cmd-kick` is the cumulative number of kick commands. * `cmd-stats` is the cumulative number of stats commands. * `cmd-stats-job` is the cumulative number of stats-job commands. * `cmd-stats-tube` is the cumulative number of stats-tube commands. * `cmd-list-tubes` is the cumulative number of list-tubes commands. * `cmd-list-tube-used` is the cumulative number of list-tube-used commands. * `cmd-list-tubes-watched` is the cumulative number of list-tubes-watched commands. * `cmd-pause-tube` is the cumulative number of pause-tube commands. * `job-timeouts` is the cumulative count of times a job has timed out. * `total-jobs` is the cumulative count of jobs created. * `max-job-size` is the maximum number of bytes in a job. * `current-tubes` is the number of currently-existing tubes. * `current-connections` is the number of currently open connections. * `current-producers` is the number of open connections that have each issued at least one put command. * `current-workers` is the number of open connections that have each issued at least one reserve command. * `current-waiting` is the number of open connections that have issued a reserve command but not yet received a response. * `total-connections` is the cumulative count of connections. * `pid` is the process id of the server. * `version` is the version string of the server. * `rusage-utime` is the cumulative user CPU time of this process in seconds and microseconds. * `rusage-stime` is the cumulative system CPU time of this process in seconds and microseconds. * `uptime` is the number of seconds since this server process started running. * `binlog-oldest-index` is the index of the oldest binlog file needed to store the current jobs. * `binlog-current-index` is the index of the current binlog file being written to. If binlog is not active this value will be 0. * `binlog-max-size` is the maximum size in bytes a binlog file is allowed to get before a new binlog file is opened. * `binlog-records-written` is the cumulative number of records written to the binlog. * `binlog-records-migrated` is the cumulative number of records written as part of compaction. * `id` is a random id string for this server process, generated when each beanstalkd process starts. * `hostname` is the hostname of the machine as determined by uname. #### `list-tubes` command The list-tubes command returns a list of all existing tubes. Its form is: ``` list-tubes\r\n ``` ##### `list-tubes` responses The response is: ``` OK \r\n \r\n ``` * `` is the size of the following data section in bytes. * `` is a sequence of bytes of length from the previous line. It is a YAML file containing all tube names as a list of strings. #### `list-tube-used` command The list-tube-used command returns the tube currently being used by the client. Its form is: ``` list-tube-used\r\n ``` ##### `list-tube-used` responses The response is: ``` USING \r\n ``` * `` is the name of the tube being used. #### `list-tubes-watched` command The list-tubes-watched command returns a list tubes currently being watched by the client. Its form is: ``` list-tubes-watched\r\n ``` ##### `list-tubes-watched` responses The response is: ``` OK \r\n \r\n ``` * `` is the size of the following data section in bytes. * `` is a sequence of bytes of length from the previous line. It is a YAML file containing watched tube names as a list of strings. #### `quit` command The quit command simply closes the connection. Its form is: ``` quit\r\n ``` #### `pause-tube` command The pause-tube command can delay any new job being reserved for a given time. Its form is: ``` pause-tube \r\n ``` ##### `pause-tube` options * `` is the tube to pause * `` is an integer number of seconds to wait before reserving any more jobs from the queue ##### `pause-tube` responses There are two possible responses: * `PAUSED\r\n` to indicate success. * `NOT_FOUND\r\n` if the tube does not exist. beanstalkd-1.10/doc/protocol.txt000066400000000000000000000577261237005101200167220ustar00rootroot00000000000000= Beanstalk Protocol = Protocol -------- The beanstalk protocol runs over TCP using ASCII encoding. Clients connect, send commands and data, wait for responses, and close the connection. For each connection, the server processes commands serially in the order in which they were received and sends responses in the same order. All integers in the protocol are formatted in decimal and (unless otherwise indicated) nonnegative. Names, in this protocol, are ASCII strings. They may contain letters (A-Z and a-z), numerals (0-9), hyphen ("-"), plus ("+"), slash ("/"), semicolon (";"), dot ("."), dollar-sign ("$"), underscore ("_"), and parentheses ("(" and ")"), but they may not begin with a hyphen. They are terminated by white space (either a space char or end of line). Each name must be at least one character long. The protocol contains two kinds of data: text lines and unstructured chunks of data. Text lines are used for client commands and server responses. Chunks are used to transfer job bodies and stats information. Each job body is an opaque sequence of bytes. The server never inspects or modifies a job body and always sends it back in its original form. It is up to the clients to agree on a meaningful interpretation of job bodies. The client may issue the "quit" command, or simply close the TCP connection when it no longer has use for the server. However, beanstalkd performs very well with a large number of open connections, so it is usually better for the client to keep its connection open and reuse it as much as possible. This also avoids the overhead of establishing new TCP connections. If a client violates the protocol (such as by sending a request that is not well-formed or a command that does not exist) or if the server has an error, the server will reply with one of the following error messages: - "OUT_OF_MEMORY\r\n" The server cannot allocate enough memory for the job. The client should try again later. - "INTERNAL_ERROR\r\n" This indicates a bug in the server. It should never happen. If it does happen, please report it at http://groups.google.com/group/beanstalk-talk. - "BAD_FORMAT\r\n" The client sent a command line that was not well-formed. This can happen if the line does not end with \r\n, if non-numeric characters occur where an integer is expected, if the wrong number of arguments are present, or if the command line is mal-formed in any other way. - "UNKNOWN_COMMAND\r\n" The client sent a command that the server does not know. These error responses will not be listed in this document for individual commands in the following sections, but they are implicitly included in the description of all commands. Clients should be prepared to receive an error response after any command. As a last resort, if the server has a serious error that prevents it from continuing service to the current client, the server will close the connection. Job Lifecycle ------------- A job in beanstalk gets created by a client with the "put" command. During its life it can be in one of four states: "ready", "reserved", "delayed", or "buried". After the put command, a job typically starts out ready. It waits in the ready queue until a worker comes along and runs the "reserve" command. If this job is next in the queue, it will be reserved for the worker. The worker will execute the job; when it is finished the worker will send a "delete" command to delete the job. Here is a picture of the typical job lifecycle: put reserve delete -----> [READY] ---------> [RESERVED] --------> *poof* Here is a picture with more possibilities: put with delay release with delay ----------------> [DELAYED] <------------. | | | (time passes) | | | put v reserve | delete -----------------> [READY] ---------> [RESERVED] --------> *poof* ^ ^ | | | \ release | | | `-------------' | | | | kick | | | | bury | [BURIED] <---------------' | | delete `--------> *poof* The system has one or more tubes. Each tube consists of a ready queue and a delay queue. Each job spends its entire life in one tube. Consumers can show interest in tubes by sending the "watch" command; they can show disinterest by sending the "ignore" command. This set of interesting tubes is said to be a consumer's "watch list". When a client reserves a job, it may come from any of the tubes in its watch list. When a client connects, its watch list is initially just the tube named "default". If it submits jobs without having sent a "use" command, they will live in the tube named "default". Tubes are created on demand whenever they are referenced. If a tube is empty (that is, it contains no ready, delayed, or buried jobs) and no client refers to it, it will be deleted. Producer Commands ----------------- The "put" command is for any process that wants to insert a job into the queue. It comprises a command line followed by the job body: put \r\n \r\n It inserts a job into the client's currently used tube (see the "use" command below). - is an integer < 2**32. Jobs with smaller priority values will be scheduled before jobs with larger priorities. The most urgent priority is 0; the least urgent priority is 4,294,967,295. - is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. - -- time to run -- is an integer number of seconds to allow a worker to run this job. This time is counted from the moment a worker reserves this job. If the worker does not delete, release, or bury the job within seconds, the job will time out and the server will release the job. The minimum ttr is 1. If the client sends 0, the server will silently increase the ttr to 1. - is an integer indicating the size of the job body, not including the trailing "\r\n". This value must be less than max-job-size (default: 2**16). - is the job body -- a sequence of bytes of length from the previous line. After sending the command line and body, the client waits for a reply, which may be: - "INSERTED \r\n" to indicate success. - is the integer id of the new job - "BURIED \r\n" if the server ran out of memory trying to grow the priority queue data structure. - is the integer id of the new job - "EXPECTED_CRLF\r\n" The job body must be followed by a CR-LF pair, that is, "\r\n". These two bytes are not counted in the job size given by the client in the put command line. - "JOB_TOO_BIG\r\n" The client has requested to put a job with a body larger than max-job-size bytes. - "DRAINING\r\n" This means that the server has been put into "drain mode" and is no longer accepting new jobs. The client should try another server or disconnect and try again later. The "use" command is for producers. Subsequent put commands will put jobs into the tube specified by this command. If no use command has been issued, jobs will be put into the tube named "default". use \r\n - is a name at most 200 bytes. It specifies the tube to use. If the tube does not exist, it will be created. The only reply is: USING \r\n - is the name of the tube now being used. Worker Commands --------------- A process that wants to consume jobs from the queue uses "reserve", "delete", "release", and "bury". The first worker command, "reserve", looks like this: reserve\r\n Alternatively, you can specify a timeout as follows: reserve-with-timeout \r\n This will return a newly-reserved job. If no job is available to be reserved, beanstalkd will wait to send a response until one becomes available. Once a job is reserved for the client, the client has limited time to run (TTR) the job before the job times out. When the job times out, the server will put the job back into the ready queue. Both the TTR and the actual time left can be found in response to the stats-job command. If more than one job is ready, beanstalkd will choose the one with the smallest priority value. Within each priority, it will choose the one that was received first. A timeout value of 0 will cause the server to immediately return either a response or TIMED_OUT. A positive value of timeout will limit the amount of time the client will block on the reserve request until a job becomes available. During the TTR of a reserved job, the last second is kept by the server as a safety margin, during which the client will not be made to wait for another job. If the client issues a reserve command during the safety margin, or if the safety margin arrives while the client is waiting on a reserve command, the server will respond with: DEADLINE_SOON\r\n This gives the client a chance to delete or release its reserved job before the server automatically releases it. TIMED_OUT\r\n If a non-negative timeout was specified and the timeout exceeded before a job became available, or if the client's connection is half-closed, the server will respond with TIMED_OUT. Otherwise, the only other response to this command is a successful reservation in the form of a text line followed by the job body: RESERVED \r\n \r\n - is the job id -- an integer unique to this job in this instance of beanstalkd. - is an integer indicating the size of the job body, not including the trailing "\r\n". - is the job body -- a sequence of bytes of length from the previous line. This is a verbatim copy of the bytes that were originally sent to the server in the put command for this job. The delete command removes a job from the server entirely. It is normally used by the client when the job has successfully run to completion. A client can delete jobs that it has reserved, ready jobs, delayed jobs, and jobs that are buried. The delete command looks like this: delete \r\n - is the job id to delete. The client then waits for one line of response, which may be: - "DELETED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not either reserved by the client, ready, or buried. This could happen if the job timed out before the client sent the delete command. The release command puts a reserved job back into the ready queue (and marks its state as "ready") to be run by any client. It is normally used when the job fails because of a transitory error. It looks like this: release \r\n - is the job id to release. - is a new priority to assign to the job. - is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. The client expects one line of response, which may be: - "RELEASED\r\n" to indicate success. - "BURIED\r\n" if the server ran out of memory trying to grow the priority queue data structure. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The bury command puts a job into the "buried" state. Buried jobs are put into a FIFO linked list and will not be touched by the server again until a client kicks them with the "kick" command. The bury command looks like this: bury \r\n - is the job id to release. - is a new priority to assign to the job. There are two possible responses: - "BURIED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The "touch" command allows a worker to request more time to work on a job. This is useful for jobs that potentially take a long time, but you still want the benefits of a TTR pulling a job away from an unresponsive worker. A worker may periodically tell the server that it's still alive and processing a job (e.g. it may do this on DEADLINE_SOON). The command postpones the auto release of a reserved job until TTR seconds from when the command is issued. The touch command looks like this: touch \r\n - is the ID of a job reserved by the current connection. There are two possible responses: - "TOUCHED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The "watch" command adds the named tube to the watch list for the current connection. A reserve command will take a job from any of the tubes in the watch list. For each new connection, the watch list initially consists of one tube, named "default". watch \r\n - is a name at most 200 bytes. It specifies a tube to add to the watch list. If the tube doesn't exist, it will be created. The reply is: WATCHING \r\n - is the integer number of tubes currently in the watch list. The "ignore" command is for consumers. It removes the named tube from the watch list for the current connection. ignore \r\n The reply is one of: - "WATCHING \r\n" to indicate success. - is the integer number of tubes currently in the watch list. - "NOT_IGNORED\r\n" if the client attempts to ignore the only tube in its watch list. Other Commands -------------- The peek commands let the client inspect a job in the system. There are four variations. All but the first operate only on the currently used tube. - "peek \r\n" - return job . - "peek-ready\r\n" - return the next ready job. - "peek-delayed\r\n" - return the delayed job with the shortest delay left. - "peek-buried\r\n" - return the next job in the list of buried jobs. There are two possible responses, either a single line: - "NOT_FOUND\r\n" if the requested job doesn't exist or there are no jobs in the requested state. Or a line followed by a chunk of data, if the command was successful: FOUND \r\n \r\n - is the job id. - is an integer indicating the size of the job body, not including the trailing "\r\n". - is the job body -- a sequence of bytes of length from the previous line. The kick command applies only to the currently used tube. It moves jobs into the ready queue. If there are any buried jobs, it will only kick buried jobs. Otherwise it will kick delayed jobs. It looks like: kick \r\n - is an integer upper bound on the number of jobs to kick. The server will kick no more than jobs. The response is of the form: KICKED \r\n - is an integer indicating the number of jobs actually kicked. The kick-job command is a variant of kick that operates with a single job identified by its job id. If the given job id exists and is in a buried or delayed state, it will be moved to the ready queue of the the same tube where it currently belongs. The syntax is: kick-job \r\n - is the job id to kick. The response is one of: - "NOT_FOUND\r\n" if the job does not exist or is not in a kickable state. This can also happen upon internal errors. - "KICKED\r\n" when the operation succeeded. The stats-job command gives statistical information about the specified job if it exists. Its form is: stats-job \r\n - is a job id. The response is one of: - "NOT_FOUND\r\n" if the job does not exist. - "OK \r\n\r\n" - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented a dictionary. The stats-job data is a YAML file representing a single dictionary of strings to scalars. It contains these keys: - "id" is the job id - "tube" is the name of the tube that contains this job - "state" is "ready" or "delayed" or "reserved" or "buried" - "pri" is the priority value set by the put, release, or bury commands. - "age" is the time in seconds since the put command that created this job. - "time-left" is the number of seconds left until the server puts this job into the ready queue. This number is only meaningful if the job is reserved or delayed. If the job is reserved and this amount of time elapses before its state changes, it is considered to have timed out. - "file" is the number of the earliest binlog file containing this job. If -b wasn't used, this will be 0. - "reserves" is the number of times this job has been reserved. - "timeouts" is the number of times this job has timed out during a reservation. - "releases" is the number of times a client has released this job from a reservation. - "buries" is the number of times this job has been buried. - "kicks" is the number of times this job has been kicked. The stats-tube command gives statistical information about the specified tube if it exists. Its form is: stats-tube \r\n - is a name at most 200 bytes. Stats will be returned for this tube. The response is one of: - "NOT_FOUND\r\n" if the tube does not exist. - "OK \r\n\r\n" - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented a dictionary. The stats-tube data is a YAML file representing a single dictionary of strings to scalars. It contains these keys: - "name" is the tube's name. - "current-jobs-urgent" is the number of ready jobs with priority < 1024 in this tube. - "current-jobs-ready" is the number of jobs in the ready queue in this tube. - "current-jobs-reserved" is the number of jobs reserved by all clients in this tube. - "current-jobs-delayed" is the number of delayed jobs in this tube. - "current-jobs-buried" is the number of buried jobs in this tube. - "total-jobs" is the cumulative count of jobs created in this tube in the current beanstalkd process. - "current-using" is the number of open connections that are currently using this tube. - "current-waiting" is the number of open connections that have issued a reserve command while watching this tube but not yet received a response. - "current-watching" is the number of open connections that are currently watching this tube. - "pause" is the number of seconds the tube has been paused for. - "cmd-delete" is the cumulative number of delete commands for this tube - "cmd-pause-tube" is the cumulative number of pause-tube commands for this tube. - "pause-time-left" is the number of seconds until the tube is un-paused. The stats command gives statistical information about the system as a whole. Its form is: stats\r\n The server will respond: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented a dictionary. The stats data for the system is a YAML file representing a single dictionary of strings to scalars. Entries described as "cumulative" are reset when the beanstalkd process starts; they are not stored on disk with the -b flag. - "current-jobs-urgent" is the number of ready jobs with priority < 1024. - "current-jobs-ready" is the number of jobs in the ready queue. - "current-jobs-reserved" is the number of jobs reserved by all clients. - "current-jobs-delayed" is the number of delayed jobs. - "current-jobs-buried" is the number of buried jobs. - "cmd-put" is the cumulative number of put commands. - "cmd-peek" is the cumulative number of peek commands. - "cmd-peek-ready" is the cumulative number of peek-ready commands. - "cmd-peek-delayed" is the cumulative number of peek-delayed commands. - "cmd-peek-buried" is the cumulative number of peek-buried commands. - "cmd-reserve" is the cumulative number of reserve commands. - "cmd-use" is the cumulative number of use commands. - "cmd-watch" is the cumulative number of watch commands. - "cmd-ignore" is the cumulative number of ignore commands. - "cmd-delete" is the cumulative number of delete commands. - "cmd-release" is the cumulative number of release commands. - "cmd-bury" is the cumulative number of bury commands. - "cmd-kick" is the cumulative number of kick commands. - "cmd-stats" is the cumulative number of stats commands. - "cmd-stats-job" is the cumulative number of stats-job commands. - "cmd-stats-tube" is the cumulative number of stats-tube commands. - "cmd-list-tubes" is the cumulative number of list-tubes commands. - "cmd-list-tube-used" is the cumulative number of list-tube-used commands. - "cmd-list-tubes-watched" is the cumulative number of list-tubes-watched commands. - "cmd-pause-tube" is the cumulative number of pause-tube commands. - "job-timeouts" is the cumulative count of times a job has timed out. - "total-jobs" is the cumulative count of jobs created. - "max-job-size" is the maximum number of bytes in a job. - "current-tubes" is the number of currently-existing tubes. - "current-connections" is the number of currently open connections. - "current-producers" is the number of open connections that have each issued at least one put command. - "current-workers" is the number of open connections that have each issued at least one reserve command. - "current-waiting" is the number of open connections that have issued a reserve command but not yet received a response. - "total-connections" is the cumulative count of connections. - "pid" is the process id of the server. - "version" is the version string of the server. - "rusage-utime" is the cumulative user CPU time of this process in seconds and microseconds. - "rusage-stime" is the cumulative system CPU time of this process in seconds and microseconds. - "uptime" is the number of seconds since this server process started running. - "binlog-oldest-index" is the index of the oldest binlog file needed to store the current jobs. - "binlog-current-index" is the index of the current binlog file being written to. If binlog is not active this value will be 0. - "binlog-max-size" is the maximum size in bytes a binlog file is allowed to get before a new binlog file is opened. - "binlog-records-written" is the cumulative number of records written to the binlog. - "binlog-records-migrated" is the cumulative number of records written as part of compaction. - "id" is a random id string for this server process, generated when each beanstalkd process starts. - "hostname" the hostname of the machine as determined by uname. The list-tubes command returns a list of all existing tubes. Its form is: list-tubes\r\n The response is: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file containing all tube names as a list of strings. The list-tube-used command returns the tube currently being used by the client. Its form is: list-tube-used\r\n The response is: USING \r\n - is the name of the tube being used. The list-tubes-watched command returns a list tubes currently being watched by the client. Its form is: list-tubes-watched\r\n The response is: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file containing watched tube names as a list of strings. The quit command simply closes the connection. Its form is: quit\r\n The pause-tube command can delay any new job being reserved for a given time. Its form is: pause-tube \r\n - is the tube to pause - is an integer number of seconds to wait before reserving any more jobs from the queue There are two possible responses: - "PAUSED\r\n" to indicate success. - "NOT_FOUND\r\n" if the tube does not exist. beanstalkd-1.10/file.c000066400000000000000000000277731237005101200146350ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include "dat.h" static int readrec(File*, job, int*); static int readrec5(File*, job, int*); static int readfull(File*, void*, int, int*, char*); static void warnpos(File*, int, char*, ...) __attribute__((format(printf, 3, 4))); FAlloc *falloc = &rawfalloc; enum { Walver5 = 5 }; typedef struct Jobrec5 Jobrec5; struct Jobrec5 { uint64 id; uint32 pri; uint64 delay; // usec uint64 ttr; // usec int32 body_size; uint64 created_at; // usec uint64 deadline_at; // usec uint32 reserve_ct; uint32 timeout_ct; uint32 release_ct; uint32 bury_ct; uint32 kick_ct; byte state; char pad[1]; }; enum { Jobrec5size = offsetof(Jobrec5, pad) }; void fileincref(File *f) { if (!f) return; f->refs++; } void filedecref(File *f) { if (!f) return; f->refs--; if (f->refs < 1) { walgc(f->w); } } void fileaddjob(File *f, job j) { job h; h = &f->jlist; if (!h->fprev) h->fprev = h; j->file = f; j->fprev = h->fprev; j->fnext = h; h->fprev->fnext = j; h->fprev = j; fileincref(f); } void filermjob(File *f, job j) { if (!f) return; if (f != j->file) return; j->fnext->fprev = j->fprev; j->fprev->fnext = j->fnext; j->fnext = 0; j->fprev = 0; j->file = NULL; f->w->alive -= j->walused; j->walused = 0; filedecref(f); } // Fileread reads jobs from f->path into list. // It returns 0 on success, or 1 if any errors occurred. int fileread(File *f, job list) { int err = 0, v; if (!readfull(f, &v, sizeof(v), &err, "version")) { return err; } switch (v) { case Walver: fileincref(f); while (readrec(f, list, &err)); filedecref(f); return err; case Walver5: fileincref(f); while (readrec5(f, list, &err)); filedecref(f); return err; } warnx("%s: unknown version: %d", f->path, v); return 1; } // Readrec reads a record from f->fd into linked list l. // If an error occurs, it sets *err to 1. // Readrec returns the number of records read, either 1 or 0. static int readrec(File *f, job l, int *err) { int r, sz = 0; int namelen; Jobrec jr; job j; tube t; char tubename[MAX_TUBE_NAME_LEN]; r = read(f->fd, &namelen, sizeof(int)); if (r == -1) { twarn("read"); warnpos(f, 0, "error"); *err = 1; return 0; } if (r != sizeof(int)) { return 0; } sz += r; if (namelen >= MAX_TUBE_NAME_LEN) { warnpos(f, -r, "namelen %d exceeds maximum of %d", namelen, MAX_TUBE_NAME_LEN - 1); *err = 1; return 0; } if (namelen < 0) { warnpos(f, -r, "namelen %d is negative", namelen); *err = 1; return 0; } if (namelen) { r = readfull(f, tubename, namelen, err, "tube name"); if (!r) { return 0; } sz += r; } tubename[namelen] = '\0'; r = readfull(f, &jr, sizeof(Jobrec), err, "job struct"); if (!r) { return 0; } sz += r; // are we reading trailing zeroes? if (!jr.id) return 0; j = job_find(jr.id); if (!(j || namelen)) { // We read a short record without having seen a // full record for this job, so the full record // was in an earlier file that has been deleted. // Therefore the job itself has either been // deleted or migrated; either way, this record // should be ignored. return 1; } switch (jr.state) { case Reserved: jr.state = Ready; case Ready: case Buried: case Delayed: if (!j) { if (jr.body_size > job_data_size_limit) { warnpos(f, -r, "job %"PRIu64" is too big (%"PRId32" > %zu)", jr.id, jr.body_size, job_data_size_limit); goto Error; } t = tube_find_or_make(tubename); j = make_job_with_id(jr.pri, jr.delay, jr.ttr, jr.body_size, t, jr.id); j->next = j->prev = j; j->r.created_at = jr.created_at; } j->r = jr; job_insert(l, j); // full record; read the job body if (namelen) { if (jr.body_size != j->r.body_size) { warnpos(f, -r, "job %"PRIu64" size changed", j->r.id); warnpos(f, -r, "was %d, now %d", j->r.body_size, jr.body_size); goto Error; } r = readfull(f, j->body, j->r.body_size, err, "job body"); if (!r) { goto Error; } sz += r; // since this is a full record, we can move // the file pointer and decref the old // file, if any filermjob(j->file, j); fileaddjob(f, j); } j->walused += sz; f->w->alive += sz; return 1; case Invalid: if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 1; } Error: *err = 1; if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 0; } // Readrec5 is like readrec, but it reads a record in "version 5" // of the log format. static int readrec5(File *f, job l, int *err) { int r, sz = 0; size_t namelen; Jobrec5 jr; job j; tube t; char tubename[MAX_TUBE_NAME_LEN]; r = read(f->fd, &namelen, sizeof(namelen)); if (r == -1) { twarn("read"); warnpos(f, 0, "error"); *err = 1; return 0; } if (r != sizeof(namelen)) { return 0; } sz += r; if (namelen >= MAX_TUBE_NAME_LEN) { warnpos(f, -r, "namelen %zu exceeds maximum of %d", namelen, MAX_TUBE_NAME_LEN - 1); *err = 1; return 0; } if (namelen) { r = readfull(f, tubename, namelen, err, "v5 tube name"); if (!r) { return 0; } sz += r; } tubename[namelen] = '\0'; r = readfull(f, &jr, Jobrec5size, err, "v5 job struct"); if (!r) { return 0; } sz += r; // are we reading trailing zeroes? if (!jr.id) return 0; j = job_find(jr.id); if (!(j || namelen)) { // We read a short record without having seen a // full record for this job, so the full record // was in an eariler file that has been deleted. // Therefore the job itself has either been // deleted or migrated; either way, this record // should be ignored. return 1; } switch (jr.state) { case Reserved: jr.state = Ready; case Ready: case Buried: case Delayed: if (!j) { if (jr.body_size > job_data_size_limit) { warnpos(f, -r, "job %"PRIu64" is too big (%"PRId32" > %zu)", jr.id, jr.body_size, job_data_size_limit); goto Error; } t = tube_find_or_make(tubename); j = make_job_with_id(jr.pri, jr.delay, jr.ttr, jr.body_size, t, jr.id); j->next = j->prev = j; j->r.created_at = jr.created_at; } j->r.id = jr.id; j->r.pri = jr.pri; j->r.delay = jr.delay * 1000; // us => ns j->r.ttr = jr.ttr * 1000; // us => ns j->r.body_size = jr.body_size; j->r.created_at = jr.created_at * 1000; // us => ns j->r.deadline_at = jr.deadline_at * 1000; // us => ns j->r.reserve_ct = jr.reserve_ct; j->r.timeout_ct = jr.timeout_ct; j->r.release_ct = jr.release_ct; j->r.bury_ct = jr.bury_ct; j->r.kick_ct = jr.kick_ct; j->r.state = jr.state; job_insert(l, j); // full record; read the job body if (namelen) { if (jr.body_size != j->r.body_size) { warnpos(f, -r, "job %"PRIu64" size changed", j->r.id); warnpos(f, -r, "was %"PRId32", now %"PRId32, j->r.body_size, jr.body_size); goto Error; } r = readfull(f, j->body, j->r.body_size, err, "v5 job body"); if (!r) { goto Error; } sz += r; // since this is a full record, we can move // the file pointer and decref the old // file, if any filermjob(j->file, j); fileaddjob(f, j); } j->walused += sz; f->w->alive += sz; return 1; case Invalid: if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 1; } Error: *err = 1; if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 0; } static int readfull(File *f, void *c, int n, int *err, char *desc) { int r; r = read(f->fd, c, n); if (r == -1) { twarn("read"); warnpos(f, 0, "error reading %s", desc); *err = 1; return 0; } if (r != n) { warnpos(f, -r, "unexpected EOF reading %d bytes (got %d): %s", n, r, desc); *err = 1; return 0; } return r; } static void warnpos(File *f, int adj, char *fmt, ...) { int off; va_list ap; off = lseek(f->fd, 0, SEEK_CUR); fprintf(stderr, "%s:%u: ", f->path, off+adj); va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); fputc('\n', stderr); } // Opens f for writing, writes a header, and initializes // f->free and f->resv. // Sets f->iswopen if successful. void filewopen(File *f) { int fd, r; int n; int ver = Walver; fd = open(f->path, O_WRONLY|O_CREAT, 0400); if (fd < 0) { twarn("open %s", f->path); return; } r = falloc(fd, f->w->filesize); if (r) { close(fd); errno = r; twarn("falloc %s", f->path); r = unlink(f->path); if (r) { twarn("unlink %s", f->path); } return; } n = write(fd, &ver, sizeof(int)); if (n < sizeof(int)) { twarn("write %s", f->path); close(fd); return; } f->fd = fd; f->iswopen = 1; fileincref(f); f->free = f->w->filesize - n; f->resv = 0; } static int filewrite(File *f, job j, void *buf, int len) { int r; r = write(f->fd, buf, len); if (r != len) { twarn("write"); return 0; } f->w->resv -= r; f->resv -= r; j->walresv -= r; j->walused += r; f->w->alive += r; return 1; } int filewrjobshort(File *f, job j) { int r, nl; nl = 0; // name len 0 indicates short record r = filewrite(f, j, &nl, sizeof nl) && filewrite(f, j, &j->r, sizeof j->r); if (!r) return 0; if (j->r.state == Invalid) { filermjob(j->file, j); } return r; } int filewrjobfull(File *f, job j) { int nl; fileaddjob(f, j); nl = strlen(j->tube->name); return filewrite(f, j, &nl, sizeof nl) && filewrite(f, j, j->tube->name, nl) && filewrite(f, j, &j->r, sizeof j->r) && filewrite(f, j, j->body, j->r.body_size); } void filewclose(File *f) { if (!f) return; if (!f->iswopen) return; if (f->free) { (void)ftruncate(f->fd, f->w->filesize - f->free); } close(f->fd); f->iswopen = 0; filedecref(f); } int fileinit(File *f, Wal *w, int n) { f->w = w; f->seq = n; f->path = fmtalloc("%s/binlog.%d", w->dir, n); return !!f->path; } // Adds f to the linked list in w, // updating w->tail and w->head as necessary. Wal* fileadd(File *f, Wal *w) { if (w->tail) { w->tail->next = f; } w->tail = f; if (!w->head) { w->head = f; } w->nfile++; return w; } beanstalkd-1.10/freebsd.c000066400000000000000000000000241237005101200153040ustar00rootroot00000000000000#include "darwin.c" beanstalkd-1.10/heap.c000066400000000000000000000035071237005101200146200ustar00rootroot00000000000000#include #include #include #include "dat.h" static void set(Heap *h, int k, void *x) { h->data[k] = x; h->rec(x, k); } static void swap(Heap *h, int a, int b) { void *tmp; tmp = h->data[a]; set(h, a, h->data[b]); set(h, b, tmp); } static int less(Heap *h, int a, int b) { return h->less(h->data[a], h->data[b]); } static void siftdown(Heap *h, int k) { for (;;) { int p = (k-1) / 2; /* parent */ if (k == 0 || less(h, p, k)) { return; } swap(h, k, p); k = p; } } static void siftup(Heap *h, int k) { for (;;) { int l, r, s; l = k*2 + 1; /* left child */ r = k*2 + 2; /* right child */ /* find the smallest of the three */ s = k; if (l < h->len && less(h, l, s)) s = l; if (r < h->len && less(h, r, s)) s = r; if (s == k) { return; /* satisfies the heap property */ } swap(h, k, s); k = s; } } // Heapinsert inserts x into heap h according to h->less. // It returns 1 on success, otherwise 0. int heapinsert(Heap *h, void *x) { int k; if (h->len == h->cap) { void **ndata; int ncap = (h->len+1) * 2; /* allocate twice what we need */ ndata = malloc(sizeof(void*) * ncap); if (!ndata) { return 0; } memcpy(ndata, h->data, sizeof(void*)*h->len); free(h->data); h->data = ndata; h->cap = ncap; } k = h->len; h->len++; set(h, k, x); siftdown(h, k); return 1; } void * heapremove(Heap *h, int k) { void *x; if (k >= h->len) { return 0; } x = h->data[k]; h->len--; set(h, k, h->data[h->len]); siftdown(h, k); siftup(h, k); h->rec(x, -1); return x; } beanstalkd-1.10/job.c000066400000000000000000000112771237005101200144600ustar00rootroot00000000000000#include #include #include #include "dat.h" static uint64 next_id = 1; static int cur_prime = 0; static job all_jobs_init[12289] = {0}; static job *all_jobs = all_jobs_init; static size_t all_jobs_cap = 12289; /* == primes[0] */ static size_t all_jobs_used = 0; static int hash_table_was_oom = 0; static void rehash(); static int _get_job_hash_index(uint64 job_id) { return job_id % all_jobs_cap; } static void store_job(job j) { int index = 0; index = _get_job_hash_index(j->r.id); j->ht_next = all_jobs[index]; all_jobs[index] = j; all_jobs_used++; /* accept a load factor of 4 */ if (all_jobs_used > (all_jobs_cap << 2)) rehash(); } static void rehash() { job *old = all_jobs; size_t old_cap = all_jobs_cap, old_used = all_jobs_used, i; if (cur_prime >= NUM_PRIMES) return; if (hash_table_was_oom) return; all_jobs_cap = primes[++cur_prime]; all_jobs = calloc(all_jobs_cap, sizeof(job)); if (!all_jobs) { twarnx("Failed to allocate %zu new hash buckets", all_jobs_cap); hash_table_was_oom = 1; --cur_prime; all_jobs = old; all_jobs_cap = old_cap; all_jobs_used = old_used; return; } all_jobs_used = 0; for (i = 0; i < old_cap; i++) { while (old[i]) { job j = old[i]; old[i] = j->ht_next; j->ht_next = NULL; store_job(j); } } if (old != all_jobs_init) { free(old); } } job job_find(uint64 job_id) { job jh = NULL; int index = _get_job_hash_index(job_id); for (jh = all_jobs[index]; jh && jh->r.id != job_id; jh = jh->ht_next); return jh; } job allocate_job(int body_size) { job j; j = malloc(sizeof(struct job) + body_size); if (!j) return twarnx("OOM"), (job) 0; memset(j, 0, sizeof(struct job)); j->r.created_at = nanoseconds(); j->r.body_size = body_size; j->next = j->prev = j; /* not in a linked list */ return j; } job make_job_with_id(uint pri, int64 delay, int64 ttr, int body_size, tube tube, uint64 id) { job j; j = allocate_job(body_size); if (!j) return twarnx("OOM"), (job) 0; if (id) { j->r.id = id; if (id >= next_id) next_id = id + 1; } else { j->r.id = next_id++; } j->r.pri = pri; j->r.delay = delay; j->r.ttr = ttr; store_job(j); TUBE_ASSIGN(j->tube, tube); return j; } static void job_hash_free(job j) { job *slot; slot = &all_jobs[_get_job_hash_index(j->r.id)]; while (*slot && *slot != j) slot = &(*slot)->ht_next; if (*slot) { *slot = (*slot)->ht_next; --all_jobs_used; } } void job_free(job j) { if (j) { TUBE_ASSIGN(j->tube, NULL); if (j->r.state != Copy) job_hash_free(j); } free(j); } void job_setheappos(void *j, int pos) { ((job)j)->heap_index = pos; } int job_pri_less(void *ax, void *bx) { job a = ax, b = bx; if (a->r.pri < b->r.pri) return 1; if (a->r.pri > b->r.pri) return 0; return a->r.id < b->r.id; } int job_delay_less(void *ax, void *bx) { job a = ax, b = bx; if (a->r.deadline_at < b->r.deadline_at) return 1; if (a->r.deadline_at > b->r.deadline_at) return 0; return a->r.id < b->r.id; } job job_copy(job j) { job n; if (!j) return NULL; n = malloc(sizeof(struct job) + j->r.body_size); if (!n) return twarnx("OOM"), (job) 0; memcpy(n, j, sizeof(struct job) + j->r.body_size); n->next = n->prev = n; /* not in a linked list */ n->file = NULL; /* copies do not have refcnt on the wal */ n->tube = 0; /* Don't use memcpy for the tube, which we must refcount. */ TUBE_ASSIGN(n->tube, j->tube); /* Mark this job as a copy so it can be appropriately freed later on */ n->r.state = Copy; return n; } const char * job_state(job j) { if (j->r.state == Ready) return "ready"; if (j->r.state == Reserved) return "reserved"; if (j->r.state == Buried) return "buried"; if (j->r.state == Delayed) return "delayed"; return "invalid"; } int job_list_any_p(job head) { return head->next != head || head->prev != head; } job job_remove(job j) { if (!j) return NULL; if (!job_list_any_p(j)) return NULL; /* not in a doubly-linked list */ j->next->prev = j->prev; j->prev->next = j->next; j->prev = j->next = j; return j; } void job_insert(job head, job j) { if (job_list_any_p(j)) return; /* already in a linked list */ j->prev = head->prev; j->next = head; head->prev->next = j; head->prev = j; } uint64 total_jobs() { return next_id - 1; } /* for unit tests */ size_t get_all_jobs_used() { return all_jobs_used; } beanstalkd-1.10/linux.c000066400000000000000000000031701237005101200150360ustar00rootroot00000000000000#define _XOPEN_SOURCE 600 #include #include #include #include #include #include "dat.h" #ifndef EPOLLRDHUP #define EPOLLRDHUP 0x2000 #endif static int epfd; /* Allocate disk space. * Expects fd's offset to be 0; may also reset fd's offset to 0. * Returns 0 on success, and a positive errno otherwise. */ int rawfalloc(int fd, int len) { return posix_fallocate(fd, 0, len); } int sockinit(void) { epfd = epoll_create(1); if (epfd == -1) { twarn("epoll_create"); return -1; } return 0; } int sockwant(Socket *s, int rw) { int op; struct epoll_event ev = {}; if (!s->added && !rw) { return 0; } else if (!s->added && rw) { s->added = 1; op = EPOLL_CTL_ADD; } else if (!rw) { op = EPOLL_CTL_DEL; } else { op = EPOLL_CTL_MOD; } switch (rw) { case 'r': ev.events = EPOLLIN; break; case 'w': ev.events = EPOLLOUT; break; } ev.events |= EPOLLRDHUP | EPOLLPRI; ev.data.ptr = s; return epoll_ctl(epfd, op, s->fd, &ev); } int socknext(Socket **s, int64 timeout) { int r; struct epoll_event ev; r = epoll_wait(epfd, &ev, 1, (int)(timeout/1000000)); if (r == -1 && errno != EINTR) { twarn("epoll_wait"); exit(1); } if (r > 0) { *s = ev.data.ptr; if (ev.events & (EPOLLHUP|EPOLLRDHUP)) { return 'h'; } else if (ev.events & EPOLLIN) { return 'r'; } else if (ev.events & EPOLLOUT) { return 'w'; } } return 0; } beanstalkd-1.10/main.c000066400000000000000000000040431237005101200146230ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include "dat.h" static void su(const char *user) { int r; struct passwd *pwent; errno = 0; pwent = getpwnam(user); if (errno) twarn("getpwnam(\"%s\")", user), exit(32); if (!pwent) twarnx("getpwnam(\"%s\"): no such user", user), exit(33); r = setgid(pwent->pw_gid); if (r == -1) twarn("setgid(%d \"%s\")", pwent->pw_gid, user), exit(34); r = setuid(pwent->pw_uid); if (r == -1) twarn("setuid(%d \"%s\")", pwent->pw_uid, user), exit(34); } static void set_sig_handlers() { int r; struct sigaction sa; sa.sa_handler = SIG_IGN; sa.sa_flags = 0; r = sigemptyset(&sa.sa_mask); if (r == -1) twarn("sigemptyset()"), exit(111); r = sigaction(SIGPIPE, &sa, 0); if (r == -1) twarn("sigaction(SIGPIPE)"), exit(111); sa.sa_handler = enter_drain_mode; r = sigaction(SIGUSR1, &sa, 0); if (r == -1) twarn("sigaction(SIGUSR1)"), exit(111); } int main(int argc, char **argv) { int r; struct job list = {}; progname = argv[0]; setlinebuf(stdout); optparse(&srv, argv+1); if (verbose) { printf("pid %d\n", getpid()); } r = make_server_socket(srv.addr, srv.port); if (r == -1) twarnx("make_server_socket()"), exit(111); srv.sock.fd = r; prot_init(); if (srv.user) su(srv.user); set_sig_handlers(); if (srv.wal.use) { // We want to make sure that only one beanstalkd tries // to use the wal directory at a time. So acquire a lock // now and never release it. if (!waldirlock(&srv.wal)) { twarnx("failed to lock wal dir %s", srv.wal.dir); exit(10); } list.prev = list.next = &list; walinit(&srv.wal, &list); r = prot_replay(&srv, &list); if (!r) { twarnx("failed to replay log"); return 1; } } srvserve(&srv); return 0; } beanstalkd-1.10/ms.c000066400000000000000000000031421237005101200143150ustar00rootroot00000000000000#include #include #include #include "dat.h" void ms_init(ms a, ms_event_fn oninsert, ms_event_fn onremove) { a->used = a->cap = a->last = 0; a->items = NULL; a->oninsert = oninsert; a->onremove = onremove; } static void grow(ms a) { void **nitems; size_t ncap = (a->cap << 1) ? : 1; nitems = malloc(ncap * sizeof(void *)); if (!nitems) return; memcpy(nitems, a->items, a->used * sizeof(void *)); free(a->items); a->items = nitems; a->cap = ncap; } int ms_append(ms a, void *item) { if (a->used >= a->cap) grow(a); if (a->used >= a->cap) return 0; a->items[a->used++] = item; if (a->oninsert) a->oninsert(a, item, a->used - 1); return 1; } static int ms_delete(ms a, size_t i) { void *item; if (i >= a->used) return 0; item = a->items[i]; a->items[i] = a->items[--a->used]; /* it has already been removed now */ if (a->onremove) a->onremove(a, item, i); return 1; } void ms_clear(ms a) { while (ms_delete(a, 0)); free(a->items); ms_init(a, a->oninsert, a->onremove); } int ms_remove(ms a, void *item) { size_t i; for (i = 0; i < a->used; i++) { if (a->items[i] == item) return ms_delete(a, i); } return 0; } int ms_contains(ms a, void *item) { size_t i; for (i = 0; i < a->used; i++) { if (a->items[i] == item) return 1; } return 0; } void * ms_take(ms a) { void *item; if (!a->used) return NULL; a->last = a->last % a->used; item = a->items[a->last]; ms_delete(a, a->last); ++a->last; return item; } beanstalkd-1.10/net.c000066400000000000000000000067561237005101200145020ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include "dat.h" #include "sd-daemon.h" int make_server_socket(char *host, char *port) { int fd = -1, flags, r; struct linger linger = {0, 0}; struct addrinfo *airoot, *ai, hints; /* See if we got a listen fd from systemd. If so, all socket options etc * are already set, so we check that the fd is a TCP listen socket and * return. */ r = sd_listen_fds(1); if (r < 0) { return twarn("sd_listen_fds"), -1; } if (r > 0) { if (r > 1) { twarnx("inherited more than one listen socket;" " ignoring all but the first"); } fd = SD_LISTEN_FDS_START; r = sd_is_socket_inet(fd, 0, SOCK_STREAM, 1, 0); if (r < 0) { errno = -r; twarn("sd_is_socket_inet"); return -1; } if (!r) { twarnx("inherited fd is not a TCP listen socket"); return -1; } return fd; } memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; r = getaddrinfo(host, port, &hints, &airoot); if (r == -1) return twarn("getaddrinfo()"), -1; for(ai = airoot; ai; ai = ai->ai_next) { fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); if (fd == -1) { twarn("socket()"); continue; } flags = fcntl(fd, F_GETFL, 0); if (flags < 0) { twarn("getting flags"); close(fd); continue; } r = fcntl(fd, F_SETFL, flags | O_NONBLOCK); if (r == -1) { twarn("setting O_NONBLOCK"); close(fd); continue; } flags = 1; r = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof flags); if (r == -1) { twarn("setting SO_REUSEADDR on fd %d", fd); close(fd); continue; } r = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &flags, sizeof flags); if (r == -1) { twarn("setting SO_KEEPALIVE on fd %d", fd); close(fd); continue; } r = setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger, sizeof linger); if (r == -1) { twarn("setting SO_LINGER on fd %d", fd); close(fd); continue; } r = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof flags); if (r == -1) { twarn("setting TCP_NODELAY on fd %d", fd); close(fd); continue; } if (verbose) { char hbuf[NI_MAXHOST], pbuf[NI_MAXSERV], *h = host, *p = port; r = getnameinfo(ai->ai_addr, ai->ai_addrlen, hbuf, sizeof hbuf, pbuf, sizeof pbuf, NI_NUMERICHOST|NI_NUMERICSERV); if (!r) { h = hbuf; p = pbuf; } if (ai->ai_family == AF_INET6) { printf("bind %d [%s]:%s\n", fd, h, p); } else { printf("bind %d %s:%s\n", fd, h, p); } } r = bind(fd, ai->ai_addr, ai->ai_addrlen); if (r == -1) { twarn("bind()"); close(fd); continue; } r = listen(fd, 1024); if (r == -1) { twarn("listen()"); close(fd); continue; } break; } freeaddrinfo(airoot); if(ai == NULL) fd = -1; return fd; } beanstalkd-1.10/pkg/000077500000000000000000000000001237005101200143135ustar00rootroot00000000000000beanstalkd-1.10/pkg/beanstalkd.spec.in000066400000000000000000000047371237005101200177170ustar00rootroot00000000000000%define beanstalkd_user beanstalkd %define beanstalkd_group %{beanstalkd_user} %define beanstalkd_home %{_localstatedir}/lib/beanstalkd %define beanstalkd_logdir %{_localstatedir}/log/beanstalkd Name: beanstalkd Version: @VERSION@ Release: 0%{?dist} Summary: A simple, fast workqueue service Group: System Environment/Daemons License: GPLv3+ URL: http://xph.us/software/%{name}/ Source0: http://xph.us/dist/%{name}/rel/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires(pre): %{_sbindir}/useradd Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig, /sbin/service Requires(postun): /sbin/service %description Beanstalk is a simple, fast workqueue service. Its interface is generic, but was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. %prep %setup -q if [ ! -e configure ]; then sh buildconf.sh fi %build %configure --disable-rpath --docdir=%{_defaultdocdir}/%{name}-%{version} make %{?_smp_mflags} %install rm -rf $RPM_BUILD_ROOT make install DESTDIR=$RPM_BUILD_ROOT %{__install} -p -D -m 0644 doc/%{name}.1 %{buildroot}%{_mandir}/man1/%{name}.1 %{__install} -p -D -m 0755 scripts/%{name}.init %{buildroot}%{_initrddir}/%{name} %{__install} -p -D -m 0644 scripts/%{name}.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} %clean rm -rf $RPM_BUILD_ROOT %pre %{_sbindir}/useradd -c "beanstalkd user" -s /bin/false -r -m -d %{beanstalkd_home} %{beanstalkd_user} 2>/dev/null || : %post /sbin/chkconfig --add %{name} %preun if [ $1 = 0 ]; then /sbin/service %{name} stop >/dev/null 2>&1 /sbin/chkconfig --del %{name} fi %postun if [ $1 -ge 1 ]; then /sbin/service %{name} condrestart > /dev/null 2>&1 || : fi %files %defattr(-,root,root,-) %doc %{_defaultdocdir}/%{name}-%{version}/protocol.txt %doc README COPYING doc/protocol.txt %{_initrddir}/%{name} %{_bindir}/%{name} %{_mandir}/man1/%{name}.1.gz %config(noreplace) %{_sysconfdir}/sysconfig/%{name} %changelog * Thu Oct 1 2009 Keith Rarick - 1.4-0 - Convert this file to an autoconf template - Tweak the summary and description * Sun Jan 4 2009 Ask Bjørn Hansen - 1.2-0 - 1.2-tobe - Use man page and .init/sysconfig scripts from .tar.gz * Sat Nov 22 2008 Jeremy Hinegardner - 1.1-1 - initial spec creation beanstalkd-1.10/pkg/bloghead.in000066400000000000000000000002661237005101200164140ustar00rootroot00000000000000--- layout: post title: Beanstalkd @VERSION@ Release Notes version: @VERSION@ dist: https://github.com/kr/beanstalkd/archive/v@VERSION@.tar.gz file: beanstalkd-@VERSION@.tar.gz --- beanstalkd-1.10/pkg/dist.sh000077500000000000000000000026121237005101200156160ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -o pipefail exp() { sed s/@VERSION@/$ver/ | sed s/@PARENT@/$prev/ } clean() { rm -f "$GIT_INDEX_FILE" } mkobj() { git hash-object -w --stdin } die() { echo >&2 "$@" exit 2 } ver=`./vers.sh` case $ver in *+*) die bad ver $ver ;; esac prev=`git describe --abbrev=0 --match=dev* --tags dev$ver^|sed s/^dev//` test -n "$prev" || die no prev ver test -f News || die no News export GIT_INDEX_FILE GIT_INDEX_FILE=`mktemp -t beanstalkd-dist-index` trap clean EXIT git read-tree dev$ver newsobj=`cat News pkg/newstail.in|exp|mkobj` versobj=`echo "printf '$ver'"|mkobj` specobj=`exp /dev/null parent=`git rev-parse --verify gh-pages` git read-tree $parent postobj=`(exp &2 "$@" exit 2 } addr=beanstalk-talk@googlegroups.com ver=`./vers.sh` case $ver in *+*) die bad ver $ver ;; esac (cat < Subject: [ANN] beanstalkd $ver end beanstalkd-1.10/pkg/newstail.in000066400000000000000000000007521237005101200164750ustar00rootroot00000000000000 Full list of changes (includes authorship information): Our Urls -------- Download the @VERSION@ tarball directly: Learn all about beanstalk: Talk about beanstalk development or use at: Bugs ---- Please report any bugs to: beanstalkd-1.10/primes.c000066400000000000000000000016211237005101200151750ustar00rootroot00000000000000#include size_t primes[] = { 12289, 24593, 49193, 98387, 196799, 393611, 787243, 1574491, 3148987, 6297979, 12595991, 25191989, 50383981, 100767977, 201535967, 403071937, 806143879, 1612287763, 3224575537UL, #if _LP64 6449151103, 12898302233, 25796604473, 51593208973, 103186417951, 206372835917, 412745671837, 825491343683, 1650982687391, 3301965374803, 6603930749621, 13207861499251, 26415722998507, 52831445997037, 105662891994103, 211325783988211, 422651567976461, 845303135952931, 1690606271905871, 3381212543811743, 6762425087623523, 13524850175247127, 27049700350494287, 54099400700988593, 108198801401977301, 216397602803954641, 432795205607909293, 865590411215818597, 1731180822431637217, #endif }; beanstalkd-1.10/prot.c000066400000000000000000001506521237005101200146730ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "dat.h" /* job body cannot be greater than this many bytes long */ size_t job_data_size_limit = JOB_DATA_SIZE_LIMIT_DEFAULT; #define NAME_CHARS \ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" \ "abcdefghijklmnopqrstuvwxyz" \ "0123456789-+/;.$_()" #define CMD_PUT "put " #define CMD_PEEKJOB "peek " #define CMD_PEEK_READY "peek-ready" #define CMD_PEEK_DELAYED "peek-delayed" #define CMD_PEEK_BURIED "peek-buried" #define CMD_RESERVE "reserve" #define CMD_RESERVE_TIMEOUT "reserve-with-timeout " #define CMD_DELETE "delete " #define CMD_RELEASE "release " #define CMD_BURY "bury " #define CMD_KICK "kick " #define CMD_JOBKICK "kick-job " #define CMD_TOUCH "touch " #define CMD_STATS "stats" #define CMD_JOBSTATS "stats-job " #define CMD_USE "use " #define CMD_WATCH "watch " #define CMD_IGNORE "ignore " #define CMD_LIST_TUBES "list-tubes" #define CMD_LIST_TUBE_USED "list-tube-used" #define CMD_LIST_TUBES_WATCHED "list-tubes-watched" #define CMD_STATS_TUBE "stats-tube " #define CMD_QUIT "quit" #define CMD_PAUSE_TUBE "pause-tube" #define CONSTSTRLEN(m) (sizeof(m) - 1) #define CMD_PEEK_READY_LEN CONSTSTRLEN(CMD_PEEK_READY) #define CMD_PEEK_DELAYED_LEN CONSTSTRLEN(CMD_PEEK_DELAYED) #define CMD_PEEK_BURIED_LEN CONSTSTRLEN(CMD_PEEK_BURIED) #define CMD_PEEKJOB_LEN CONSTSTRLEN(CMD_PEEKJOB) #define CMD_RESERVE_LEN CONSTSTRLEN(CMD_RESERVE) #define CMD_RESERVE_TIMEOUT_LEN CONSTSTRLEN(CMD_RESERVE_TIMEOUT) #define CMD_DELETE_LEN CONSTSTRLEN(CMD_DELETE) #define CMD_RELEASE_LEN CONSTSTRLEN(CMD_RELEASE) #define CMD_BURY_LEN CONSTSTRLEN(CMD_BURY) #define CMD_KICK_LEN CONSTSTRLEN(CMD_KICK) #define CMD_JOBKICK_LEN CONSTSTRLEN(CMD_JOBKICK) #define CMD_TOUCH_LEN CONSTSTRLEN(CMD_TOUCH) #define CMD_STATS_LEN CONSTSTRLEN(CMD_STATS) #define CMD_JOBSTATS_LEN CONSTSTRLEN(CMD_JOBSTATS) #define CMD_USE_LEN CONSTSTRLEN(CMD_USE) #define CMD_WATCH_LEN CONSTSTRLEN(CMD_WATCH) #define CMD_IGNORE_LEN CONSTSTRLEN(CMD_IGNORE) #define CMD_LIST_TUBES_LEN CONSTSTRLEN(CMD_LIST_TUBES) #define CMD_LIST_TUBE_USED_LEN CONSTSTRLEN(CMD_LIST_TUBE_USED) #define CMD_LIST_TUBES_WATCHED_LEN CONSTSTRLEN(CMD_LIST_TUBES_WATCHED) #define CMD_STATS_TUBE_LEN CONSTSTRLEN(CMD_STATS_TUBE) #define CMD_PAUSE_TUBE_LEN CONSTSTRLEN(CMD_PAUSE_TUBE) #define MSG_FOUND "FOUND" #define MSG_NOTFOUND "NOT_FOUND\r\n" #define MSG_RESERVED "RESERVED" #define MSG_DEADLINE_SOON "DEADLINE_SOON\r\n" #define MSG_TIMED_OUT "TIMED_OUT\r\n" #define MSG_DELETED "DELETED\r\n" #define MSG_RELEASED "RELEASED\r\n" #define MSG_BURIED "BURIED\r\n" #define MSG_KICKED "KICKED\r\n" #define MSG_TOUCHED "TOUCHED\r\n" #define MSG_BURIED_FMT "BURIED %"PRIu64"\r\n" #define MSG_INSERTED_FMT "INSERTED %"PRIu64"\r\n" #define MSG_NOT_IGNORED "NOT_IGNORED\r\n" #define MSG_NOTFOUND_LEN CONSTSTRLEN(MSG_NOTFOUND) #define MSG_DELETED_LEN CONSTSTRLEN(MSG_DELETED) #define MSG_TOUCHED_LEN CONSTSTRLEN(MSG_TOUCHED) #define MSG_RELEASED_LEN CONSTSTRLEN(MSG_RELEASED) #define MSG_BURIED_LEN CONSTSTRLEN(MSG_BURIED) #define MSG_KICKED_LEN CONSTSTRLEN(MSG_KICKED) #define MSG_NOT_IGNORED_LEN CONSTSTRLEN(MSG_NOT_IGNORED) #define MSG_OUT_OF_MEMORY "OUT_OF_MEMORY\r\n" #define MSG_INTERNAL_ERROR "INTERNAL_ERROR\r\n" #define MSG_DRAINING "DRAINING\r\n" #define MSG_BAD_FORMAT "BAD_FORMAT\r\n" #define MSG_UNKNOWN_COMMAND "UNKNOWN_COMMAND\r\n" #define MSG_EXPECTED_CRLF "EXPECTED_CRLF\r\n" #define MSG_JOB_TOO_BIG "JOB_TOO_BIG\r\n" #define STATE_WANTCOMMAND 0 #define STATE_WANTDATA 1 #define STATE_SENDJOB 2 #define STATE_SENDWORD 3 #define STATE_WAIT 4 #define STATE_BITBUCKET 5 #define STATE_CLOSE 6 #define OP_UNKNOWN 0 #define OP_PUT 1 #define OP_PEEKJOB 2 #define OP_RESERVE 3 #define OP_DELETE 4 #define OP_RELEASE 5 #define OP_BURY 6 #define OP_KICK 7 #define OP_STATS 8 #define OP_JOBSTATS 9 #define OP_PEEK_BURIED 10 #define OP_USE 11 #define OP_WATCH 12 #define OP_IGNORE 13 #define OP_LIST_TUBES 14 #define OP_LIST_TUBE_USED 15 #define OP_LIST_TUBES_WATCHED 16 #define OP_STATS_TUBE 17 #define OP_PEEK_READY 18 #define OP_PEEK_DELAYED 19 #define OP_RESERVE_TIMEOUT 20 #define OP_TOUCH 21 #define OP_QUIT 22 #define OP_PAUSE_TUBE 23 #define OP_JOBKICK 24 #define TOTAL_OPS 25 #define STATS_FMT "---\n" \ "current-jobs-urgent: %u\n" \ "current-jobs-ready: %u\n" \ "current-jobs-reserved: %u\n" \ "current-jobs-delayed: %u\n" \ "current-jobs-buried: %u\n" \ "cmd-put: %" PRIu64 "\n" \ "cmd-peek: %" PRIu64 "\n" \ "cmd-peek-ready: %" PRIu64 "\n" \ "cmd-peek-delayed: %" PRIu64 "\n" \ "cmd-peek-buried: %" PRIu64 "\n" \ "cmd-reserve: %" PRIu64 "\n" \ "cmd-reserve-with-timeout: %" PRIu64 "\n" \ "cmd-delete: %" PRIu64 "\n" \ "cmd-release: %" PRIu64 "\n" \ "cmd-use: %" PRIu64 "\n" \ "cmd-watch: %" PRIu64 "\n" \ "cmd-ignore: %" PRIu64 "\n" \ "cmd-bury: %" PRIu64 "\n" \ "cmd-kick: %" PRIu64 "\n" \ "cmd-touch: %" PRIu64 "\n" \ "cmd-stats: %" PRIu64 "\n" \ "cmd-stats-job: %" PRIu64 "\n" \ "cmd-stats-tube: %" PRIu64 "\n" \ "cmd-list-tubes: %" PRIu64 "\n" \ "cmd-list-tube-used: %" PRIu64 "\n" \ "cmd-list-tubes-watched: %" PRIu64 "\n" \ "cmd-pause-tube: %" PRIu64 "\n" \ "job-timeouts: %" PRIu64 "\n" \ "total-jobs: %" PRIu64 "\n" \ "max-job-size: %zu\n" \ "current-tubes: %zu\n" \ "current-connections: %u\n" \ "current-producers: %u\n" \ "current-workers: %u\n" \ "current-waiting: %u\n" \ "total-connections: %u\n" \ "pid: %ld\n" \ "version: %s\n" \ "rusage-utime: %d.%06d\n" \ "rusage-stime: %d.%06d\n" \ "uptime: %u\n" \ "binlog-oldest-index: %d\n" \ "binlog-current-index: %d\n" \ "binlog-records-migrated: %" PRId64 "\n" \ "binlog-records-written: %" PRId64 "\n" \ "binlog-max-size: %d\n" \ "id: %s\n" \ "hostname: %s\n" \ "\r\n" #define STATS_TUBE_FMT "---\n" \ "name: %s\n" \ "current-jobs-urgent: %u\n" \ "current-jobs-ready: %u\n" \ "current-jobs-reserved: %u\n" \ "current-jobs-delayed: %u\n" \ "current-jobs-buried: %u\n" \ "total-jobs: %" PRIu64 "\n" \ "current-using: %u\n" \ "current-watching: %u\n" \ "current-waiting: %u\n" \ "cmd-delete: %" PRIu64 "\n" \ "cmd-pause-tube: %u\n" \ "pause: %" PRIu64 "\n" \ "pause-time-left: %" PRId64 "\n" \ "\r\n" #define STATS_JOB_FMT "---\n" \ "id: %" PRIu64 "\n" \ "tube: %s\n" \ "state: %s\n" \ "pri: %u\n" \ "age: %" PRId64 "\n" \ "delay: %" PRId64 "\n" \ "ttr: %" PRId64 "\n" \ "time-left: %" PRId64 "\n" \ "file: %d\n" \ "reserves: %u\n" \ "timeouts: %u\n" \ "releases: %u\n" \ "buries: %u\n" \ "kicks: %u\n" \ "\r\n" /* this number is pretty arbitrary */ #define BUCKET_BUF_SIZE 1024 static char bucket[BUCKET_BUF_SIZE]; static uint ready_ct = 0; static struct stats global_stat = {0, 0, 0, 0, 0}; static tube default_tube; static int drain_mode = 0; static int64 started_at; enum { NumIdBytes = 8 }; static char id[NumIdBytes * 2 + 1]; // hex-encoded len of NumIdBytes static struct utsname node_info; static uint64 op_ct[TOTAL_OPS], timeout_ct = 0; static Conn *dirty; static const char * op_names[] = { "", CMD_PUT, CMD_PEEKJOB, CMD_RESERVE, CMD_DELETE, CMD_RELEASE, CMD_BURY, CMD_KICK, CMD_STATS, CMD_JOBSTATS, CMD_PEEK_BURIED, CMD_USE, CMD_WATCH, CMD_IGNORE, CMD_LIST_TUBES, CMD_LIST_TUBE_USED, CMD_LIST_TUBES_WATCHED, CMD_STATS_TUBE, CMD_PEEK_READY, CMD_PEEK_DELAYED, CMD_RESERVE_TIMEOUT, CMD_TOUCH, CMD_QUIT, CMD_PAUSE_TUBE, CMD_JOBKICK, }; static job remove_buried_job(job j); static int buried_job_p(tube t) { return job_list_any_p(&t->buried); } static void reply(Conn *c, char *line, int len, int state) { if (!c) return; connwant(c, 'w'); c->next = dirty; dirty = c; c->reply = line; c->reply_len = len; c->reply_sent = 0; c->state = state; if (verbose >= 2) { printf(">%d reply %.*s\n", c->sock.fd, len-2, line); } } static void protrmdirty(Conn *c) { Conn *x, *newdirty = NULL; while (dirty) { x = dirty; dirty = dirty->next; x->next = NULL; if (x != c) { x->next = newdirty; newdirty = x; } } dirty = newdirty; } #define reply_msg(c,m) reply((c),(m),CONSTSTRLEN(m),STATE_SENDWORD) #define reply_serr(c,e) (twarnx("server error: %s",(e)),\ reply_msg((c),(e))) static void reply_line(Conn*, int, const char*, ...) __attribute__((format(printf, 3, 4))); static void reply_line(Conn *c, int state, const char *fmt, ...) { int r; va_list ap; va_start(ap, fmt); r = vsnprintf(c->reply_buf, LINE_BUF_SIZE, fmt, ap); va_end(ap); /* Make sure the buffer was big enough. If not, we have a bug. */ if (r >= LINE_BUF_SIZE) return reply_serr(c, MSG_INTERNAL_ERROR); return reply(c, c->reply_buf, r, state); } static void reply_job(Conn *c, job j, const char *word) { /* tell this connection which job to send */ c->out_job = j; c->out_job_sent = 0; return reply_line(c, STATE_SENDJOB, "%s %"PRIu64" %u\r\n", word, j->r.id, j->r.body_size - 2); } Conn * remove_waiting_conn(Conn *c) { tube t; size_t i; if (!conn_waiting(c)) return NULL; c->type &= ~CONN_TYPE_WAITING; global_stat.waiting_ct--; for (i = 0; i < c->watch.used; i++) { t = c->watch.items[i]; t->stat.waiting_ct--; ms_remove(&t->waiting, c); } return c; } static void reserve_job(Conn *c, job j) { j->r.deadline_at = nanoseconds() + j->r.ttr; global_stat.reserved_ct++; /* stats */ j->tube->stat.reserved_ct++; j->r.reserve_ct++; j->r.state = Reserved; job_insert(&c->reserved_jobs, j); j->reserver = c; c->pending_timeout = -1; if (c->soonest_job && j->r.deadline_at < c->soonest_job->r.deadline_at) { c->soonest_job = j; } return reply_job(c, j, MSG_RESERVED); } static job next_eligible_job(int64 now) { tube t; size_t i; job j = NULL, candidate; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; if (t->pause) { if (t->deadline_at > now) continue; t->pause = 0; } if (t->waiting.used && t->ready.len) { candidate = t->ready.data[0]; if (!j || job_pri_less(candidate, j)) { j = candidate; } } } return j; } static void process_queue() { job j; int64 now = nanoseconds(); while ((j = next_eligible_job(now))) { heapremove(&j->tube->ready, j->heap_index); ready_ct--; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct--; j->tube->stat.urgent_ct--; } reserve_job(remove_waiting_conn(ms_take(&j->tube->waiting)), j); } } static job delay_q_peek() { int i; tube t; job j = NULL, nj; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; if (t->delay.len == 0) { continue; } nj = t->delay.data[0]; if (!j || nj->r.deadline_at < j->r.deadline_at) j = nj; } return j; } static int enqueue_job(Server *s, job j, int64 delay, char update_store) { int r; j->reserver = NULL; if (delay) { j->r.deadline_at = nanoseconds() + delay; r = heapinsert(&j->tube->delay, j); if (!r) return 0; j->r.state = Delayed; } else { r = heapinsert(&j->tube->ready, j); if (!r) return 0; j->r.state = Ready; ready_ct++; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct++; j->tube->stat.urgent_ct++; } } if (update_store) { if (!walwrite(&s->wal, j)) { return 0; } walmaint(&s->wal); } process_queue(); return 1; } static int bury_job(Server *s, job j, char update_store) { int z; if (update_store) { z = walresvupdate(&s->wal, j); if (!z) return 0; j->walresv += z; } job_insert(&j->tube->buried, j); global_stat.buried_ct++; j->tube->stat.buried_ct++; j->r.state = Buried; j->reserver = NULL; j->r.bury_ct++; if (update_store) { if (!walwrite(&s->wal, j)) { return 0; } walmaint(&s->wal); } return 1; } void enqueue_reserved_jobs(Conn *c) { int r; job j; while (job_list_any_p(&c->reserved_jobs)) { j = job_remove(c->reserved_jobs.next); r = enqueue_job(c->srv, j, 0, 0); if (r < 1) bury_job(c->srv, j, 0); global_stat.reserved_ct--; j->tube->stat.reserved_ct--; c->soonest_job = NULL; } } static job delay_q_take() { job j = delay_q_peek(); if (!j) { return 0; } heapremove(&j->tube->delay, j->heap_index); return j; } static int kick_buried_job(Server *s, job j) { int r; int z; z = walresvupdate(&s->wal, j); if (!z) return 0; j->walresv += z; remove_buried_job(j); j->r.kick_ct++; r = enqueue_job(s, j, 0, 1); if (r == 1) return 1; /* ready queue is full, so bury it */ bury_job(s, j, 0); return 0; } static uint get_delayed_job_ct() { tube t; size_t i; uint count = 0; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; count += t->delay.len; } return count; } static int kick_delayed_job(Server *s, job j) { int r; int z; z = walresvupdate(&s->wal, j); if (!z) return 0; j->walresv += z; heapremove(&j->tube->delay, j->heap_index); j->r.kick_ct++; r = enqueue_job(s, j, 0, 1); if (r == 1) return 1; /* ready queue is full, so delay it again */ r = enqueue_job(s, j, j->r.delay, 0); if (r == 1) return 0; /* last resort */ bury_job(s, j, 0); return 0; } /* return the number of jobs successfully kicked */ static uint kick_buried_jobs(Server *s, tube t, uint n) { uint i; for (i = 0; (i < n) && buried_job_p(t); ++i) { kick_buried_job(s, t->buried.next); } return i; } /* return the number of jobs successfully kicked */ static uint kick_delayed_jobs(Server *s, tube t, uint n) { uint i; for (i = 0; (i < n) && (t->delay.len > 0); ++i) { kick_delayed_job(s, (job)t->delay.data[0]); } return i; } static uint kick_jobs(Server *s, tube t, uint n) { if (buried_job_p(t)) return kick_buried_jobs(s, t, n); return kick_delayed_jobs(s, t, n); } static job remove_buried_job(job j) { if (!j || j->r.state != Buried) return NULL; j = job_remove(j); if (j) { global_stat.buried_ct--; j->tube->stat.buried_ct--; } return j; } static job remove_delayed_job(job j) { if (!j || j->r.state != Delayed) return NULL; heapremove(&j->tube->delay, j->heap_index); return j; } static job remove_ready_job(job j) { if (!j || j->r.state != Ready) return NULL; heapremove(&j->tube->ready, j->heap_index); ready_ct--; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct--; j->tube->stat.urgent_ct--; } return j; } static void enqueue_waiting_conn(Conn *c) { tube t; size_t i; global_stat.waiting_ct++; c->type |= CONN_TYPE_WAITING; for (i = 0; i < c->watch.used; i++) { t = c->watch.items[i]; t->stat.waiting_ct++; ms_append(&t->waiting, c); } } static job find_reserved_job_in_conn(Conn *c, job j) { return (j && j->reserver == c && j->r.state == Reserved) ? j : NULL; } static job touch_job(Conn *c, job j) { j = find_reserved_job_in_conn(c, j); if (j) { j->r.deadline_at = nanoseconds() + j->r.ttr; c->soonest_job = NULL; } return j; } static job peek_job(uint64 id) { return job_find(id); } static void check_err(Conn *c, const char *s) { if (errno == EAGAIN) return; if (errno == EINTR) return; if (errno == EWOULDBLOCK) return; twarn("%s", s); c->state = STATE_CLOSE; return; } /* Scan the given string for the sequence "\r\n" and return the line length. * Always returns at least 2 if a match is found. Returns 0 if no match. */ static int scan_line_end(const char *s, int size) { char *match; match = memchr(s, '\r', size - 1); if (!match) return 0; /* this is safe because we only scan size - 1 chars above */ if (match[1] == '\n') return match - s + 2; return 0; } static int cmd_len(Conn *c) { return scan_line_end(c->cmd, c->cmd_read); } /* parse the command line */ static int which_cmd(Conn *c) { #define TEST_CMD(s,c,o) if (strncmp((s), (c), CONSTSTRLEN(c)) == 0) return (o); TEST_CMD(c->cmd, CMD_PUT, OP_PUT); TEST_CMD(c->cmd, CMD_PEEKJOB, OP_PEEKJOB); TEST_CMD(c->cmd, CMD_PEEK_READY, OP_PEEK_READY); TEST_CMD(c->cmd, CMD_PEEK_DELAYED, OP_PEEK_DELAYED); TEST_CMD(c->cmd, CMD_PEEK_BURIED, OP_PEEK_BURIED); TEST_CMD(c->cmd, CMD_RESERVE_TIMEOUT, OP_RESERVE_TIMEOUT); TEST_CMD(c->cmd, CMD_RESERVE, OP_RESERVE); TEST_CMD(c->cmd, CMD_DELETE, OP_DELETE); TEST_CMD(c->cmd, CMD_RELEASE, OP_RELEASE); TEST_CMD(c->cmd, CMD_BURY, OP_BURY); TEST_CMD(c->cmd, CMD_KICK, OP_KICK); TEST_CMD(c->cmd, CMD_JOBKICK, OP_JOBKICK); TEST_CMD(c->cmd, CMD_TOUCH, OP_TOUCH); TEST_CMD(c->cmd, CMD_JOBSTATS, OP_JOBSTATS); TEST_CMD(c->cmd, CMD_STATS_TUBE, OP_STATS_TUBE); TEST_CMD(c->cmd, CMD_STATS, OP_STATS); TEST_CMD(c->cmd, CMD_USE, OP_USE); TEST_CMD(c->cmd, CMD_WATCH, OP_WATCH); TEST_CMD(c->cmd, CMD_IGNORE, OP_IGNORE); TEST_CMD(c->cmd, CMD_LIST_TUBES_WATCHED, OP_LIST_TUBES_WATCHED); TEST_CMD(c->cmd, CMD_LIST_TUBE_USED, OP_LIST_TUBE_USED); TEST_CMD(c->cmd, CMD_LIST_TUBES, OP_LIST_TUBES); TEST_CMD(c->cmd, CMD_QUIT, OP_QUIT); TEST_CMD(c->cmd, CMD_PAUSE_TUBE, OP_PAUSE_TUBE); return OP_UNKNOWN; } /* Copy up to body_size trailing bytes into the job, then the rest into the cmd * buffer. If c->in_job exists, this assumes that c->in_job->body is empty. * This function is idempotent(). */ static void fill_extra_data(Conn *c) { int extra_bytes, job_data_bytes = 0, cmd_bytes; if (!c->sock.fd) return; /* the connection was closed */ if (!c->cmd_len) return; /* we don't have a complete command */ /* how many extra bytes did we read? */ extra_bytes = c->cmd_read - c->cmd_len; /* how many bytes should we put into the job body? */ if (c->in_job) { job_data_bytes = min(extra_bytes, c->in_job->r.body_size); memcpy(c->in_job->body, c->cmd + c->cmd_len, job_data_bytes); c->in_job_read = job_data_bytes; } else if (c->in_job_read) { /* we are in bit-bucket mode, throwing away data */ job_data_bytes = min(extra_bytes, c->in_job_read); c->in_job_read -= job_data_bytes; } /* how many bytes are left to go into the future cmd? */ cmd_bytes = extra_bytes - job_data_bytes; memmove(c->cmd, c->cmd + c->cmd_len + job_data_bytes, cmd_bytes); c->cmd_read = cmd_bytes; c->cmd_len = 0; /* we no longer know the length of the new command */ } static void _skip(Conn *c, int n, char *line, int len) { /* Invert the meaning of in_job_read while throwing away data -- it * counts the bytes that remain to be thrown away. */ c->in_job = 0; c->in_job_read = n; fill_extra_data(c); if (c->in_job_read == 0) return reply(c, line, len, STATE_SENDWORD); c->reply = line; c->reply_len = len; c->reply_sent = 0; c->state = STATE_BITBUCKET; return; } #define skip(c,n,m) (_skip(c,n,m,CONSTSTRLEN(m))) static void enqueue_incoming_job(Conn *c) { int r; job j = c->in_job; c->in_job = NULL; /* the connection no longer owns this job */ c->in_job_read = 0; /* check if the trailer is present and correct */ if (memcmp(j->body + j->r.body_size - 2, "\r\n", 2)) { job_free(j); return reply_msg(c, MSG_EXPECTED_CRLF); } if (verbose >= 2) { printf("<%d job %"PRIu64"\n", c->sock.fd, j->r.id); } if (drain_mode) { job_free(j); return reply_serr(c, MSG_DRAINING); } if (j->walresv) return reply_serr(c, MSG_INTERNAL_ERROR); j->walresv = walresvput(&c->srv->wal, j); if (!j->walresv) return reply_serr(c, MSG_OUT_OF_MEMORY); /* we have a complete job, so let's stick it in the pqueue */ r = enqueue_job(c->srv, j, j->r.delay, 1); if (r < 0) return reply_serr(c, MSG_INTERNAL_ERROR); global_stat.total_jobs_ct++; j->tube->stat.total_jobs_ct++; if (r == 1) return reply_line(c, STATE_SENDWORD, MSG_INSERTED_FMT, j->r.id); /* out of memory trying to grow the queue, so it gets buried */ bury_job(c->srv, j, 0); reply_line(c, STATE_SENDWORD, MSG_BURIED_FMT, j->r.id); } static uint uptime() { return (nanoseconds() - started_at) / 1000000000; } static int fmt_stats(char *buf, size_t size, void *x) { int whead = 0, wcur = 0; Server *srv; struct rusage ru = {{0, 0}, {0, 0}}; srv = x; if (srv->wal.head) { whead = srv->wal.head->seq; } if (srv->wal.cur) { wcur = srv->wal.cur->seq; } getrusage(RUSAGE_SELF, &ru); /* don't care if it fails */ return snprintf(buf, size, STATS_FMT, global_stat.urgent_ct, ready_ct, global_stat.reserved_ct, get_delayed_job_ct(), global_stat.buried_ct, op_ct[OP_PUT], op_ct[OP_PEEKJOB], op_ct[OP_PEEK_READY], op_ct[OP_PEEK_DELAYED], op_ct[OP_PEEK_BURIED], op_ct[OP_RESERVE], op_ct[OP_RESERVE_TIMEOUT], op_ct[OP_DELETE], op_ct[OP_RELEASE], op_ct[OP_USE], op_ct[OP_WATCH], op_ct[OP_IGNORE], op_ct[OP_BURY], op_ct[OP_KICK], op_ct[OP_TOUCH], op_ct[OP_STATS], op_ct[OP_JOBSTATS], op_ct[OP_STATS_TUBE], op_ct[OP_LIST_TUBES], op_ct[OP_LIST_TUBE_USED], op_ct[OP_LIST_TUBES_WATCHED], op_ct[OP_PAUSE_TUBE], timeout_ct, global_stat.total_jobs_ct, job_data_size_limit, tubes.used, count_cur_conns(), count_cur_producers(), count_cur_workers(), global_stat.waiting_ct, count_tot_conns(), (long) getpid(), version, (int) ru.ru_utime.tv_sec, (int) ru.ru_utime.tv_usec, (int) ru.ru_stime.tv_sec, (int) ru.ru_stime.tv_usec, uptime(), whead, wcur, srv->wal.nmig, srv->wal.nrec, srv->wal.filesize, id, node_info.nodename); } /* Read a priority value from the given buffer and place it in pri. * Update end to point to the address after the last character consumed. * Pri and end can be NULL. If they are both NULL, read_pri() will do the * conversion and return the status code but not update any values. This is an * easy way to check for errors. * If end is NULL, read_pri will also check that the entire input string was * consumed and return an error code otherwise. * Return 0 on success, or nonzero on failure. * If a failure occurs, pri and end are not modified. */ static int read_pri(uint *pri, const char *buf, char **end) { char *tend; uint tpri; errno = 0; while (buf[0] == ' ') buf++; if (buf[0] < '0' || '9' < buf[0]) return -1; tpri = strtoul(buf, &tend, 10); if (tend == buf) return -1; if (errno && errno != ERANGE) return -1; if (!end && tend[0] != '\0') return -1; if (pri) *pri = tpri; if (end) *end = tend; return 0; } /* Read a delay value from the given buffer and place it in delay. * The interface and behavior are analogous to read_pri(). */ static int read_delay(int64 *delay, const char *buf, char **end) { int r; uint delay_sec; r = read_pri(&delay_sec, buf, end); if (r) return r; *delay = ((int64) delay_sec) * 1000000000; return 0; } /* Read a timeout value from the given buffer and place it in ttr. * The interface and behavior are the same as in read_delay(). */ static int read_ttr(int64 *ttr, const char *buf, char **end) { return read_delay(ttr, buf, end); } /* Read a tube name from the given buffer moving the buffer to the name start */ static int read_tube_name(char **tubename, char *buf, char **end) { size_t len; while (buf[0] == ' ') buf++; len = strspn(buf, NAME_CHARS); if (len == 0) return -1; if (tubename) *tubename = buf; if (end) *end = buf + len; return 0; } static void wait_for_job(Conn *c, int timeout) { c->state = STATE_WAIT; enqueue_waiting_conn(c); /* Set the pending timeout to the requested timeout amount */ c->pending_timeout = timeout; connwant(c, 'h'); // only care if they hang up c->next = dirty; dirty = c; } typedef int(*fmt_fn)(char *, size_t, void *); static void do_stats(Conn *c, fmt_fn fmt, void *data) { int r, stats_len; /* first, measure how big a buffer we will need */ stats_len = fmt(NULL, 0, data) + 16; c->out_job = allocate_job(stats_len); /* fake job to hold stats data */ if (!c->out_job) return reply_serr(c, MSG_OUT_OF_MEMORY); /* Mark this job as a copy so it can be appropriately freed later on */ c->out_job->r.state = Copy; /* now actually format the stats data */ r = fmt(c->out_job->body, stats_len, data); /* and set the actual body size */ c->out_job->r.body_size = r; if (r > stats_len) return reply_serr(c, MSG_INTERNAL_ERROR); c->out_job_sent = 0; return reply_line(c, STATE_SENDJOB, "OK %d\r\n", r - 2); } static void do_list_tubes(Conn *c, ms l) { char *buf; tube t; size_t i, resp_z; /* first, measure how big a buffer we will need */ resp_z = 6; /* initial "---\n" and final "\r\n" */ for (i = 0; i < l->used; i++) { t = l->items[i]; resp_z += 3 + strlen(t->name); /* including "- " and "\n" */ } c->out_job = allocate_job(resp_z); /* fake job to hold response data */ if (!c->out_job) return reply_serr(c, MSG_OUT_OF_MEMORY); /* Mark this job as a copy so it can be appropriately freed later on */ c->out_job->r.state = Copy; /* now actually format the response */ buf = c->out_job->body; buf += snprintf(buf, 5, "---\n"); for (i = 0; i < l->used; i++) { t = l->items[i]; buf += snprintf(buf, 4 + strlen(t->name), "- %s\n", t->name); } buf[0] = '\r'; buf[1] = '\n'; c->out_job_sent = 0; return reply_line(c, STATE_SENDJOB, "OK %zu\r\n", resp_z - 2); } static int fmt_job_stats(char *buf, size_t size, job j) { int64 t; int64 time_left; int file = 0; t = nanoseconds(); if (j->r.state == Reserved || j->r.state == Delayed) { time_left = (j->r.deadline_at - t) / 1000000000; } else { time_left = 0; } if (j->file) { file = j->file->seq; } return snprintf(buf, size, STATS_JOB_FMT, j->r.id, j->tube->name, job_state(j), j->r.pri, (t - j->r.created_at) / 1000000000, j->r.delay / 1000000000, j->r.ttr / 1000000000, time_left, file, j->r.reserve_ct, j->r.timeout_ct, j->r.release_ct, j->r.bury_ct, j->r.kick_ct); } static int fmt_stats_tube(char *buf, size_t size, tube t) { uint64 time_left; if (t->pause > 0) { time_left = (t->deadline_at - nanoseconds()) / 1000000000; } else { time_left = 0; } return snprintf(buf, size, STATS_TUBE_FMT, t->name, t->stat.urgent_ct, t->ready.len, t->stat.reserved_ct, t->delay.len, t->stat.buried_ct, t->stat.total_jobs_ct, t->using_ct, t->watching_ct, t->stat.waiting_ct, t->stat.total_delete_ct, t->stat.pause_ct, t->pause / 1000000000, time_left); } static void maybe_enqueue_incoming_job(Conn *c) { job j = c->in_job; /* do we have a complete job? */ if (c->in_job_read == j->r.body_size) return enqueue_incoming_job(c); /* otherwise we have incomplete data, so just keep waiting */ c->state = STATE_WANTDATA; } /* j can be NULL */ static job remove_this_reserved_job(Conn *c, job j) { j = job_remove(j); if (j) { global_stat.reserved_ct--; j->tube->stat.reserved_ct--; j->reserver = NULL; } c->soonest_job = NULL; return j; } static job remove_reserved_job(Conn *c, job j) { return remove_this_reserved_job(c, find_reserved_job_in_conn(c, j)); } static int name_is_ok(const char *name, size_t max) { size_t len = strlen(name); return len > 0 && len <= max && strspn(name, NAME_CHARS) == len && name[0] != '-'; } void prot_remove_tube(tube t) { ms_remove(&tubes, t); } static void dispatch_cmd(Conn *c) { int r, i, timeout = -1; int z; uint count; job j = 0; byte type; char *size_buf, *delay_buf, *ttr_buf, *pri_buf, *end_buf, *name; uint pri, body_size; int64 delay, ttr; uint64 id; tube t = NULL; /* NUL-terminate this string so we can use strtol and friends */ c->cmd[c->cmd_len - 2] = '\0'; /* check for possible maliciousness */ if (strlen(c->cmd) != c->cmd_len - 2) { return reply_msg(c, MSG_BAD_FORMAT); } type = which_cmd(c); if (verbose >= 2) { printf("<%d command %s\n", c->sock.fd, op_names[type]); } switch (type) { case OP_PUT: r = read_pri(&pri, c->cmd + 4, &delay_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_delay(&delay, delay_buf, &ttr_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_ttr(&ttr, ttr_buf, &size_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); errno = 0; body_size = strtoul(size_buf, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; if (body_size > job_data_size_limit) { /* throw away the job body and respond with JOB_TOO_BIG */ return skip(c, body_size + 2, MSG_JOB_TOO_BIG); } /* don't allow trailing garbage */ if (end_buf[0] != '\0') return reply_msg(c, MSG_BAD_FORMAT); connsetproducer(c); if (ttr < 1000000000) { ttr = 1000000000; } c->in_job = make_job(pri, delay, ttr, body_size + 2, c->use); /* OOM? */ if (!c->in_job) { /* throw away the job body and respond with OUT_OF_MEMORY */ twarnx("server error: " MSG_OUT_OF_MEMORY); return skip(c, body_size + 2, MSG_OUT_OF_MEMORY); } fill_extra_data(c); /* it's possible we already have a complete job */ maybe_enqueue_incoming_job(c); break; case OP_PEEK_READY: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_READY_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; if (c->use->ready.len) { j = job_copy(c->use->ready.data[0]); } if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_PEEK_DELAYED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_DELAYED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; if (c->use->delay.len) { j = job_copy(c->use->delay.data[0]); } if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_PEEK_BURIED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_BURIED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; j = job_copy(buried_job_p(c->use)? j = c->use->buried.next : NULL); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_PEEKJOB: errno = 0; id = strtoull(c->cmd + CMD_PEEKJOB_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; /* So, peek is annoying, because some other connection might free the * job while we are still trying to write it out. So we copy it and * then free the copy when it's done sending. */ j = job_copy(peek_job(id)); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_RESERVE_TIMEOUT: errno = 0; timeout = strtol(c->cmd + CMD_RESERVE_TIMEOUT_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); case OP_RESERVE: /* FALLTHROUGH */ /* don't allow trailing garbage */ if (type == OP_RESERVE && c->cmd_len != CMD_RESERVE_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; connsetworker(c); if (conndeadlinesoon(c) && !conn_ready(c)) { return reply_msg(c, MSG_DEADLINE_SOON); } /* try to get a new job for this guy */ wait_for_job(c, timeout); process_queue(); break; case OP_DELETE: errno = 0; id = strtoull(c->cmd + CMD_DELETE_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = job_find(id); j = remove_reserved_job(c, j) ? : remove_ready_job(j) ? : remove_buried_job(j) ? : remove_delayed_job(j); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); j->tube->stat.total_delete_ct++; j->r.state = Invalid; r = walwrite(&c->srv->wal, j); walmaint(&c->srv->wal); job_free(j); if (!r) return reply_serr(c, MSG_INTERNAL_ERROR); reply(c, MSG_DELETED, MSG_DELETED_LEN, STATE_SENDWORD); break; case OP_RELEASE: errno = 0; id = strtoull(c->cmd + CMD_RELEASE_LEN, &pri_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); r = read_pri(&pri, pri_buf, &delay_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_delay(&delay, delay_buf, NULL); if (r) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = remove_reserved_job(c, job_find(id)); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); /* We want to update the delay deadline on disk, so reserve space for * that. */ if (delay) { z = walresvupdate(&c->srv->wal, j); if (!z) return reply_serr(c, MSG_OUT_OF_MEMORY); j->walresv += z; } j->r.pri = pri; j->r.delay = delay; j->r.release_ct++; r = enqueue_job(c->srv, j, delay, !!delay); if (r < 0) return reply_serr(c, MSG_INTERNAL_ERROR); if (r == 1) { return reply(c, MSG_RELEASED, MSG_RELEASED_LEN, STATE_SENDWORD); } /* out of memory trying to grow the queue, so it gets buried */ bury_job(c->srv, j, 0); reply(c, MSG_BURIED, MSG_BURIED_LEN, STATE_SENDWORD); break; case OP_BURY: errno = 0; id = strtoull(c->cmd + CMD_BURY_LEN, &pri_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); r = read_pri(&pri, pri_buf, NULL); if (r) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = remove_reserved_job(c, job_find(id)); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); j->r.pri = pri; r = bury_job(c->srv, j, 1); if (!r) return reply_serr(c, MSG_INTERNAL_ERROR); reply(c, MSG_BURIED, MSG_BURIED_LEN, STATE_SENDWORD); break; case OP_KICK: errno = 0; count = strtoul(c->cmd + CMD_KICK_LEN, &end_buf, 10); if (end_buf == c->cmd + CMD_KICK_LEN) { return reply_msg(c, MSG_BAD_FORMAT); } if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; i = kick_jobs(c->srv, c->use, count); return reply_line(c, STATE_SENDWORD, "KICKED %u\r\n", i); case OP_JOBKICK: errno = 0; id = strtoull(c->cmd + CMD_JOBKICK_LEN, &end_buf, 10); if (errno) return twarn("strtoull"), reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = job_find(id); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); if ((j->r.state == Buried && kick_buried_job(c->srv, j)) || (j->r.state == Delayed && kick_delayed_job(c->srv, j))) { reply(c, MSG_KICKED, MSG_KICKED_LEN, STATE_SENDWORD); } else { return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); } break; case OP_TOUCH: errno = 0; id = strtoull(c->cmd + CMD_TOUCH_LEN, &end_buf, 10); if (errno) return twarn("strtoull"), reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = touch_job(c, job_find(id)); if (j) { reply(c, MSG_TOUCHED, MSG_TOUCHED_LEN, STATE_SENDWORD); } else { return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); } break; case OP_STATS: /* don't allow trailing garbage */ if (c->cmd_len != CMD_STATS_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; do_stats(c, fmt_stats, c->srv); break; case OP_JOBSTATS: errno = 0; id = strtoull(c->cmd + CMD_JOBSTATS_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = peek_job(id); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); if (!j->tube) return reply_serr(c, MSG_INTERNAL_ERROR); do_stats(c, (fmt_fn) fmt_job_stats, j); break; case OP_STATS_TUBE: name = c->cmd + CMD_STATS_TUBE_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; t = tube_find(name); if (!t) return reply_msg(c, MSG_NOTFOUND); do_stats(c, (fmt_fn) fmt_stats_tube, t); t = NULL; break; case OP_LIST_TUBES: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBES_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; do_list_tubes(c, &tubes); break; case OP_LIST_TUBE_USED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBE_USED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; reply_line(c, STATE_SENDWORD, "USING %s\r\n", c->use->name); break; case OP_LIST_TUBES_WATCHED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBES_WATCHED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; do_list_tubes(c, &c->watch); break; case OP_USE: name = c->cmd + CMD_USE_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; TUBE_ASSIGN(t, tube_find_or_make(name)); if (!t) return reply_serr(c, MSG_OUT_OF_MEMORY); c->use->using_ct--; TUBE_ASSIGN(c->use, t); TUBE_ASSIGN(t, NULL); c->use->using_ct++; reply_line(c, STATE_SENDWORD, "USING %s\r\n", c->use->name); break; case OP_WATCH: name = c->cmd + CMD_WATCH_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; TUBE_ASSIGN(t, tube_find_or_make(name)); if (!t) return reply_serr(c, MSG_OUT_OF_MEMORY); r = 1; if (!ms_contains(&c->watch, t)) r = ms_append(&c->watch, t); TUBE_ASSIGN(t, NULL); if (!r) return reply_serr(c, MSG_OUT_OF_MEMORY); reply_line(c, STATE_SENDWORD, "WATCHING %zu\r\n", c->watch.used); break; case OP_IGNORE: name = c->cmd + CMD_IGNORE_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; t = NULL; for (i = 0; i < c->watch.used; i++) { t = c->watch.items[i]; if (strncmp(t->name, name, MAX_TUBE_NAME_LEN) == 0) break; t = NULL; } if (t && c->watch.used < 2) return reply_msg(c, MSG_NOT_IGNORED); if (t) ms_remove(&c->watch, t); /* may free t if refcount => 0 */ t = NULL; reply_line(c, STATE_SENDWORD, "WATCHING %zu\r\n", c->watch.used); break; case OP_QUIT: c->state = STATE_CLOSE; break; case OP_PAUSE_TUBE: op_ct[type]++; r = read_tube_name(&name, c->cmd + CMD_PAUSE_TUBE_LEN, &delay_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_delay(&delay, delay_buf, NULL); if (r) return reply_msg(c, MSG_BAD_FORMAT); *delay_buf = '\0'; t = tube_find(name); if (!t) return reply_msg(c, MSG_NOTFOUND); // Always pause for a positive amount of time, to make sure // that waiting clients wake up when the deadline arrives. if (delay == 0) { delay = 1; } t->deadline_at = nanoseconds() + delay; t->pause = delay; t->stat.pause_ct++; reply_line(c, STATE_SENDWORD, "PAUSED\r\n"); break; default: return reply_msg(c, MSG_UNKNOWN_COMMAND); } } /* There are three reasons this function may be called. We need to check for * all of them. * * 1. A reserved job has run out of time. * 2. A waiting client's reserved job has entered the safety margin. * 3. A waiting client's requested timeout has occurred. * * If any of these happen, we must do the appropriate thing. */ static void conn_timeout(Conn *c) { int r, should_timeout = 0; job j; /* Check if the client was trying to reserve a job. */ if (conn_waiting(c) && conndeadlinesoon(c)) should_timeout = 1; /* Check if any reserved jobs have run out of time. We should do this * whether or not the client is waiting for a new reservation. */ while ((j = connsoonestjob(c))) { if (j->r.deadline_at >= nanoseconds()) break; /* This job is in the middle of being written out. If we return it to * the ready queue, someone might free it before we finish writing it * out to the socket. So we'll copy it here and free the copy when it's * done sending. */ if (j == c->out_job) { c->out_job = job_copy(c->out_job); } timeout_ct++; /* stats */ j->r.timeout_ct++; r = enqueue_job(c->srv, remove_this_reserved_job(c, j), 0, 0); if (r < 1) bury_job(c->srv, j, 0); /* out of memory, so bury it */ connsched(c); } if (should_timeout) { return reply_msg(remove_waiting_conn(c), MSG_DEADLINE_SOON); } else if (conn_waiting(c) && c->pending_timeout >= 0) { c->pending_timeout = -1; return reply_msg(remove_waiting_conn(c), MSG_TIMED_OUT); } } void enter_drain_mode(int sig) { drain_mode = 1; } static void do_cmd(Conn *c) { dispatch_cmd(c); fill_extra_data(c); } static void reset_conn(Conn *c) { connwant(c, 'r'); c->next = dirty; dirty = c; /* was this a peek or stats command? */ if (c->out_job && c->out_job->r.state == Copy) job_free(c->out_job); c->out_job = NULL; c->reply_sent = 0; /* now that we're done, reset this */ c->state = STATE_WANTCOMMAND; } static void conn_data(Conn *c) { int r, to_read; job j; struct iovec iov[2]; switch (c->state) { case STATE_WANTCOMMAND: r = read(c->sock.fd, c->cmd + c->cmd_read, LINE_BUF_SIZE - c->cmd_read); if (r == -1) return check_err(c, "read()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->cmd_read += r; /* we got some bytes */ c->cmd_len = cmd_len(c); /* find the EOL */ /* yay, complete command line */ if (c->cmd_len) return do_cmd(c); /* c->cmd_read > LINE_BUF_SIZE can't happen */ /* command line too long? */ if (c->cmd_read == LINE_BUF_SIZE) { c->cmd_read = 0; /* discard the input so far */ return reply_msg(c, MSG_BAD_FORMAT); } /* otherwise we have an incomplete line, so just keep waiting */ break; case STATE_BITBUCKET: /* Invert the meaning of in_job_read while throwing away data -- it * counts the bytes that remain to be thrown away. */ to_read = min(c->in_job_read, BUCKET_BUF_SIZE); r = read(c->sock.fd, bucket, to_read); if (r == -1) return check_err(c, "read()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->in_job_read -= r; /* we got some bytes */ /* (c->in_job_read < 0) can't happen */ if (c->in_job_read == 0) { return reply(c, c->reply, c->reply_len, STATE_SENDWORD); } break; case STATE_WANTDATA: j = c->in_job; r = read(c->sock.fd, j->body + c->in_job_read, j->r.body_size -c->in_job_read); if (r == -1) return check_err(c, "read()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->in_job_read += r; /* we got some bytes */ /* (j->in_job_read > j->r.body_size) can't happen */ maybe_enqueue_incoming_job(c); break; case STATE_SENDWORD: r= write(c->sock.fd, c->reply + c->reply_sent, c->reply_len - c->reply_sent); if (r == -1) return check_err(c, "write()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->reply_sent += r; /* we got some bytes */ /* (c->reply_sent > c->reply_len) can't happen */ if (c->reply_sent == c->reply_len) return reset_conn(c); /* otherwise we sent an incomplete reply, so just keep waiting */ break; case STATE_SENDJOB: j = c->out_job; iov[0].iov_base = (void *)(c->reply + c->reply_sent); iov[0].iov_len = c->reply_len - c->reply_sent; /* maybe 0 */ iov[1].iov_base = j->body + c->out_job_sent; iov[1].iov_len = j->r.body_size - c->out_job_sent; r = writev(c->sock.fd, iov, 2); if (r == -1) return check_err(c, "writev()"); if (r == 0) { c->state = STATE_CLOSE; return; } /* update the sent values */ c->reply_sent += r; if (c->reply_sent >= c->reply_len) { c->out_job_sent += c->reply_sent - c->reply_len; c->reply_sent = c->reply_len; } /* (c->out_job_sent > j->r.body_size) can't happen */ /* are we done? */ if (c->out_job_sent == j->r.body_size) { if (verbose >= 2) { printf(">%d job %"PRIu64"\n", c->sock.fd, j->r.id); } return reset_conn(c); } /* otherwise we sent incomplete data, so just keep waiting */ break; case STATE_WAIT: if (c->halfclosed) { c->pending_timeout = -1; return reply_msg(remove_waiting_conn(c), MSG_TIMED_OUT); } break; } } #define want_command(c) ((c)->sock.fd && ((c)->state == STATE_WANTCOMMAND)) #define cmd_data_ready(c) (want_command(c) && (c)->cmd_read) static void update_conns() { int r; Conn *c; while (dirty) { c = dirty; dirty = dirty->next; c->next = NULL; r = sockwant(&c->sock, c->rw); if (r == -1) { twarn("sockwant"); connclose(c); } } } static void h_conn(const int fd, const short which, Conn *c) { if (fd != c->sock.fd) { twarnx("Argh! event fd doesn't match conn fd."); close(fd); connclose(c); update_conns(); return; } if (which == 'h') { c->halfclosed = 1; } conn_data(c); while (cmd_data_ready(c) && (c->cmd_len = cmd_len(c))) do_cmd(c); if (c->state == STATE_CLOSE) { protrmdirty(c); connclose(c); } update_conns(); } static void prothandle(Conn *c, int ev) { h_conn(c->sock.fd, ev, c); } int64 prottick(Server *s) { int r; job j; int64 now; int i; tube t; int64 period = 0x34630B8A000LL; /* 1 hour in nanoseconds */ int64 d; now = nanoseconds(); while ((j = delay_q_peek())) { d = j->r.deadline_at - now; if (d > 0) { period = min(period, d); break; } j = delay_q_take(); r = enqueue_job(s, j, 0, 0); if (r < 1) bury_job(s, j, 0); /* out of memory, so bury it */ } for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; d = t->deadline_at - now; if (t->pause && d <= 0) { t->pause = 0; process_queue(); } else if (d > 0) { period = min(period, d); } } while (s->conns.len) { Conn *c = s->conns.data[0]; d = c->tickat - now; if (d > 0) { period = min(period, d); break; } heapremove(&s->conns, 0); conn_timeout(c); } update_conns(); return period; } void h_accept(const int fd, const short which, Server *s) { Conn *c; int cfd, flags, r; socklen_t addrlen; struct sockaddr_in6 addr; addrlen = sizeof addr; cfd = accept(fd, (struct sockaddr *)&addr, &addrlen); if (cfd == -1) { if (errno != EAGAIN && errno != EWOULDBLOCK) twarn("accept()"); update_conns(); return; } if (verbose) { printf("accept %d\n", cfd); } flags = fcntl(cfd, F_GETFL, 0); if (flags < 0) { twarn("getting flags"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } r = fcntl(cfd, F_SETFL, flags | O_NONBLOCK); if (r < 0) { twarn("setting O_NONBLOCK"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } c = make_conn(cfd, STATE_WANTCOMMAND, default_tube, default_tube); if (!c) { twarnx("make_conn() failed"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } c->srv = s; c->sock.x = c; c->sock.f = (Handle)prothandle; c->sock.fd = cfd; r = sockwant(&c->sock, 'r'); if (r == -1) { twarn("sockwant"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } update_conns(); } void prot_init() { started_at = nanoseconds(); memset(op_ct, 0, sizeof(op_ct)); int dev_random = open("/dev/urandom", O_RDONLY); if (dev_random < 0) { twarn("open /dev/urandom"); exit(50); } int i, r; byte rand_data[NumIdBytes]; r = read(dev_random, &rand_data, NumIdBytes); if (r != NumIdBytes) { twarn("read /dev/urandom"); exit(50); } for (i = 0; i < NumIdBytes; i++) { sprintf(id + (i * 2), "%02x", rand_data[i]); } close(dev_random); if (uname(&node_info) == -1) { warn("uname"); exit(50); } ms_init(&tubes, NULL, NULL); TUBE_ASSIGN(default_tube, tube_find_or_make("default")); if (!default_tube) twarnx("Out of memory during startup!"); } // For each job in list, inserts the job into the appropriate data // structures and adds it to the log. // // Returns 1 on success, 0 on failure. int prot_replay(Server *s, job list) { job j, nj; int64 t, delay; int r, z; for (j = list->next ; j != list ; j = nj) { nj = j->next; job_remove(j); z = walresvupdate(&s->wal, j); if (!z) { twarnx("failed to reserve space"); return 0; } delay = 0; switch (j->r.state) { case Buried: bury_job(s, j, 0); break; case Delayed: t = nanoseconds(); if (t < j->r.deadline_at) { delay = j->r.deadline_at - t; } /* fall through */ default: r = enqueue_job(s, j, delay, 0); if (r < 1) twarnx("error recovering job %"PRIu64, j->r.id); } } return 1; } beanstalkd-1.10/sd-daemon.c000066400000000000000000000261741237005101200155570ustar00rootroot00000000000000/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ /*** Copyright 2010 Lennart Poettering Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ***/ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include #include #include #include "sd-daemon.h" int sd_listen_fds(int unset_environment) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) return 0; #else int r, fd; const char *e; char *p = NULL; unsigned long l; if (!(e = getenv("LISTEN_PID"))) { r = 0; goto finish; } errno = 0; l = strtoul(e, &p, 10); if (errno != 0) { r = -errno; goto finish; } if (!p || *p || l <= 0) { r = -EINVAL; goto finish; } /* Is this for us? */ if (getpid() != (pid_t) l) { r = 0; goto finish; } if (!(e = getenv("LISTEN_FDS"))) { r = 0; goto finish; } errno = 0; l = strtoul(e, &p, 10); if (errno != 0) { r = -errno; goto finish; } if (!p || *p) { r = -EINVAL; goto finish; } for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + (int) l; fd ++) { int flags; if ((flags = fcntl(fd, F_GETFD)) < 0) { r = -errno; goto finish; } if (flags & FD_CLOEXEC) continue; if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) { r = -errno; goto finish; } } r = (int) l; finish: if (unset_environment) { unsetenv("LISTEN_PID"); unsetenv("LISTEN_FDS"); } return r; #endif } int sd_is_fifo(int fd, const char *path) { struct stat st_fd; if (fd < 0) return -EINVAL; memset(&st_fd, 0, sizeof(st_fd)); if (fstat(fd, &st_fd) < 0) return -errno; if (!S_ISFIFO(st_fd.st_mode)) return 0; if (path) { struct stat st_path; memset(&st_path, 0, sizeof(st_path)); if (stat(path, &st_path) < 0) { if (errno == ENOENT || errno == ENOTDIR) return 0; return -errno; } return st_path.st_dev == st_fd.st_dev && st_path.st_ino == st_fd.st_ino; } return 1; } static int sd_is_socket_internal(int fd, int type, int listening) { struct stat st_fd; if (fd < 0 || type < 0) return -EINVAL; if (fstat(fd, &st_fd) < 0) return -errno; if (!S_ISSOCK(st_fd.st_mode)) return 0; if (type != 0) { int other_type = 0; socklen_t l = sizeof(other_type); if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &other_type, &l) < 0) return -errno; if (l != sizeof(other_type)) return -EINVAL; if (other_type != type) return 0; } if (listening >= 0) { int accepting = 0; socklen_t l = sizeof(accepting); if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &accepting, &l) < 0) return -errno; if (l != sizeof(accepting)) return -EINVAL; if (!accepting != !listening) return 0; } return 1; } union sockaddr_union { struct sockaddr sa; struct sockaddr_in in4; struct sockaddr_in6 in6; struct sockaddr_un un; struct sockaddr_storage storage; }; int sd_is_socket(int fd, int family, int type, int listening) { int r; if (family < 0) return -EINVAL; if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) return r; if (family > 0) { union sockaddr_union sockaddr; socklen_t l; memset(&sockaddr, 0, sizeof(sockaddr)); l = sizeof(sockaddr); if (getsockname(fd, &sockaddr.sa, &l) < 0) return -errno; if (l < sizeof(sa_family_t)) return -EINVAL; return sockaddr.sa.sa_family == family; } return 1; } int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) { union sockaddr_union sockaddr; socklen_t l; int r; if (family != 0 && family != AF_INET && family != AF_INET6) return -EINVAL; if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) return r; memset(&sockaddr, 0, sizeof(sockaddr)); l = sizeof(sockaddr); if (getsockname(fd, &sockaddr.sa, &l) < 0) return -errno; if (l < sizeof(sa_family_t)) return -EINVAL; if (sockaddr.sa.sa_family != AF_INET && sockaddr.sa.sa_family != AF_INET6) return 0; if (family > 0) if (sockaddr.sa.sa_family != family) return 0; if (port > 0) { if (sockaddr.sa.sa_family == AF_INET) { if (l < sizeof(struct sockaddr_in)) return -EINVAL; return htons(port) == sockaddr.in4.sin_port; } else { if (l < sizeof(struct sockaddr_in6)) return -EINVAL; return htons(port) == sockaddr.in6.sin6_port; } } return 1; } int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) { union sockaddr_union sockaddr; socklen_t l; int r; if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) return r; memset(&sockaddr, 0, sizeof(sockaddr)); l = sizeof(sockaddr); if (getsockname(fd, &sockaddr.sa, &l) < 0) return -errno; if (l < sizeof(sa_family_t)) return -EINVAL; if (sockaddr.sa.sa_family != AF_UNIX) return 0; if (path) { if (length <= 0) length = strlen(path); if (length <= 0) /* Unnamed socket */ return l == sizeof(sa_family_t); if (path[0]) /* Normal path socket */ return (l >= sizeof(sa_family_t) + length + 1) && memcmp(path, sockaddr.un.sun_path, length+1) == 0; else /* Abstract namespace socket */ return (l == sizeof(sa_family_t) + length) && memcmp(path, sockaddr.un.sun_path, length) == 0; } return 1; } int sd_notify(int unset_environment, const char *state) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) || !defined(SOCK_CLOEXEC) return 0; #else int fd = -1, r; struct msghdr msghdr; struct iovec iovec; union sockaddr_union sockaddr; const char *e; if (!state) { r = -EINVAL; goto finish; } if (!(e = getenv("NOTIFY_SOCKET"))) return 0; /* Must be an abstract socket, or an absolute path */ if ((e[0] != '@' && e[0] != '/') || e[1] == 0) { r = -EINVAL; goto finish; } if ((fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) { r = -errno; goto finish; } memset(&sockaddr, 0, sizeof(sockaddr)); sockaddr.sa.sa_family = AF_UNIX; strncpy(sockaddr.un.sun_path, e, sizeof(sockaddr.un.sun_path)); if (sockaddr.un.sun_path[0] == '@') sockaddr.un.sun_path[0] = 0; memset(&iovec, 0, sizeof(iovec)); iovec.iov_base = (char*) state; iovec.iov_len = strlen(state); memset(&msghdr, 0, sizeof(msghdr)); msghdr.msg_name = &sockaddr; msghdr.msg_namelen = sizeof(sa_family_t) + strlen(e); if (msghdr.msg_namelen > sizeof(struct sockaddr_un)) msghdr.msg_namelen = sizeof(struct sockaddr_un); msghdr.msg_iov = &iovec; msghdr.msg_iovlen = 1; if (sendmsg(fd, &msghdr, MSG_NOSIGNAL) < 0) { r = -errno; goto finish; } r = 1; finish: if (unset_environment) unsetenv("NOTIFY_SOCKET"); if (fd >= 0) close(fd); return r; #endif } int sd_notifyf(int unset_environment, const char *format, ...) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) return 0; #else va_list ap; char *p = NULL; int r; va_start(ap, format); r = vasprintf(&p, format, ap); va_end(ap); if (r < 0 || !p) return -ENOMEM; r = sd_notify(unset_environment, p); free(p); return r; #endif } int sd_booted(void) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) return 0; #else struct stat a, b; /* We simply test whether the systemd cgroup hierarchy is * mounted */ if (lstat("/sys/fs/cgroup", &a) < 0) return 0; if (lstat("/sys/fs/cgroup/systemd", &b) < 0) return 0; return a.st_dev != b.st_dev; #endif } beanstalkd-1.10/sd-daemon.h000066400000000000000000000240271237005101200155570ustar00rootroot00000000000000/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ #ifndef foosddaemonhfoo #define foosddaemonhfoo /*** Copyright 2010 Lennart Poettering Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ***/ #include #include #ifdef __cplusplus extern "C" { #endif /* Reference implementation of a few systemd related interfaces for writing daemons. These interfaces are trivial to implement. To simplify porting we provide this reference implementation. Applications are welcome to reimplement the algorithms described here if they do not want to include these two source files. The following functionality is provided: - Support for logging with log levels on stderr - File descriptor passing for socket-based activation - Daemon startup and status notification - Detection of systemd boots You may compile this with -DDISABLE_SYSTEMD to disable systemd support. This makes all those calls NOPs that are directly related to systemd (i.e. only sd_is_xxx() will stay useful). Since this is drop-in code we don't want any of our symbols to be exported in any case. Hence we declare hidden visibility for all of them. You may find an up-to-date version of these source files online: http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c This should compile on non-Linux systems, too, but with the exception of the sd_is_xxx() calls all functions will become NOPs. See sd-daemon(7) for more information. */ #if __GNUC__ >= 4 #define _sd_printf_attr_(a,b) __attribute__ ((format (printf, a, b))) #else #define _sd_printf_attr_(a,b) #endif #if (__GNUC__ >= 4) && !defined(SD_EXPORT_SYMBOLS) #define _sd_hidden_ __attribute__ ((visibility("hidden"))) #else #define _sd_hidden_ #endif /* Log levels for usage on stderr: fprintf(stderr, SD_NOTICE "Hello World!\n"); This is similar to printk() usage in the kernel. */ #define SD_EMERG "<0>" /* system is unusable */ #define SD_ALERT "<1>" /* action must be taken immediately */ #define SD_CRIT "<2>" /* critical conditions */ #define SD_ERR "<3>" /* error conditions */ #define SD_WARNING "<4>" /* warning conditions */ #define SD_NOTICE "<5>" /* normal but significant condition */ #define SD_INFO "<6>" /* informational */ #define SD_DEBUG "<7>" /* debug-level messages */ /* The first passed file descriptor is fd 3 */ #define SD_LISTEN_FDS_START 3 /* Returns how many file descriptors have been passed, or a negative errno code on failure. Optionally, removes the $LISTEN_FDS and $LISTEN_PID file descriptors from the environment (recommended, but problematic in threaded environments). If r is the return value of this function you'll find the file descriptors passed as fds SD_LISTEN_FDS_START to SD_LISTEN_FDS_START+r-1. Returns a negative errno style error code on failure. This function call ensures that the FD_CLOEXEC flag is set for the passed file descriptors, to make sure they are not passed on to child processes. If FD_CLOEXEC shall not be set, the caller needs to unset it after this call for all file descriptors that are used. See sd_listen_fds(3) for more information. */ int sd_listen_fds(int unset_environment) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is a FIFO in the file system stored under the specified path, 0 otherwise. If path is NULL a path name check will not be done and the call only verifies if the file descriptor refers to a FIFO. Returns a negative errno style error code on failure. See sd_is_fifo(3) for more information. */ int sd_is_fifo(int fd, const char *path) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is a socket of the specified family (AF_INET, ...) and type (SOCK_DGRAM, SOCK_STREAM, ...), 0 otherwise. If family is 0 a socket family check will not be done. If type is 0 a socket type check will not be done and the call only verifies if the file descriptor refers to a socket. If listening is > 0 it is verified that the socket is in listening mode. (i.e. listen() has been called) If listening is == 0 it is verified that the socket is not in listening mode. If listening is < 0 no listening mode check is done. Returns a negative errno style error code on failure. See sd_is_socket(3) for more information. */ int sd_is_socket(int fd, int family, int type, int listening) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is an Internet socket, of the specified family (either AF_INET or AF_INET6) and the specified type (SOCK_DGRAM, SOCK_STREAM, ...), 0 otherwise. If version is 0 a protocol version check is not done. If type is 0 a socket type check will not be done. If port is 0 a socket port check will not be done. The listening flag is used the same way as in sd_is_socket(). Returns a negative errno style error code on failure. See sd_is_socket_inet(3) for more information. */ int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is an AF_UNIX socket of the specified type (SOCK_DGRAM, SOCK_STREAM, ...) and path, 0 otherwise. If type is 0 a socket type check will not be done. If path is NULL a socket path check will not be done. For normal AF_UNIX sockets set length to 0. For abstract namespace sockets set length to the length of the socket name (including the initial 0 byte), and pass the full socket path in path (including the initial 0 byte). The listening flag is used the same way as in sd_is_socket(). Returns a negative errno style error code on failure. See sd_is_socket_unix(3) for more information. */ int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) _sd_hidden_; /* Informs systemd about changed daemon state. This takes a number of newline separated environment-style variable assignments in a string. The following variables are known: READY=1 Tells systemd that daemon startup is finished (only relevant for services of Type=notify). The passed argument is a boolean "1" or "0". Since there is little value in signalling non-readiness the only value daemons should send is "READY=1". STATUS=... Passes a single-line status string back to systemd that describes the daemon state. This is free-from and can be used for various purposes: general state feedback, fsck-like programs could pass completion percentages and failing programs could pass a human readable error message. Example: "STATUS=Completed 66% of file system check..." ERRNO=... If a daemon fails, the errno-style error code, formatted as string. Example: "ERRNO=2" for ENOENT. BUSERROR=... If a daemon fails, the D-Bus error-style error code. Example: "BUSERROR=org.freedesktop.DBus.Error.TimedOut" MAINPID=... The main pid of a daemon, in case systemd did not fork off the process itself. Example: "MAINPID=4711" Daemons can choose to send additional variables. However, it is recommened to prefix variable names not listed above with X_. Returns a negative errno-style error code on failure. Returns > 0 if systemd could be notified, 0 if it couldn't possibly because systemd is not running. Example: When a daemon finished starting up, it could issue this call to notify systemd about it: sd_notify(0, "READY=1"); See sd_notifyf() for more complete examples. See sd_notify(3) for more information. */ int sd_notify(int unset_environment, const char *state) _sd_hidden_; /* Similar to sd_notify() but takes a format string. Example 1: A daemon could send the following after initialization: sd_notifyf(0, "READY=1\n" "STATUS=Processing requests...\n" "MAINPID=%lu", (unsigned long) getpid()); Example 2: A daemon could send the following shortly before exiting, on failure: sd_notifyf(0, "STATUS=Failed to start up: %s\n" "ERRNO=%i", strerror(errno), errno); See sd_notifyf(3) for more information. */ int sd_notifyf(int unset_environment, const char *format, ...) _sd_printf_attr_(2,3) _sd_hidden_; /* Returns > 0 if the system was booted with systemd. Returns < 0 on error. Returns 0 if the system was not booted with systemd. Note that all of the functions above handle non-systemd boots just fine. You should NOT protect them with a call to this function. Also note that this function checks whether the system, not the user session is controlled by systemd. However the functions above work for both session and system services. See sd_booted(3) for more information. */ int sd_booted(void) _sd_hidden_; #ifdef __cplusplus } #endif #endif beanstalkd-1.10/serv.c000066400000000000000000000017261237005101200146630ustar00rootroot00000000000000#include #include #include #include "dat.h" struct Server srv = { Portdef, NULL, NULL, { Filesizedef, }, }; void srvserve(Server *s) { int r; Socket *sock; int64 period; if (sockinit() == -1) { twarnx("sockinit"); exit(1); } s->sock.x = s; s->sock.f = (Handle)srvaccept; s->conns.less = (Less)connless; s->conns.rec = (Record)connrec; r = listen(s->sock.fd, 1024); if (r == -1) { twarn("listen"); return; } r = sockwant(&s->sock, 'r'); if (r == -1) { twarn("sockwant"); exit(2); } for (;;) { period = prottick(s); int rw = socknext(&sock, period); if (rw == -1) { twarnx("socknext"); exit(1); } if (rw) { sock->f(sock->x, rw); } } } void srvaccept(Server *s, int ev) { h_accept(s->sock.fd, ev, s); } beanstalkd-1.10/testheap.c000066400000000000000000000130701237005101200155140ustar00rootroot00000000000000#include #include #include #include #include #include "ct/ct.h" #include "dat.h" void cttestheap_insert_one() { Heap h = {0}; job j; h.less = job_pri_less; h.rec = job_setheappos; j = make_job(1, 0, 1, 0, 0); assertf(j, "allocate job"); heapinsert(&h, j); assertf(h.len == 1, "h should contain one item."); assertf(j->heap_index == 0, "should match"); } void cttestheap_insert_and_remove_one() { Heap h = {0}; int r; job j, j1; h.less = job_pri_less; h.rec = job_setheappos; j1 = make_job(1, 0, 1, 0, 0); assertf(j1, "allocate job"); r = heapinsert(&h, j1); assertf(r, "insert should succeed"); j = heapremove(&h, 0); assertf(j == j1, "j1 should come back out"); assertf(h.len == 0, "h should be empty."); printf("j->heap_index is %zu\n", j->heap_index); assertf(j->heap_index == -1, "j's heap index should be invalid"); } void cttestheap_priority() { Heap h = {0}; int r; job j, j1, j2, j3; h.less = job_pri_less; h.rec = job_setheappos; j1 = make_job(1, 0, 1, 0, 0); j2 = make_job(2, 0, 1, 0, 0); j3 = make_job(3, 0, 1, 0, 0); assertf(j1, "allocate job"); assertf(j2, "allocate job"); assertf(j3, "allocate job"); r = heapinsert(&h, j2); assertf(r, "insert should succeed"); assertf(j2->heap_index == 0, "should match"); r = heapinsert(&h, j3); assertf(r, "insert should succeed"); assertf(j2->heap_index == 0, "should match"); assertf(j3->heap_index == 1, "should match"); r = heapinsert(&h, j1); assertf(r, "insert should succeed"); assertf(j1->heap_index == 0, "should match"); assertf(j2->heap_index == 2, "should match"); assertf(j3->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j1, "j1 should come out first."); assertf(j2->heap_index == 0, "should match"); assertf(j3->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j2, "j2 should come out second."); assertf(j3->heap_index == 0, "should match"); j = heapremove(&h, 0); assertf(j == j3, "j3 should come out third."); } void cttestheap_fifo_property() { Heap h = {0}; int r; job j, j3a, j3b, j3c; h.less = job_pri_less; h.rec = job_setheappos; j3a = make_job(3, 0, 1, 0, 0); j3b = make_job(3, 0, 1, 0, 0); j3c = make_job(3, 0, 1, 0, 0); assertf(j3a, "allocate job"); assertf(j3b, "allocate job"); assertf(j3c, "allocate job"); r = heapinsert(&h, j3a); assertf(r, "insert should succeed"); assertf(h.data[0] == j3a, "j3a should be in pos 0"); assertf(j3a->heap_index == 0, "should match"); r = heapinsert(&h, j3b); assertf(r, "insert should succeed"); assertf(h.data[1] == j3b, "j3b should be in pos 1"); assertf(j3a->heap_index == 0, "should match"); assertf(j3b->heap_index == 1, "should match"); r = heapinsert(&h, j3c); assertf(r, "insert should succeed"); assertf(h.data[2] == j3c, "j3c should be in pos 2"); assertf(j3a->heap_index == 0, "should match"); assertf(j3b->heap_index == 1, "should match"); assertf(j3c->heap_index == 2, "should match"); j = heapremove(&h, 0); assertf(j == j3a, "j3a should come out first."); assertf(j3b->heap_index == 0, "should match"); assertf(j3c->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j3b, "j3b should come out second."); assertf(j3c->heap_index == 0, "should match"); j = heapremove(&h, 0); assertf(j == j3c, "j3c should come out third."); } void cttestheap_many_jobs() { Heap h = {0}; uint last_pri; int r, i, n = 20; job j; h.less = job_pri_less; h.rec = job_setheappos; for (i = 0; i < n; i++) { j = make_job(1 + rand() % 8192, 0, 1, 0, 0); assertf(j, "allocation"); r = heapinsert(&h, j); assertf(r, "heapinsert"); } last_pri = 0; for (i = 0; i < n; i++) { j = heapremove(&h, 0); assertf(j->r.pri >= last_pri, "should come out in order"); last_pri = j->r.pri; } } void cttestheap_remove_k() { Heap h = {0}; uint last_pri; int r, i, c, n = 20; job j; h.less = job_pri_less; h.rec = job_setheappos; for (c = 0; c < 50; c++) { for (i = 0; i < n; i++) { j = make_job(1 + rand() % 8192, 0, 1, 0, 0); assertf(j, "allocation"); r = heapinsert(&h, j); assertf(r, "heapinsert"); } /* remove one from the middle */ heapremove(&h, 25); /* now make sure the rest are still a valid heap */ last_pri = 0; for (i = 1; i < n; i++) { j = heapremove(&h, 0); assertf(j->r.pri >= last_pri, "should come out in order"); last_pri = j->r.pri; } } } void ctbenchheapinsert(int n) { job *j; int i; j = calloc(n, sizeof *j); assert(j); for (i = 0; i < n; i++) { j[i] = make_job(1, 0, 1, 0, 0); assert(j[i]); j[i]->r.pri = -j[i]->r.id; } Heap h = {0}; h.less = job_pri_less; h.rec = job_setheappos; ctresettimer(); for (i = 0; i < n; i++) { heapinsert(&h, j[i]); } } void ctbenchheapremove(int n) { Heap h = {0}; job j; int i; h.less = job_pri_less; h.rec = job_setheappos; for (i = 0; i < n; i++) { j = make_job(1, 0, 1, 0, 0); assertf(j, "allocate job"); heapinsert(&h, j); } ctresettimer(); for (i = 0; i < n; i++) { heapremove(&h, 0); } } beanstalkd-1.10/testjobs.c000066400000000000000000000055751237005101200155470ustar00rootroot00000000000000#include #include #include #include #include #include "ct/ct.h" #include "dat.h" static tube default_tube; void cttestjob_creation() { job j; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job(1, 0, 1, 0, default_tube); assertf(j->r.pri == 1, "priority should match"); } void cttestjob_cmp_pris() { job a, b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(1 << 27, 0, 1, 0, default_tube); assertf(job_pri_less(a, b), "should be less"); } void cttestjob_cmp_ids() { job a, b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(1, 0, 1, 0, default_tube); b->r.id <<= 49; assertf(job_pri_less(a, b), "should be less"); } void cttestjob_large_pris() { job a, b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(-5, 0, 1, 0, default_tube); assertf(job_pri_less(a, b), "should be less"); a = make_job(-5, 0, 1, 0, default_tube); b = make_job(1, 0, 1, 0, default_tube); assertf(!job_pri_less(a, b), "should not be less"); } void cttestjob_hash_free() { job j; uint64 jid = 83; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job_with_id(0, 0, 1, 0, default_tube, jid); job_free(j); assertf(!job_find(jid), "job should be missing"); } void cttestjob_hash_free_next() { job a, b; uint64 aid = 97, bid = 12386; TUBE_ASSIGN(default_tube, make_tube("default")); b = make_job_with_id(0, 0, 1, 0, default_tube, bid); a = make_job_with_id(0, 0, 1, 0, default_tube, aid); assertf(a->ht_next == b, "b should be chained to a"); job_free(b); assertf(a->ht_next == NULL, "job should be missing"); } void cttestjob_all_jobs_used() { job j, x; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job(0, 0, 1, 0, default_tube); assertf(get_all_jobs_used() == 1, "should match"); x = allocate_job(10); assertf(get_all_jobs_used() == 1, "should match"); job_free(x); assertf(get_all_jobs_used() == 1, "should match"); job_free(j); assertf(get_all_jobs_used() == 0, "should match"); } void cttestjob_100_000_jobs() { int i; TUBE_ASSIGN(default_tube, make_tube("default")); for (i = 0; i < 100000; i++) { make_job(0, 0, 1, 0, default_tube); } assertf(get_all_jobs_used() == 100000, "should match"); for (i = 1; i <= 100000; i++) { job_free(job_find(i)); } fprintf(stderr, "get_all_jobs_used() => %zu\n", get_all_jobs_used()); assertf(get_all_jobs_used() == 0, "should match"); } void ctbenchmakejob(int n) { int i; TUBE_ASSIGN(default_tube, make_tube("default")); for (i = 0; i < n; i++) { make_job(0, 0, 1, 0, default_tube); } } beanstalkd-1.10/testserv.c000066400000000000000000000672451237005101200155730ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ct/ct.h" #include "dat.h" static int srvpid, port, fd, size; static int64 timeout = 5000000000LL; // 5s static byte fallocpat[3]; static int wrapfalloc(int fd, int size) { static int c = 0; printf("\nwrapfalloc: fd=%d size=%d\n", fd, size); if (c >= sizeof(fallocpat) || !fallocpat[c++]) { return ENOSPC; } return rawfalloc(fd, size); } static void muststart(char *a0, char *a1, char *a2, char *a3, char *a4) { srvpid = fork(); if (srvpid < 0) { twarn("fork"); exit(1); } if (srvpid > 0) { printf("%s %s %s %s %s\n", a0, a1, a2, a3, a4); printf("start server pid=%d\n", srvpid); usleep(100000); // .1s; time for the child to bind to its port return; } /* now in child */ execlp(a0, a0, a1, a2, a3, a4, NULL); } static int mustdiallocal(int port) { int r, fd; struct sockaddr_in addr = {}; addr.sin_family = AF_INET; addr.sin_port = htons(port); r = inet_aton("127.0.0.1", &addr.sin_addr); if (!r) { errno = EINVAL; twarn("inet_aton"); exit(1); } fd = socket(PF_INET, SOCK_STREAM, 0); if (fd == -1) { twarn("socket"); exit(1); } r = connect(fd, (struct sockaddr*)&addr, sizeof addr); if (r == -1) { twarn("connect"); exit(1); } return fd; } #define SERVER() (progname=__func__, mustforksrv()) static int mustforksrv() { int r, len, port, ok; struct sockaddr_in addr; srv.sock.fd = make_server_socket("127.0.0.1", "0"); if (srv.sock.fd == -1) { puts("mustforksrv failed"); exit(1); } len = sizeof(addr); r = getsockname(srv.sock.fd, (struct sockaddr*)&addr, (socklen_t*)&len); if (r == -1 || len > sizeof(addr)) { puts("mustforksrv failed"); exit(1); } port = ntohs(addr.sin_port); srvpid = fork(); if (srvpid < 0) { twarn("fork"); exit(1); } if (srvpid > 0) { printf("start server port=%d pid=%d\n", port, srvpid); return port; } /* now in child */ prot_init(); if (srv.wal.use) { struct job list = {}; // We want to make sure that only one beanstalkd tries // to use the wal directory at a time. So acquire a lock // now and never release it. if (!waldirlock(&srv.wal)) { twarnx("failed to lock wal dir %s", srv.wal.dir); exit(10); } list.prev = list.next = &list; walinit(&srv.wal, &list); ok = prot_replay(&srv, &list); if (!ok) { twarnx("failed to replay log"); exit(11); } } srvserve(&srv); /* does not return */ exit(1); /* satisfy the compiler */ } static char * readline(int fd) { int r, i = 0; char c = 0, p = 0; static char buf[1024]; fd_set rfd; struct timeval tv; printf("<%d ", fd); fflush(stdout); for (;;) { FD_ZERO(&rfd); FD_SET(fd, &rfd); tv.tv_sec = timeout / 1000000000; tv.tv_usec = (timeout/1000) % 1000000; r = select(fd+1, &rfd, NULL, NULL, &tv); switch (r) { case 1: break; case 0: fputs("timeout", stderr); exit(8); case -1: perror("select"); exit(1); default: fputs("unknown error", stderr); exit(3); } r = read(fd, &c, 1); if (r == -1) { perror("write"); exit(1); } if (i >= sizeof(buf)-1) { fputs("response too big", stderr); exit(4); } putc(c, stdout); fflush(stdout); buf[i++] = c; if (p == '\r' && c == '\n') { break; } p = c; } buf[i] = '\0'; return buf; } static void ckresp(int fd, char *exp) { char *line; line = readline(fd); assertf(strcmp(exp, line) == 0, "\"%s\" != \"%s\"", exp, line); } static void ckrespsub(int fd, char *sub) { char *line; line = readline(fd); assertf(strstr(line, sub), "\"%s\" not in \"%s\"", sub, line); } static void writefull(int fd, char *s, int n) { int c; for (; n; n -= c) { c = write(fd, s, n); if (c == -1) { perror("write"); exit(1); } s += c; } } static void mustsend(int fd, char *s) { writefull(fd, s, strlen(s)); printf(">%d %s", fd, s); fflush(stdout); } static int filesize(char *path) { int r; struct stat s; r = stat(path, &s); if (r == -1) { twarn("stat"); exit(1); } return s.st_size; } static int exist(char *path) { int r; struct stat s; r = stat(path, &s); return r != -1; } void cttestpause() { int64 s; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "INSERTED 1\r\n"); s = nanoseconds(); mustsend(fd, "pause-tube default 1\r\n"); ckresp(fd, "PAUSED\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 1\r\n"); ckresp(fd, "x\r\n"); assert(nanoseconds() - s >= 1000000000); // 1s } void cttestunderscore() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use x_y\r\n"); ckresp(fd, "USING x_y\r\n"); } void cttest2cmdpacket() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use a\r\nuse b\r\n"); ckresp(fd, "USING a\r\n"); ckresp(fd, "USING b\r\n"); } void cttesttoobig() { job_data_size_limit = 10; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 11\r\n"); mustsend(fd, "delete 9999\r\n"); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "JOB_TOO_BIG\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttestdeleteready() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestmultitube() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use abc\r\n"); ckresp(fd, "USING abc\r\n"); mustsend(fd, "put 999999 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "use def\r\n"); ckresp(fd, "USING def\r\n"); mustsend(fd, "put 99 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "watch abc\r\n"); ckresp(fd, "WATCHING 2\r\n"); mustsend(fd, "watch def\r\n"); ckresp(fd, "WATCHING 3\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 2 0\r\n"); } void cttestnonegativedelay() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 512 -1 100 0\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); } void cttestomittimeleft() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 5 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntime-left: 0\n"); } void cttestsmalldelay() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 1 1 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void ctteststatstube() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use tubea\r\n"); ckresp(fd, "USING tubea\r\n"); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nname: tubea\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-urgent: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-ready: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-reserved: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-delayed: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-buried: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntotal-jobs: 1\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-using: 1\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-watching: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-waiting: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-delete: 1\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-pause-tube: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause-time-left: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nname: default\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-urgent: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-ready: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-reserved: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-delayed: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-buried: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntotal-jobs: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-using: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-watching: 1\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-waiting: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-delete: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-pause-tube: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause-time-left: 0\n"); } void cttestttrlarge() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 120 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 4294 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "put 0 0 4295 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 3\r\n"); mustsend(fd, "put 0 0 4296 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 4\r\n"); mustsend(fd, "put 0 0 4297 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 5\r\n"); mustsend(fd, "put 0 0 5000 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 6\r\n"); mustsend(fd, "put 0 0 21600 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 7\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 120\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4294\n"); mustsend(fd, "stats-job 3\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4295\n"); mustsend(fd, "stats-job 4\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4296\n"); mustsend(fd, "stats-job 5\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4297\n"); mustsend(fd, "stats-job 6\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 5000\n"); mustsend(fd, "stats-job 7\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 21600\n"); } void cttestttrsmall() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 1\n"); } void cttestzerodelay() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 1 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttestreservewithtimeout2conn() { int fd0, fd1; job_data_size_limit = 10; port = SERVER(); fd0 = mustdiallocal(port); fd1 = mustdiallocal(port); mustsend(fd0, "watch foo\r\n"); ckresp(fd0, "WATCHING 2\r\n"); mustsend(fd0, "reserve-with-timeout 1\r\n"); mustsend(fd1, "watch foo\r\n"); ckresp(fd1, "WATCHING 2\r\n"); timeout = 1100000000; // 1.1s ckresp(fd0, "TIMED_OUT\r\n"); } void cttestunpausetube() { int fd0, fd1; port = SERVER(); fd0 = mustdiallocal(port); fd1 = mustdiallocal(port); mustsend(fd0, "put 0 0 0 0\r\n"); mustsend(fd0, "\r\n"); ckresp(fd0, "INSERTED 1\r\n"); mustsend(fd0, "pause-tube default 86400\r\n"); ckresp(fd0, "PAUSED\r\n"); mustsend(fd1, "reserve\r\n"); mustsend(fd0, "pause-tube default 0\r\n"); ckresp(fd0, "PAUSED\r\n"); // ckresp will time out if this takes too long, so the // test will not pass. ckresp(fd1, "RESERVED 1 0\r\n"); ckresp(fd1, "\r\n"); } void cttestbinlogemptyexit() { srv.wal.dir = ctdir(); srv.wal.use = 1; job_data_size_limit = 10; port = SERVER(); kill(srvpid, 9); waitpid(srvpid, NULL, 0); port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttestbinlogbury() { srv.wal.dir = ctdir(); srv.wal.use = 1; job_data_size_limit = 10; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 0\r\n"); ckresp(fd, "\r\n"); mustsend(fd, "bury 1 0\r\n"); ckresp(fd, "BURIED\r\n"); } void cttestbinlogbasic() { srv.wal.dir = ctdir(); srv.wal.use = 1; job_data_size_limit = 10; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); kill(srvpid, 9); waitpid(srvpid, NULL, 0); port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestbinlogsizelimit() { int i = 0; char *b2; int gotsize; size = 1024; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); b2 = fmtalloc("%s/binlog.2", ctdir()); while (!exist(b2)) { mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, fmtalloc("INSERTED %d\r\n", ++i)); } gotsize = filesize(fmtalloc("%s/binlog.1", ctdir())); assertf(gotsize == size, "binlog.1 %d != %d", gotsize, size); gotsize = filesize(b2); assertf(gotsize == size, "binlog.2 %d != %d", gotsize, size); } void cttestbinlogallocation() { int i = 0; size = 601; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); for (i = 1; i <= 96; i++) { mustsend(fd, "put 0 0 120 22\r\n"); mustsend(fd, "job payload xxxxxxxxxx\r\n"); ckresp(fd, fmtalloc("INSERTED %d\r\n", i)); } for (i = 1; i <= 96; i++) { mustsend(fd, fmtalloc("delete %d\r\n", i)); ckresp(fd, "DELETED\r\n"); } } void cttestbinlogread() { srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use test\r\n"); ckresp(fd, "USING test\r\n"); mustsend(fd, "put 0 0 120 4\r\n"); mustsend(fd, "test\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 120 4\r\n"); mustsend(fd, "tes1\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "watch test\r\n"); ckresp(fd, "WATCHING 2\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 4\r\n"); ckresp(fd, "test\r\n"); mustsend(fd, "release 1 1 1\r\n"); ckresp(fd, "RELEASED\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 2 4\r\n"); ckresp(fd, "tes1\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "DELETED\r\n"); kill(srvpid, 9); waitpid(srvpid, NULL, 0); port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "watch test\r\n"); ckresp(fd, "WATCHING 2\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 4\r\n"); ckresp(fd, "test\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "NOT_FOUND\r\n"); } void cttestbinlogdiskfull() { size = 1000; falloc = &wrapfalloc; fallocpat[0] = 1; fallocpat[2] = 1; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 3\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 4\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "OUT_OF_MEMORY\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 6\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 7\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 8\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 9\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 3\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 4\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 6\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 7\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 8\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 9\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestbinlogdiskfulldelete() { size = 1000; falloc = &wrapfalloc; fallocpat[0] = 1; fallocpat[1] = 1; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 3\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 4\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 5\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 6\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 7\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 8\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "OUT_OF_MEMORY\r\n"); assert(exist(fmtalloc("%s/binlog.1", ctdir()))); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 3\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 4\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 5\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 6\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 7\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 8\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestbinlogv5() { char portstr[10]; if (system("which beanstalkd-1.4.6") != 0) { puts("beanstalkd 1.4.6 not found, skipping"); exit(0); } progname=__func__; port = (rand()&0xfbff) + 1024; sprintf(portstr, "%d", port); muststart("beanstalkd-1.4.6", "-b", ctdir(), "-p", portstr); fd = mustdiallocal(port); mustsend(fd, "use test\r\n"); ckresp(fd, "USING test\r\n"); mustsend(fd, "put 1 2 3 4\r\n"); mustsend(fd, "test\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 4 3 2 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 2\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 3\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 4\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 3\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); kill(srvpid, 9); waitpid(srvpid, NULL, 0); srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 2\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 3\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 4\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 3\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); } static void benchputdeletesize(int n, int size) { port = SERVER(); fd = mustdiallocal(port); char buf[50], put[50]; char body[size+1]; memset(body, 'a', size); body[size] = 0; ctsetbytes(size); sprintf(put, "put 0 0 0 %d\r\n", size); int i; for (i = 0; i < n; i++) { mustsend(fd, put); mustsend(fd, body); mustsend(fd, "\r\n"); ckrespsub(fd, "INSERTED "); sprintf(buf, "delete %d\r\n", i+1); mustsend(fd, buf); ckresp(fd, "DELETED\r\n"); } } void ctbenchputdelete8byte(int n) { benchputdeletesize(n, 8); } void ctbenchputdelete1k(int n) { benchputdeletesize(n, 1024); } void ctbenchputdelete8k(int n) { benchputdeletesize(n, 8192); } beanstalkd-1.10/testutil.c000066400000000000000000000065351237005101200155640ustar00rootroot00000000000000#include #include #include #include #include #include #include "ct/ct.h" #include "dat.h" void cttestallocf() { char *got; got = fmtalloc("hello, %s %d", "world", 5); assertf(strcmp("hello, world 5", got) == 0, "got \"%s\"", got); } void cttestoptnone() { char *args[] = { NULL, }; optparse(&srv, args); assert(strcmp(srv.port, Portdef) == 0); assert(srv.addr == NULL); assert(job_data_size_limit == JOB_DATA_SIZE_LIMIT_DEFAULT); assert(srv.wal.filesize == Filesizedef); assert(srv.wal.nocomp == 0); assert(srv.wal.wantsync == 0); assert(srv.user == NULL); assert(srv.wal.dir == NULL); assert(srv.wal.use == 0); assert(verbose == 0); } static void success(void) { _exit(0); } void cttestoptminus() { char *args[] = { "-", NULL, }; atexit(success); optparse(&srv, args); assertf(0, "optparse failed to call exit"); } void cttestoptp() { char *args[] = { "-p1234", NULL, }; optparse(&srv, args); assert(strcmp(srv.port, "1234") == 0); } void cttestoptl() { char *args[] = { "-llocalhost", NULL, }; optparse(&srv, args); assert(strcmp(srv.addr, "localhost") == 0); } void cttestoptlseparate() { char *args[] = { "-l", "localhost", NULL, }; optparse(&srv, args); assert(strcmp(srv.addr, "localhost") == 0); } void cttestoptz() { char *args[] = { "-z1234", NULL, }; optparse(&srv, args); assert(job_data_size_limit == 1234); } void cttestopts() { char *args[] = { "-s1234", NULL, }; optparse(&srv, args); assert(srv.wal.filesize == 1234); } void cttestoptc() { char *args[] = { "-n", "-c", NULL, }; optparse(&srv, args); assert(srv.wal.nocomp == 0); } void cttestoptn() { char *args[] = { "-n", NULL, }; optparse(&srv, args); assert(srv.wal.nocomp == 1); } void cttestoptf() { char *args[] = { "-f1234", NULL, }; optparse(&srv, args); assert(srv.wal.syncrate == 1234000000); assert(srv.wal.wantsync == 1); } void cttestoptF() { char *args[] = { "-f1234", "-F", NULL, }; optparse(&srv, args); assert(srv.wal.wantsync == 0); } void cttestoptu() { char *args[] = { "-ukr", NULL, }; optparse(&srv, args); assert(strcmp(srv.user, "kr") == 0); } void cttestoptb() { char *args[] = { "-bfoo", NULL, }; optparse(&srv, args); assert(strcmp(srv.wal.dir, "foo") == 0); assert(srv.wal.use == 1); } void cttestoptV() { char *args[] = { "-V", NULL, }; optparse(&srv, args); assert(verbose == 1); } void cttestoptV_V() { char *args[] = { "-V", "-V", NULL, }; optparse(&srv, args); assert(verbose == 2); } void cttestoptVVV() { char *args[] = { "-VVV", NULL, }; optparse(&srv, args); assert(verbose == 3); } void cttestoptVnVu() { char *args[] = { "-VnVukr", NULL, }; optparse(&srv, args); assert(verbose == 2); assert(srv.wal.nocomp == 1); assert(strcmp(srv.user, "kr") == 0); } beanstalkd-1.10/time.c000066400000000000000000000004651237005101200146410ustar00rootroot00000000000000#include #include #include #include "dat.h" int64 nanoseconds(void) { int r; struct timeval tv; r = gettimeofday(&tv, 0); if (r != 0) return warnx("gettimeofday"), -1; // can't happen return ((int64)tv.tv_sec)*1000000000 + ((int64)tv.tv_usec)*1000; } beanstalkd-1.10/tube.c000066400000000000000000000032461237005101200146420ustar00rootroot00000000000000#include #include #include #include "dat.h" struct ms tubes; tube make_tube(const char *name) { tube t; t = new(struct tube); if (!t) return NULL; t->name[MAX_TUBE_NAME_LEN - 1] = '\0'; strncpy(t->name, name, MAX_TUBE_NAME_LEN - 1); if (t->name[MAX_TUBE_NAME_LEN - 1] != '\0') twarnx("truncating tube name"); t->ready.less = job_pri_less; t->delay.less = job_delay_less; t->ready.rec = job_setheappos; t->delay.rec = job_setheappos; t->buried = (struct job) { }; t->buried.prev = t->buried.next = &t->buried; ms_init(&t->waiting, NULL, NULL); return t; } static void tube_free(tube t) { prot_remove_tube(t); free(t->ready.data); free(t->delay.data); ms_clear(&t->waiting); free(t); } void tube_dref(tube t) { if (!t) return; if (t->refs < 1) return twarnx("refs is zero for tube: %s", t->name); --t->refs; if (t->refs < 1) tube_free(t); } void tube_iref(tube t) { if (!t) return; ++t->refs; } static tube make_and_insert_tube(const char *name) { int r; tube t = NULL; t = make_tube(name); if (!t) return NULL; /* We want this global tube list to behave like "weak" refs, so don't * increment the ref count. */ r = ms_append(&tubes, t); if (!r) return tube_dref(t), (tube) 0; return t; } tube tube_find(const char *name) { tube t; size_t i; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; if (strncmp(t->name, name, MAX_TUBE_NAME_LEN) == 0) return t; } return NULL; } tube tube_find_or_make(const char *name) { return tube_find(name) ? : make_and_insert_tube(name); } beanstalkd-1.10/util.c000066400000000000000000000121771237005101200146630ustar00rootroot00000000000000#include #include #include #include #include #include #include "sd-daemon.h" #include "dat.h" const char *progname; static void vwarnx(const char *err, const char *fmt, va_list args) __attribute__((format(printf, 2, 0))); static void vwarnx(const char *err, const char *fmt, va_list args) { fprintf(stderr, "%s: ", progname); if (fmt) { vfprintf(stderr, fmt, args); if (err) fprintf(stderr, ": %s", err); } fputc('\n', stderr); } void warn(const char *fmt, ...) { char *err = strerror(errno); /* must be done first thing */ va_list args; va_start(args, fmt); vwarnx(err, fmt, args); va_end(args); } void warnx(const char *fmt, ...) { va_list args; va_start(args, fmt); vwarnx(NULL, fmt, args); va_end(args); } char* fmtalloc(char *fmt, ...) { int n; char *buf; va_list ap; // find out how much space is needed va_start(ap, fmt); n = vsnprintf(0, 0, fmt, ap) + 1; // include space for trailing NUL va_end(ap); buf = malloc(n); if (buf) { va_start(ap, fmt); vsnprintf(buf, n, fmt, ap); va_end(ap); } return buf; } // Zalloc allocates n bytes of zeroed memory and // returns a pointer to it. // If insufficient memory is available, zalloc returns 0. void* zalloc(int n) { void *p; p = malloc(n); if (p) { memset(p, 0, n); } return p; } static void warn_systemd_ignored_option(char *opt, char *arg) { if (sd_listen_fds(0) > 0) { warnx("inherited listen fd; ignoring option: %s %s", opt, arg); } } static void usage(int code) __attribute__ ((noreturn)); static void usage(int code) { fprintf(stderr, "Use: %s [OPTIONS]\n" "\n" "Options:\n" " -b DIR wal directory\n" " -f MS fsync at most once every MS milliseconds" " (use -f0 for \"always fsync\")\n" " -F never fsync (default)\n" " -l ADDR listen on address (default is 0.0.0.0)\n" " -p PORT listen on port (default is " Portdef ")\n" " -u USER become user and group\n" " -z BYTES set the maximum job size in bytes (default is %d)\n" " -s BYTES set the size of each wal file (default is %d)\n" " (will be rounded up to a multiple of 512 bytes)\n" " -c compact the binlog (default)\n" " -n do not compact the binlog\n" " -v show version information\n" " -V increase verbosity\n" " -h show this help\n", progname, JOB_DATA_SIZE_LIMIT_DEFAULT, Filesizedef); exit(code); } static char *flagusage(char *flag) __attribute__ ((noreturn)); static char * flagusage(char *flag) { warnx("flag requires an argument: %s", flag); usage(5); } static size_t parse_size_t(char *str) { char r, x; size_t size; r = sscanf(str, "%zu%c", &size, &x); if (1 != r) { warnx("invalid size: %s", str); usage(5); } return size; } void optparse(Server *s, char **argv) { int64 ms; char *arg, c, *tmp; # define EARGF(x) (*arg ? (tmp=arg,arg="",tmp) : *argv ? *argv++ : (x)) while ((arg = *argv++) && *arg++ == '-' && *arg) { while ((c = *arg++)) { switch (c) { case 'p': s->port = EARGF(flagusage("-p")); warn_systemd_ignored_option("-p", s->port); break; case 'l': s->addr = EARGF(flagusage("-l")); warn_systemd_ignored_option("-l", s->addr); break; case 'z': job_data_size_limit = parse_size_t(EARGF(flagusage("-z"))); break; case 's': s->wal.filesize = parse_size_t(EARGF(flagusage("-s"))); break; case 'c': s->wal.nocomp = 0; break; case 'n': s->wal.nocomp = 1; break; case 'f': ms = (int64)parse_size_t(EARGF(flagusage("-f"))); s->wal.syncrate = ms * 1000000; s->wal.wantsync = 1; break; case 'F': s->wal.wantsync = 0; break; case 'u': s->user = EARGF(flagusage("-u")); break; case 'b': s->wal.dir = EARGF(flagusage("-b")); s->wal.use = 1; break; case 'h': usage(0); case 'v': printf("beanstalkd %s\n", version); exit(0); case 'V': verbose++; break; default: warnx("unknown flag: %s", arg-2); usage(5); } } } if (arg) { warnx("unknown argument: %s", arg-1); usage(5); } } beanstalkd-1.10/verc.sh000077500000000000000000000001051237005101200150240ustar00rootroot00000000000000#!/bin/sh printf 'const char version[] = "' ./vers.sh printf '";\n' beanstalkd-1.10/vers.sh000077500000000000000000000000161237005101200150450ustar00rootroot00000000000000printf '1.10' beanstalkd-1.10/walg.c000066400000000000000000000221011237005101200146240ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include "dat.h" static int reserve(Wal *w, int n); // Reads w->dir for files matching binlog.NNN, // sets w->next to the next unused number, and // returns the minimum number. // If no files are found, sets w->next to 1 and // returns a large number. static int walscandir(Wal *w) { static char base[] = "binlog."; static const int len = sizeof(base) - 1; DIR *d; struct dirent *e; int min = 1<<30; int max = 0; int n; char *p; d = opendir(w->dir); if (!d) return min; while ((e = readdir(d))) { if (strncmp(e->d_name, base, len) == 0) { n = strtol(e->d_name+len, &p, 10); if (p && *p == '\0') { if (n > max) max = n; if (n < min) min = n; } } } closedir(d); w->next = max + 1; return min; } void walgc(Wal *w) { File *f; while (w->head && !w->head->refs) { f = w->head; w->head = f->next; if (w->tail == f) { w->tail = f->next; // also, f->next == NULL } w->nfile--; unlink(f->path); free(f->path); free(f); } } // returns 1 on success, 0 on error. static int usenext(Wal *w) { File *f; f = w->cur; if (!f->next) { twarnx("there is no next wal file"); return 0; } w->cur = f->next; filewclose(f); return 1; } static int ratio(Wal *w) { int n, d; d = w->alive + w->resv; n = w->nfile*w->filesize - d; if (!d) return 0; return n / d; } // Returns the number of bytes reserved or 0 on error. static int walresvmigrate(Wal *w, job j) { int z = 0; // reserve only space for the migrated full job record // space for the delete is already reserved z += sizeof(int); z += strlen(j->tube->name); z += sizeof(Jobrec); z += j->r.body_size; return reserve(w, z); } static void moveone(Wal *w) { job j; if (w->head == w->cur || w->head->next == w->cur) { // no point in moving a job return; } j = w->head->jlist.fnext; if (!j || j == &w->head->jlist) { // head holds no jlist; can't happen twarnx("head holds no jlist"); return; } if (!walresvmigrate(w, j)) { // it will not fit, so we'll try again later return; } filermjob(w->head, j); w->nmig++; walwrite(w, j); } static void walcompact(Wal *w) { int r; for (r=ratio(w); r>=2; r--) { moveone(w); } } static void walsync(Wal *w) { int64 now; now = nanoseconds(); if (w->wantsync && now >= w->lastsync+w->syncrate) { w->lastsync = now; if (fsync(w->cur->fd) == -1) { twarn("fsync"); } } } // Walwrite writes j to the log w (if w is enabled). // On failure, walwrite disables w and returns 0; on success, it returns 1. // Unlke walresv*, walwrite should never fail because of a full disk. // If w is disabled, then walwrite takes no action and returns 1. int walwrite(Wal *w, job j) { int r = 0; if (!w->use) return 1; if (w->cur->resv > 0 || usenext(w)) { if (j->file) { r = filewrjobshort(w->cur, j); } else { r = filewrjobfull(w->cur, j); } } if (!r) { filewclose(w->cur); w->use = 0; } w->nrec++; return r; } void walmaint(Wal *w) { if (w->use) { if (!w->nocomp) { walcompact(w); } walsync(w); } } static int makenextfile(Wal *w) { File *f; f = new(File); if (!f) { twarnx("OOM"); return 0; } if (!fileinit(f, w, w->next)) { free(f); twarnx("OOM"); return 0; } filewopen(f); if (!f->iswopen) { free(f->path); free(f); return 0; } w->next++; fileadd(f, w); return 1; } static void moveresv(File *to, File *from, int n) { from->resv -= n; from->free += n; to->resv += n; to->free -= n; } static int needfree(Wal *w, int n) { if (w->tail->free >= n) return n; if (makenextfile(w)) return n; return 0; } // Ensures: // 1. b->resv is congruent to n (mod z). // 2. x->resv is congruent to 0 (mod z) for each future file x. // Assumes (and preserves) that b->resv >= n. // Reserved space is conserved (neither created nor destroyed); // we just move it around to preserve the invariant. // We might have to allocate a new file. // Returns 1 on success, otherwise 0. If there was a failure, // w->tail is not updated. static int balancerest(Wal *w, File *b, int n) { int rest, c, r; static const int z = sizeof(int) + sizeof(Jobrec); if (!b) return 1; rest = b->resv - n; r = rest % z; if (r == 0) return balancerest(w, b->next, 0); c = z - r; if (w->tail->resv >= c && b->free >= c) { moveresv(b, w->tail, c); return balancerest(w, b->next, 0); } if (needfree(w, r) != r) { twarnx("needfree"); return 0; } moveresv(w->tail, b, r); return balancerest(w, b->next, 0); } // Ensures: // 1. w->cur->resv >= n. // 2. w->cur->resv is congruent to n (mod z). // 3. x->resv is congruent to 0 (mod z) for each future file x. // (where z is the size of a delete record in the wal). // Reserved space is conserved (neither created nor destroyed); // we just move it around to preserve the invariant. // We might have to allocate a new file. // Returns 1 on success, otherwise 0. If there was a failure, // w->tail is not updated. static int balance(Wal *w, int n) { int r; // Invariant 1 // (this loop will run at most once) while (w->cur->resv < n) { int m = w->cur->resv; r = needfree(w, m); if (r != m) { twarnx("needfree"); return 0; } moveresv(w->tail, w->cur, m); usenext(w); } // Invariants 2 and 3 return balancerest(w, w->cur, n); } // Returns the number of bytes successfully reserved: either 0 or n. static int reserve(Wal *w, int n) { int r; // return value must be nonzero but is otherwise ignored if (!w->use) return 1; if (w->cur->free >= n) { w->cur->free -= n; w->cur->resv += n; w->resv += n; return n; } r = needfree(w, n); if (r != n) { twarnx("needfree"); return 0; } w->tail->free -= n; w->tail->resv += n; w->resv += n; if (!balance(w, n)) { // error; undo the reservation w->resv -= n; w->tail->resv -= n; w->tail->free += n; return 0; } return n; } // Returns the number of bytes reserved or 0 on error. int walresvput(Wal *w, job j) { int z = 0; // reserve space for the initial job record z += sizeof(int); z += strlen(j->tube->name); z += sizeof(Jobrec); z += j->r.body_size; // plus space for a delete to come later z += sizeof(int); z += sizeof(Jobrec); return reserve(w, z); } // Returns the number of bytes reserved or 0 on error. int walresvupdate(Wal *w, job j) { int z = 0; z +=sizeof(int); z +=sizeof(Jobrec); return reserve(w, z); } // Returns the number of locks acquired: either 0 or 1. int waldirlock(Wal *w) { int r; int fd; struct flock lk; char *path; size_t path_length; path_length = strlen(w->dir) + strlen("/lock") + 1; if ((path = malloc(path_length)) == NULL) { twarn("malloc"); return 0; } r = snprintf(path, path_length, "%s/lock", w->dir); fd = open(path, O_WRONLY|O_CREAT, 0600); free(path); if (fd == -1) { twarn("open"); return 0; } lk.l_type = F_WRLCK; lk.l_whence = SEEK_SET; lk.l_start = 0; lk.l_len = 0; r = fcntl(fd, F_SETLK, &lk); if (r) { twarn("fcntl"); return 0; } // intentionally leak fd, since we never want to close it // and we'll never need it again return 1; } void walread(Wal *w, job list, int min, int max) { File *f; int i, fd; int err = 0; for (i = min; i < w->next; i++) { f = new(File); if (!f) { twarnx("OOM"); exit(1); } if (!fileinit(f, w, i)) { free(f); twarnx("OOM"); exit(1); } fd = open(f->path, O_RDONLY); if (fd < 0) { twarn("open %s", f->path); free(f->path); free(f); continue; } f->fd = fd; fileadd(f, w); err |= fileread(f, list); close(fd); } if (err) { warnx("Errors reading one or more WAL files."); warnx("Continuing. You may be missing data."); } } void walinit(Wal *w, job list) { int min; min = walscandir(w); walread(w, list, min, w->next); // first writable file if (!makenextfile(w)) { twarnx("makenextfile"); exit(1); } w->cur = w->tail; }