pax_global_header00006660000000000000000000000064135056661130014520gustar00rootroot0000000000000052 comment=f03fba3aeb984b65dce17646adeeb7a26523cdfd beanstalkd-1.11/000077500000000000000000000000001350566611300135525ustar00rootroot00000000000000beanstalkd-1.11/.gitignore000066400000000000000000000000541350566611300155410ustar00rootroot00000000000000*.gcda *.gcno *.o /vers.c /beanstalkd /News beanstalkd-1.11/.travis.yml000066400000000000000000000025011350566611300156610ustar00rootroot00000000000000git: quiet: true depth: 5 language: c compiler: - clang - gcc os: - linux - osx env: global: - MAKEJOBS="-j$(getconf _NPROCESSORS_ONLN)" - TRAVIS_COMMIT_LOG=`git log --format=fuller -2` - COVERAGE=OFF # Currently works only with gcc on linux before_script: - if [ "${TRAVIS_OS_NAME}-${TRAVIS_COMPILER}" = "linux-gcc" ]; then export COVERAGE="ON"; fi script: - | if [ $COVERAGE = "ON" ]; then export LDFLAGS=" -lgcov --coverage" export CFLAGS="-O0 -ggdb -fprofile-arcs -ftest-coverage" fi - make $MAKEJOBS || ( echo "Build failure. Verbose build follows." && make V=1 ; false ) - make check $MAKEJOBS VERBOSE=1 jobs: include: - stage: Benchmark os: linux compiler: gcc before_script: - export COVERAGE=OFF script: - make $MAKEJOBS - make $MAKEJOBS bench - stage: Benchmark os: osx compiler: clang before_script: - export COVERAGE=OFF script: - make $MAKEJOBS - make $MAKEJOBS bench after_success: - if [ $COVERAGE = "ON" ]; then (bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"); fi; after_script: - printf "$TRAVIS_COMMIT_RANGE\n" - printf "$TRAVIS_COMMIT_LOG\n" beanstalkd-1.11/CodeOfConduct.txt000066400000000000000000000063461350566611300170030ustar00rootroot00000000000000Contributor Covenant Code of Conduct Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others’ private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. Scope This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at beanstalk-team@googlegroups.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. Attribution This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq beanstalkd-1.11/Contributing000066400000000000000000000026001350566611300161420ustar00rootroot00000000000000Greetings. Firstly, if you're thinking of contributing to beanstalkd, thank you! It's the hard work of people like you that keeps beanstalkd a high-quality codebase and running smoothly in the demanding, high-volume production environment of the servers of many organizations around the world. Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See CodeOfConduct.txt for details. General This is a mature project, so it rarely takes on new features. We mostly focus on stability, bug fixing, clarity, and performance, in that order. Issues When reporting a bug, please describe: - which version of beanstalkd you're using - steps to reproduce the bug - the behavior you saw - the behavior you expected If you're not using the latest version, please consider also testing with the latest. There's a good chance the bug you found has already been fixed. Pull Requests When opening a pull request, try to keep the changes focused on one topic and avoid unrelated changes (even small things, like editing punctuation or whitespace in comments). If you're making big changes, consider discussing it on the mailing list first. You might save yourself a lot of time if it turns out that the changes you want to make aren't a good fit for the project. This is especially true if you are adding new functionality. beanstalkd-1.11/LICENSE000066400000000000000000000022331350566611300145570ustar00rootroot00000000000000Copyright (c) 2007 The authors of beanstalkd. Copyright in contributions to beanstalkd is retained by the original copyright holder of each contribution. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. beanstalkd-1.11/Makefile000066400000000000000000000025721350566611300152200ustar00rootroot00000000000000PREFIX=/usr/local BINDIR=$(DESTDIR)$(PREFIX)/bin override CFLAGS+=-Wall -Werror -Wformat=2 -g override LDFLAGS?= OS=$(shell uname|tr A-Z a-z) INSTALL=install VERS=$(shell ./vers.sh) TARG=beanstalkd MOFILE=main.o OFILES=\ $(OS).o\ conn.o\ file.o\ heap.o\ job.o\ ms.o\ net.o\ primes.o\ prot.o\ sd-daemon.o\ serv.o\ time.o\ tube.o\ util.o\ vers.o\ walg.o\ TOFILES=\ testheap.o\ testjobs.o\ testserv.o\ testutil.o\ HFILES=\ dat.h\ sd-daemon.h\ ifeq ($(OS),linux) LDLIBS=\ -lrt\ endif CLEANFILES=\ vers.c\ *.gcda\ *.gcno\ .PHONY: all all: $(TARG) $(TARG): $(OFILES) $(MOFILE) $(LINK.o) -o $@ $^ $(LDLIBS) .PHONY: install install: $(BINDIR)/$(TARG) $(BINDIR)/%: % $(INSTALL) -d $(dir $@) $(INSTALL) $< $@ CLEANFILES:=$(CLEANFILES) $(TARG) $(OFILES) $(MOFILE): $(HFILES) .PHONY: clean clean: rm -f *.o $(CLEANFILES) .PHONY: check check: ct/_ctcheck ct/_ctcheck .PHONY: bench bench: ct/_ctcheck ct/_ctcheck -b ct/_ctcheck: ct/_ctcheck.o ct/ct.o $(OFILES) $(TOFILES) ct/_ctcheck.c: $(TOFILES) ct/gen ct/gen $(TOFILES) >$@.part mv $@.part $@ ct/ct.o ct/_ctcheck.o: ct/ct.h ct/internal.h $(TOFILES): $(HFILES) ct/ct.h CLEANFILES:=$(CLEANFILES) ct/_* ct/*.o ifneq ($(shell ./verc.sh),$(shell cat vers.c 2>/dev/null)) .PHONY: vers.c endif vers.c: ./verc.sh >vers.c doc/beanstalkd.1 doc/beanstalkd.1.html: doc/beanstalkd.ronn ronn $< freebsd.o: darwin.c beanstalkd-1.11/News000066400000000000000000000013711350566611300144130ustar00rootroot00000000000000News ---- This is the first release since 4th August 2014. It is a maintenance, containing various bug fixes and improvements that have been in the master branch for a long time. We hope to continue providing these releases based on updates from the community in the future. Full list of changes (includes authorship information): Our Urls -------- Download the 1.11 tarball directly: Learn all about beanstalk: Talk about beanstalk development or use at: Bugs ---- Please report any bugs to: beanstalkd-1.11/README.md000066400000000000000000000024461350566611300150370ustar00rootroot00000000000000# beanstalkd Simple and fast general purpose work queue. https://beanstalkd.github.io/ Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See CodeOfConduct.txt for details. ## Quick Start $ make $ ./beanstalkd also try, $ ./beanstalkd -h $ ./beanstalkd -VVV $ make CFLAGS=-O2 $ make CC=clang $ make check $ make install $ make install PREFIX=/usr Requires Linux (2.6.17 or later), Mac OS X, or FreeBSD. See [doc/protocol.txt](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt) for details of the network protocol. Uses ronn to generate the manual. See http://github.com/rtomayko/ronn. ## Subdirectories - `adm` - files useful for system administrators - `ct` - testing tool; vendored from https://github.com/kr/ct - `doc` - documentation - `pkg` - scripts to make releases ## Tests Unit tests are in test*.c. See https://github.com/kr/ct for information on how to write them. ## Code Status [![Build Status](https://travis-ci.org/beanstalkd/beanstalkd.svg?branch=master)](https://travis-ci.org/beanstalkd/beanstalkd) [![codecov](https://codecov.io/gh/beanstalkd/beanstalkd/branch/master/graph/badge.svg)](https://codecov.io/gh/beanstalkd/beanstalkd) beanstalkd-1.11/adm/000077500000000000000000000000001350566611300143135ustar00rootroot00000000000000beanstalkd-1.11/adm/Readme000066400000000000000000000030101350566611300154250ustar00rootroot00000000000000The usual way to run beanstalkd is to type its name in a Unix shell prompt, like this: $ beanstalkd This will start up the process and give you control over it. You can control its output (by default output is printed to the screen; you can arrange to have output go into file b.log by typing ">b.log" at the end of the command line), pause and restart the process (by pressing Control-Z and typing "fg"), and kill it (by pressing Control-C). This is most convenient while writing programs that use beanstalkd (or when working on beanstalkd itself), since you might want to start and stop it many times and regularly inspect its output. If you want beanstalkd to start when your operating system boots, the mechanism varies. Traditionally, you must add a command line to the shell script in /etc/rc (which is read by init when the system boots), using the "&" notation to run beanstalkd in the background. This would suffice for most situations, but it isn't always possible. These days, many popular operating systems have a replacement init program with its own configuration language. Example configuration files for several of these are included in subdirectories here, but the most common is probably "System V init", which reads /etc/inittab for lines describing commands to run at various times. If this file exists, you can add a line something like bean:345:respawn:su nobody -c 'exec /usr/bin/beanstalkd' and type "telinit q" to tell init to reread its configuration. Type "man 5 inittab" for details of this notation. beanstalkd-1.11/adm/launchd/000077500000000000000000000000001350566611300157315ustar00rootroot00000000000000beanstalkd-1.11/adm/launchd/beanstalkd.plist000066400000000000000000000007621350566611300211230ustar00rootroot00000000000000 Label beanstalkd UserName nobody ProgramArguments /usr/local/bin/beanstalkd KeepAlive beanstalkd-1.11/adm/systemd/000077500000000000000000000000001350566611300160035ustar00rootroot00000000000000beanstalkd-1.11/adm/systemd/beanstalkd.service000066400000000000000000000001601350566611300214720ustar00rootroot00000000000000[Unit] Description=Beanstalkd is a simple, fast work queue [Service] User=nobody ExecStart=/usr/bin/beanstalkd beanstalkd-1.11/adm/systemd/beanstalkd.socket000066400000000000000000000002041350566611300213210ustar00rootroot00000000000000[Unit] Description=Socket for beanstalkd, a simple, fast work queue [Socket] ListenStream=11300 [Install] WantedBy=sockets.target beanstalkd-1.11/adm/systemv/000077500000000000000000000000001350566611300160255ustar00rootroot00000000000000beanstalkd-1.11/adm/systemv/beanstalkd.init000077500000000000000000000052531350566611300210320ustar00rootroot00000000000000#!/bin/sh # # System V init script in charge of starting/stopping beanstalkd # # chkconfig: - 57 47 # description: beanstalkd is a simple, fast work queue # processname: beanstalkd # config: /etc/sysconfig/beanstalkd # pidfile: /var/run/beanstalkd/beanstalkd.pid ### BEGIN INIT INFO # Provides: beanstalkd # Required-Start: $local_fs $network $remote_fs # Required-Stop: $local_fs $network $remote_fs # Default-Stop: 0 1 2 6 # Short-Description: start and stop beanstalkd # Description: beanstalkd is a simple, fast work queue ### END INIT INFO # Source function library . /etc/rc.d/init.d/functions # Source networking configuration . /etc/sysconfig/network # Check that networking is up [ "$NETWORKING" = "no" ] && exit exec="/usr/bin/beanstalkd" prog=$(basename $exec) # Default options, overruled by items in sysconfig BEANSTALKD_ADDR=127.0.0.1 BEANSTALKD_PORT=11300 BEANSTALKD_USER=beanstalkd [ -e /etc/sysconfig/${prog} ] && . /etc/sysconfig/${prog} lockfile=/var/lock/subsys/${prog} start() { [ -x $exec ] || exit 5 echo -n $"Starting $prog: " options="-l ${BEANSTALKD_ADDR} -p ${BEANSTALKD_PORT} -u ${BEANSTALKD_USER}" if [ "${BEANSTALKD_MAX_JOB_SIZE}" != "" ]; then options="${options} -z ${BEANSTALKD_MAX_JOB_SIZE}" fi if [ "${BEANSTALKD_BINLOG_DIR}" != "" ]; then if [ ! -d "${BEANSTALKD_BINLOG_DIR}" ]; then echo "Creating binlog directory (${BEANSTALKD_BINLOG_DIR})" mkdir -p ${BEANSTALKD_BINLOG_DIR} chown ${BEANSTALKD_USER}:${BEANSTALKD_USER} ${BEANSTALKD_BINLOG_DIR} fi options="${options} -b ${BEANSTALKD_BINLOG_DIR}" if [ "${BEANSTALKD_BINLOG_FSYNC_PERIOD}" != "" ]; then options="${options} -f ${BEANSTALKD_BINLOG_FSYNC_PERIOD}" else options="${options} -F" fi if [ "${BEANSTALKD_BINLOG_SIZE}" != "" ]; then options="${options} -s ${BEANSTALKD_BINLOG_SIZE}" fi fi daemon "nohup ${exec} $options > /dev/null 2>&1 &" retval=$? echo [ $retval -eq 0 ] && touch $lockfile return $retval } stop() { echo -n $"Stopping $prog: " killproc $prog retval=$? echo [ $retval -eq 0 ] && rm -f $lockfile return $retval } restart() { stop start } reload() { restart } force_reload() { restart } rh_status() { # Run checks to determine if the service is running or use generic status status $prog } rh_status_q() { rh_status >/dev/null 2>&1 } case "$1" in start) rh_status_q && exit 0 $1 ;; stop) rh_status_q || exit 0 $1 ;; restart) $1 ;; reload) rh_status_q || exit 7 $1 ;; force-reload) force_reload ;; status) rh_status ;; condrestart|try-restart) rh_status_q || exit 0 restart ;; *) echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" exit 2 esac exit $?beanstalkd-1.11/adm/upstart/000077500000000000000000000000001350566611300160155ustar00rootroot00000000000000beanstalkd-1.11/adm/upstart/beanstalkd.conf000066400000000000000000000002301350566611300207670ustar00rootroot00000000000000description "simple, fast work queue" start on filesystem stop on runlevel [!2345] respawn respawn limit 5 2 setuid nobody exec /usr/bin/beanstalkd beanstalkd-1.11/beanstalkd.spec000066400000000000000000000047321350566611300165440ustar00rootroot00000000000000%define beanstalkd_user beanstalkd %define beanstalkd_group %{beanstalkd_user} %define beanstalkd_home %{_localstatedir}/lib/beanstalkd %define beanstalkd_logdir %{_localstatedir}/log/beanstalkd Name: beanstalkd Version: 1.11 Release: 0%{?dist} Summary: A simple, fast workqueue service Group: System Environment/Daemons License: GPLv3+ URL: http://xph.us/software/%{name}/ Source0: http://xph.us/dist/%{name}/rel/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires(pre): %{_sbindir}/useradd Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig, /sbin/service Requires(postun): /sbin/service %description Beanstalk is a simple, fast workqueue service. Its interface is generic, but was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. %prep %setup -q if [ ! -e configure ]; then sh buildconf.sh fi %build %configure --disable-rpath --docdir=%{_defaultdocdir}/%{name}-%{version} make %{?_smp_mflags} %install rm -rf $RPM_BUILD_ROOT make install DESTDIR=$RPM_BUILD_ROOT %{__install} -p -D -m 0644 doc/%{name}.1 %{buildroot}%{_mandir}/man1/%{name}.1 %{__install} -p -D -m 0755 scripts/%{name}.init %{buildroot}%{_initrddir}/%{name} %{__install} -p -D -m 0644 scripts/%{name}.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} %clean rm -rf $RPM_BUILD_ROOT %pre %{_sbindir}/useradd -c "beanstalkd user" -s /bin/false -r -m -d %{beanstalkd_home} %{beanstalkd_user} 2>/dev/null || : %post /sbin/chkconfig --add %{name} %preun if [ $1 = 0 ]; then /sbin/service %{name} stop >/dev/null 2>&1 /sbin/chkconfig --del %{name} fi %postun if [ $1 -ge 1 ]; then /sbin/service %{name} condrestart > /dev/null 2>&1 || : fi %files %defattr(-,root,root,-) %doc %{_defaultdocdir}/%{name}-%{version}/protocol.txt %doc README COPYING doc/protocol.txt %{_initrddir}/%{name} %{_bindir}/%{name} %{_mandir}/man1/%{name}.1.gz %config(noreplace) %{_sysconfdir}/sysconfig/%{name} %changelog * Thu Oct 1 2009 Keith Rarick - 1.4-0 - Convert this file to an autoconf template - Tweak the summary and description * Sun Jan 4 2009 Ask Bjørn Hansen - 1.2-0 - 1.2-tobe - Use man page and .init/sysconfig scripts from .tar.gz * Sat Nov 22 2008 Jeremy Hinegardner - 1.1-1 - initial spec creation beanstalkd-1.11/conn.c000066400000000000000000000102201350566611300146460ustar00rootroot00000000000000#include #include #include #include #include #include #include #include "dat.h" #define SAFETY_MARGIN (1000000000) /* 1 second */ static int cur_conn_ct = 0, cur_worker_ct = 0, cur_producer_ct = 0; static uint tot_conn_ct = 0; int verbose = 0; static void on_watch(ms a, tube t, size_t i) { tube_iref(t); t->watching_ct++; } static void on_ignore(ms a, tube t, size_t i) { t->watching_ct--; tube_dref(t); } Conn * make_conn(int fd, char start_state, tube use, tube watch) { job j; Conn *c; c = new(Conn); if (!c) return twarn("OOM"), NULL; ms_init(&c->watch, (ms_event_fn) on_watch, (ms_event_fn) on_ignore); if (!ms_append(&c->watch, watch)) { free(c); return twarn("OOM"), NULL; } TUBE_ASSIGN(c->use, use); use->using_ct++; c->sock.fd = fd; c->state = start_state; c->pending_timeout = -1; c->tickpos = -1; j = &c->reserved_jobs; j->prev = j->next = j; /* stats */ cur_conn_ct++; tot_conn_ct++; return c; } void connsetproducer(Conn *c) { if (c->type & CONN_TYPE_PRODUCER) return; c->type |= CONN_TYPE_PRODUCER; cur_producer_ct++; /* stats */ } void connsetworker(Conn *c) { if (c->type & CONN_TYPE_WORKER) return; c->type |= CONN_TYPE_WORKER; cur_worker_ct++; /* stats */ } int count_cur_conns() { return cur_conn_ct; } uint count_tot_conns() { return tot_conn_ct; } int count_cur_producers() { return cur_producer_ct; } int count_cur_workers() { return cur_worker_ct; } static int has_reserved_job(Conn *c) { return job_list_any_p(&c->reserved_jobs); } static int64 conntickat(Conn *c) { int margin = 0, should_timeout = 0; int64 t = INT64_MAX; if (conn_waiting(c)) { margin = SAFETY_MARGIN; } if (has_reserved_job(c)) { t = connsoonestjob(c)->r.deadline_at - nanoseconds() - margin; should_timeout = 1; } if (c->pending_timeout >= 0) { t = min(t, ((int64)c->pending_timeout) * 1000000000); should_timeout = 1; } if (should_timeout) { return nanoseconds() + t; } return 0; } void connwant(Conn *c, int rw) { c->rw = rw; connsched(c); } void connsched(Conn *c) { if (c->tickpos > -1) { heapremove(&c->srv->conns, c->tickpos); } c->tickat = conntickat(c); if (c->tickat) { heapinsert(&c->srv->conns, c); } } /* return the reserved job with the earliest deadline, * or NULL if there's no reserved job */ job connsoonestjob(Conn *c) { job j = NULL; job soonest = c->soonest_job; if (soonest == NULL) { for (j = c->reserved_jobs.next; j != &c->reserved_jobs; j = j->next) { if (j->r.deadline_at <= (soonest ? : j)->r.deadline_at) soonest = j; } } c->soonest_job = soonest; return soonest; } /* return true if c has a reserved job with less than one second until its * deadline */ int conndeadlinesoon(Conn *c) { int64 t = nanoseconds(); job j = connsoonestjob(c); return j && t >= j->r.deadline_at - SAFETY_MARGIN; } int conn_ready(Conn *c) { size_t i; for (i = 0; i < c->watch.used; i++) { if (((tube) c->watch.items[i])->ready.len) return 1; } return 0; } int connless(Conn *a, Conn *b) { return a->tickat < b->tickat; } void connrec(Conn *c, int i) { c->tickpos = i; } void connclose(Conn *c) { sockwant(&c->sock, 0); close(c->sock.fd); if (verbose) { printf("close %d\n", c->sock.fd); } job_free(c->in_job); /* was this a peek or stats command? */ if (c->out_job && !c->out_job->r.id) job_free(c->out_job); c->in_job = c->out_job = NULL; c->in_job_read = 0; if (c->type & CONN_TYPE_PRODUCER) cur_producer_ct--; /* stats */ if (c->type & CONN_TYPE_WORKER) cur_worker_ct--; /* stats */ cur_conn_ct--; /* stats */ remove_waiting_conn(c); if (has_reserved_job(c)) enqueue_reserved_jobs(c); ms_clear(&c->watch); c->use->using_ct--; TUBE_ASSIGN(c->use, NULL); if (c->tickpos > -1) { heapremove(&c->srv->conns, c->tickpos); } free(c); } beanstalkd-1.11/ct/000077500000000000000000000000001350566611300141605ustar00rootroot00000000000000beanstalkd-1.11/ct/.gitignore000066400000000000000000000000111350566611300161400ustar00rootroot00000000000000/*.o /_* beanstalkd-1.11/ct/License000066400000000000000000000020471350566611300154700ustar00rootroot00000000000000Copyright © 2010–2013 Keith Rarick Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. beanstalkd-1.11/ct/ct.c000066400000000000000000000246541350566611300147450ustar00rootroot00000000000000/* CT - simple-minded unit testing for C */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "internal.h" #include "ct.h" static char *curdir; static int rjobfd = -1, wjobfd = -1; int fail = 0; /* bool */ static int64 bstart, bdur; static int btiming; /* bool */ static int64 bbytes; enum { Second = 1000 * 1000 * 1000 }; enum { BenchTime = Second }; enum { MaxN = 1000 * 1000 * 1000 }; #ifdef __MACH__ # include static int64 nstime() { return (int64)mach_absolute_time(); } #else # include static int64 nstime() { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return (int64)(t.tv_sec)*Second + t.tv_nsec; } #endif void ctlogpn(const char *p, int n, const char *fmt, ...) { va_list arg; printf("%s:%d: ", p, n); va_start(arg, fmt); vprintf(fmt, arg); va_end(arg); putchar('\n'); } void ctfail(void) { fail = 1; } void ctfailnow(void) { fflush(NULL); abort(); } char * ctdir(void) { return curdir; } void ctresettimer(void) { bdur = 0; bstart = nstime(); } void ctstarttimer(void) { if (!btiming) { bstart = nstime(); btiming = 1; } } void ctstoptimer(void) { if (btiming) { bdur += nstime() - bstart; btiming = 0; } } void ctsetbytes(int n) { bbytes = (int64)n; } static void die(int code, int err, const char *msg) { putc('\n', stderr); if (msg && *msg) { fputs(msg, stderr); fputs(": ", stderr); } fputs(strerror(err), stderr); putc('\n', stderr); exit(code); } static int tmpfd(void) { FILE *f = tmpfile(); if (!f) { die(1, errno, "tmpfile"); } return fileno(f); } static int failed(int s) { return WIFSIGNALED(s) && (WTERMSIG(s) == SIGABRT); } static void waittest(Test *ts) { Test *t; int pid, stat; pid = wait3(&stat, 0, 0); if (pid == -1) { die(3, errno, "wait"); } killpg(pid, 9); for (t=ts; t->f; t++) { if (t->pid == pid) { t->status = stat; if (!t->status) { putchar('.'); } else if (failed(t->status)) { putchar('F'); } else { putchar('E'); } fflush(stdout); } } } static void start(Test *t) { t->fd = tmpfd(); strcpy(t->dir, TmpDirPat); if (mkdtemp(t->dir) == NULL) { die(1, errno, "mkdtemp"); } fflush(NULL); t->pid = fork(); if (t->pid < 0) { die(1, errno, "fork"); } else if (!t->pid) { setpgid(0, 0); if (dup2(t->fd, 1) == -1) { die(3, errno, "dup2"); } if (close(t->fd) == -1) { die(3, errno, "fclose"); } if (dup2(1, 2) == -1) { die(3, errno, "dup2"); } curdir = t->dir; t->f(); if (fail) { ctfailnow(); } exit(0); } setpgid(t->pid, t->pid); } static void runalltest(Test *ts, int limit) { int nrun = 0; Test *t; for (t=ts; t->f; t++) { if (nrun >= limit) { waittest(ts); nrun--; } start(t); nrun++; } for (; nrun; nrun--) { waittest(ts); } } static void copyfd(FILE *out, int in) { ssize_t n; char buf[1024]; /* arbitrary size */ while ((n = read(in, buf, sizeof(buf))) != 0) { if (fwrite(buf, 1, n, out) != (size_t)n) { die(3, errno, "fwrite"); } } } /* Removes path and all of its children. Writes errors to stderr and keeps going. If path doesn't exist, rmtree returns silently. */ static void rmtree(char *path) { int r = unlink(path); if (r == 0 || errno == ENOENT) { return; /* success */ } int unlinkerr = errno; DIR *d = opendir(path); if (!d) { if (errno == ENOTDIR) { fprintf(stderr, "ct: unlink: %s\n", strerror(unlinkerr)); } else { perror("ct: opendir"); } fprintf(stderr, "ct: path %s\n", path); return; } struct dirent *ent; while ((ent = readdir(d))) { if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { continue; } int n = strlen(path) + 1 + strlen(ent->d_name); char s[n+1]; sprintf(s, "%s/%s", path, ent->d_name); rmtree(s); } closedir(d); r = rmdir(path); if (r == -1) { perror("ct: rmdir"); fprintf(stderr, "ct: path %s\n", path); } } static void runbenchn(Benchmark *b, int n) { int outfd = tmpfd(); int durfd = tmpfd(); strcpy(b->dir, TmpDirPat); if (mkdtemp(b->dir) == NULL) { die(1, errno, "mkdtemp"); } fflush(NULL); int pid = fork(); if (pid < 0) { die(1, errno, "fork"); } else if (!pid) { setpgid(0, 0); if (dup2(outfd, 1) == -1) { die(3, errno, "dup2"); } if (close(outfd) == -1) { die(3, errno, "fclose"); } if (dup2(1, 2) == -1) { die(3, errno, "dup2"); } curdir = b->dir; ctstarttimer(); b->f(n); ctstoptimer(); write(durfd, &bdur, sizeof bdur); write(durfd, &bbytes, sizeof bbytes); exit(0); } setpgid(pid, pid); pid = waitpid(pid, &b->status, 0); if (pid == -1) { die(3, errno, "wait"); } killpg(pid, 9); rmtree(b->dir); if (b->status != 0) { putchar('\n'); lseek(outfd, 0, SEEK_SET); copyfd(stdout, outfd); return; } lseek(durfd, 0, SEEK_SET); int r = read(durfd, &b->dur, sizeof b->dur); if (r != sizeof b->dur) { perror("read"); b->status = 1; } r = read(durfd, &b->bytes, sizeof b->bytes); if (r != sizeof b->bytes) { perror("read"); b->status = 1; } } /* rounddown10 rounds a number down to the nearest power of 10. */ static int rounddown10(int n) { int tens = 0; /* tens = floor(log_10(n)) */ while (n >= 10) { n = n / 10; tens++; } /* result = 10**tens */ int i, result = 1; for (i = 0; i < tens; i++) { result *= 10; } return result; } /* roundup rounds n up to a number of the form [1eX, 2eX, 5eX]. */ static int roundup(int n) { int base = rounddown10(n); if (n == base) return n; if (n <= 2*base) return 2 * base; if (n <= 5*base) return 5 * base; return 10 * base; } static int min(int a, int b) { if (a < b) { return a; } return b; } static int max(int a, int b) { if (a > b) { return a; } return b; } static void runbench(Benchmark *b) { printf("%s\t", b->name); fflush(stdout); int n = 1; runbenchn(b, n); while (b->status == 0 && b->dur < BenchTime && n < MaxN) { int last = n; /* Predict iterations/sec. */ int nsop = b->dur / n; if (nsop == 0) { n = MaxN; } else { n = BenchTime / nsop; } /* Run more iterations than we think we'll need for a second (1.5x). Don't grow too fast in case we had timing errors previously. Be sure to run at least one more than last time. */ n = max(min(n+n/2, 100*last), last+1); /* Round up to something easy to read. */ n = roundup(n); runbenchn(b, n); } if (b->status == 0) { printf("%8d\t%10" PRId64 " ns/op", n, b->dur/n); if (b->bytes > 0) { double mbs = 0; if (b->dur > 0) { int64 sec = b->dur / 1000L / 1000L / 1000L; int64 nsec = b->dur % 1000000000L; double dur = (double)sec + (double)nsec*.0000000001; mbs = ((double)b->bytes * (double)n / 1000000) / dur; } printf("\t%7.2f MB/s", mbs); } putchar('\n'); } else { if (failed(b->status)) { printf("failure"); } else { printf("error"); if (WIFEXITED(b->status)) { printf(" (exit status %d)", WEXITSTATUS(b->status)); } if (WIFSIGNALED(b->status)) { printf(" (signal %d)", WTERMSIG(b->status)); } } putchar('\n'); } } static void runallbench(Benchmark *b) { for (; b->f; b++) { runbench(b); } } static int report(Test *t) { int nfail = 0, nerr = 0; putchar('\n'); for (; t->f; t++) { rmtree(t->dir); if (!t->status) { continue; } printf("\n%s: ", t->name); if (failed(t->status)) { nfail++; printf("failure"); } else { nerr++; printf("error"); if (WIFEXITED(t->status)) { printf(" (exit status %d)", WEXITSTATUS(t->status)); } if (WIFSIGNALED(t->status)) { printf(" (signal %d)", WTERMSIG(t->status)); } } putchar('\n'); lseek(t->fd, 0, SEEK_SET); copyfd(stdout, t->fd); } if (nfail || nerr) { printf("\n%d failures; %d errors.\n", nfail, nerr); } else { printf("\nPASS\n"); } return nfail || nerr; } static int readtokens() { int n = 1; char c, *s; if ((s = strstr(getenv("MAKEFLAGS"), " --jobserver-fds="))) { rjobfd = (int)strtol(s+17, &s, 10); /* skip " --jobserver-fds=" */ wjobfd = (int)strtol(s+1, NULL, 10); /* skip comma */ } if (rjobfd >= 0) { fcntl(rjobfd, F_SETFL, fcntl(rjobfd, F_GETFL)|O_NONBLOCK); while (read(rjobfd, &c, 1) > 0) { n++; } } return n; } static void writetokens(int n) { char c = '+'; if (wjobfd >= 0) { fcntl(wjobfd, F_SETFL, fcntl(wjobfd, F_GETFL)|O_NONBLOCK); for (; n>1; n--) { write(wjobfd, &c, 1); /* ignore error; nothing we can do anyway */ } } } int main(int argc, char **argv) { int n = readtokens(); runalltest(ctmaintest, n); writetokens(n); int code = report(ctmaintest); if (code != 0) { return code; } if (argc == 2 && strcmp(argv[1], "-b") == 0) { runallbench(ctmainbench); } return 0; } beanstalkd-1.11/ct/ct.h000066400000000000000000000010051350566611300147330ustar00rootroot00000000000000char *ctdir(void); void ctfail(void); void ctfailnow(void); void ctresettimer(void); void ctstarttimer(void); void ctstoptimer(void); void ctsetbytes(int); void ctlogpn(const char*, int, const char*, ...) __attribute__((format(printf, 3, 4))); #define ctlog(...) ctlogpn(__FILE__, __LINE__, __VA_ARGS__) #define assert(x) do if (!(x)) {\ ctlog("%s", "test: " #x);\ ctfailnow();\ } while (0) #define assertf(x, ...) do if (!(x)) {\ ctlog("%s", "test: " #x);\ ctlog(__VA_ARGS__);\ ctfailnow();\ } while (0) beanstalkd-1.11/ct/gen000077500000000000000000000013471350566611300146640ustar00rootroot00000000000000#!/bin/sh set -e fixsyms() { if test "`uname -s|tr A-Z a-z`" = darwin then egrep -v [.] | egrep ^_ | sed s/^_// else cat fi } syms() { prefix=$1 shift for f in "$@" do nm $f done | cut -d ' ' -f 3 | fixsyms | egrep ^$prefix } ts=`syms cttest "$@" || true` bs=`syms ctbench "$@" || true` printf '#include \n' printf '#include "internal.h"\n' for t in $ts do printf 'void %s(void);\n' $t done for b in $bs do printf 'void %s(int);\n' $b done printf 'Test ctmaintest[] = {\n' for t in $ts do printf ' {%s, "%s"},\n' $t $t done printf ' {0},\n' printf '};\n' printf 'Benchmark ctmainbench[] = {\n' for b in $bs do printf ' {%s, "%s"},\n' $b $b done printf ' {0},\n' printf '};\n' beanstalkd-1.11/ct/internal.h000066400000000000000000000007551350566611300161540ustar00rootroot00000000000000/* include */ #define TmpDirPat "/tmp/ct.XXXXXX" typedef int64_t int64; typedef struct Test Test; typedef struct Benchmark Benchmark; struct Test { void (*f)(void); const char *name; int status; int fd; int pid; char dir[sizeof TmpDirPat]; }; struct Benchmark { void (*f)(int); const char *name; int status; int64 dur; int64 bytes; char dir[sizeof TmpDirPat]; }; extern Test ctmaintest[]; extern Benchmark ctmainbench[]; beanstalkd-1.11/darwin.c000066400000000000000000000042231350566611300152030ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include "dat.h" enum { Infinity = 1 << 30 }; static int kq; static char buf0[512]; /* buffer of zeros */ /* Allocate disk space. * Expects fd's offset to be 0; may also reset fd's offset to 0. * Returns 0 on success, and a positive errno otherwise. */ int rawfalloc(int fd, int len) { int i, w; for (i = 0; i < len; i += w) { w = write(fd, buf0, sizeof buf0); if (w == -1) return errno; } lseek(fd, 0, 0); /* do not care if this fails */ return 0; } int sockinit(void) { kq = kqueue(); if (kq == -1) { twarn("kqueue"); return -1; } return 0; } int sockwant(Socket *s, int rw) { int n = 0; struct kevent evs[2] = {}, *ev = evs; struct timespec ts = {}; if (s->added) { ev->ident = s->fd; ev->filter = s->added; ev->flags = EV_DELETE; ev++; n++; } if (rw) { ev->ident = s->fd; switch (rw) { case 'r': ev->filter = EVFILT_READ; break; case 'w': ev->filter = EVFILT_WRITE; break; default: // check only for hangup ev->filter = EVFILT_READ; ev->fflags = NOTE_LOWAT; ev->data = Infinity; } ev->flags = EV_ADD; ev->udata = s; s->added = ev->filter; ev++; n++; } return kevent(kq, evs, n, NULL, 0, &ts); } int socknext(Socket **s, int64 timeout) { int r; struct kevent ev; static struct timespec ts; ts.tv_sec = timeout / 1000000000; ts.tv_nsec = timeout % 1000000000; r = kevent(kq, NULL, 0, &ev, 1, &ts); if (r == -1 && errno != EINTR) { twarn("kevent"); return -1; } if (r > 0) { *s = ev.udata; if (ev.flags & EV_EOF) { return 'h'; } switch (ev.filter) { case EVFILT_READ: return 'r'; case EVFILT_WRITE: return 'w'; } } return 0; } beanstalkd-1.11/dat.h000066400000000000000000000207651350566611300145050ustar00rootroot00000000000000// Requirements: // #include // #include typedef unsigned char uchar; typedef uchar byte; typedef unsigned int uint; typedef int32_t int32; typedef uint32_t uint32; typedef int64_t int64; typedef uint64_t uint64; #define int8_t do_not_use_int8_t #define uint8_t do_not_use_uint8_t #define int32_t do_not_use_int32_t #define uint32_t do_not_use_uint32_t #define int64_t do_not_use_int64_t #define uint64_t do_not_use_uint64_t typedef struct ms *ms; typedef struct job *job; typedef struct tube *tube; typedef struct Conn Conn; typedef struct Heap Heap; typedef struct Jobrec Jobrec; typedef struct File File; typedef struct Socket Socket; typedef struct Server Server; typedef struct Wal Wal; typedef void(*ms_event_fn)(ms a, void *item, size_t i); typedef void(*Handle)(void*, int rw); typedef int(*Less)(void*, void*); typedef void(*Record)(void*, int); typedef int(FAlloc)(int, int); #if _LP64 #define NUM_PRIMES 48 #else #define NUM_PRIMES 19 #endif #define MAX_TUBE_NAME_LEN 201 /* A command can be at most LINE_BUF_SIZE chars, including "\r\n". This value * MUST be enough to hold the longest possible command ("pause-tube a{200} 4294967295\r\n") * or reply line ("USING a{200}\r\n"). */ #define LINE_BUF_SIZE 224 /* CONN_TYPE_* are bit masks */ #define CONN_TYPE_PRODUCER 1 #define CONN_TYPE_WORKER 2 #define CONN_TYPE_WAITING 4 #define min(a,b) ((a)<(b)?(a):(b)) #define URGENT_THRESHOLD 1024 #define JOB_DATA_SIZE_LIMIT_DEFAULT ((1 << 16) - 1) extern const char version[]; extern int verbose; extern struct Server srv; // Replaced by tests to simulate failures. extern FAlloc *falloc; struct stats { uint urgent_ct; uint waiting_ct; uint buried_ct; uint reserved_ct; uint pause_ct; uint64 total_delete_ct; uint64 total_jobs_ct; }; struct Heap { int cap; int len; void **data; Less less; Record rec; }; int heapinsert(Heap *h, void *x); void* heapremove(Heap *h, int k); struct Socket { int fd; Handle f; void *x; int added; }; int sockinit(void); int sockwant(Socket*, int); int socknext(Socket**, int64); struct ms { size_t used, cap, last; void **items; ms_event_fn oninsert, onremove; }; enum { Walver = 7 }; enum // Jobrec.state { Invalid, Ready, Reserved, Buried, Delayed, Copy }; // if you modify this struct, you must increment Walver above struct Jobrec { uint64 id; uint32 pri; int64 delay; int64 ttr; int32 body_size; int64 created_at; int64 deadline_at; uint32 reserve_ct; uint32 timeout_ct; uint32 release_ct; uint32 bury_ct; uint32 kick_ct; byte state; }; struct job { Jobrec r; // persistent fields; these get written to the wal /* bookeeping fields; these are in-memory only */ char pad[6]; tube tube; job prev, next; /* linked list of jobs */ job ht_next; /* Next job in a hash table list */ size_t heap_index; /* where is this job in its current heap */ File *file; job fnext; job fprev; void *reserver; int walresv; int walused; char body[]; // written separately to the wal }; struct tube { uint refs; char name[MAX_TUBE_NAME_LEN]; Heap ready; Heap delay; struct ms waiting; /* set of conns */ struct stats stat; uint using_ct; uint watching_ct; int64 pause; int64 deadline_at; struct job buried; }; #define twarn(fmt, args...) warn("%s:%d in %s: " fmt, \ __FILE__, __LINE__, __func__, ##args) #define twarnx(fmt, args...) warnx("%s:%d in %s: " fmt, \ __FILE__, __LINE__, __func__, ##args) void warn(const char *fmt, ...) __attribute__((format(printf, 1, 2))); void warnx(const char *fmt, ...) __attribute__((format(printf, 1, 2))); char* fmtalloc(char *fmt, ...) __attribute__((format(printf, 1, 2))); void* zalloc(int n); #define new(T) zalloc(sizeof(T)) void optparse(Server*, char**); extern const char *progname; int64 nanoseconds(void); int rawfalloc(int fd, int len); void ms_init(ms a, ms_event_fn oninsert, ms_event_fn onremove); void ms_clear(ms a); int ms_append(ms a, void *item); int ms_remove(ms a, void *item); int ms_contains(ms a, void *item); void *ms_take(ms a); #define make_job(pri,delay,ttr,body_size,tube) make_job_with_id(pri,delay,ttr,body_size,tube,0) job allocate_job(int body_size); job make_job_with_id(uint pri, int64 delay, int64 ttr, int body_size, tube tube, uint64 id); void job_free(job j); /* Lookup a job by job ID */ job job_find(uint64 job_id); /* the void* parameters are really job pointers */ void job_setheappos(void*, int); int job_pri_less(void*, void*); int job_delay_less(void*, void*); job job_copy(job j); const char * job_state(job j); int job_list_any_p(job head); job job_remove(job j); void job_insert(job head, job j); /* for unit tests */ size_t get_all_jobs_used(void); extern struct ms tubes; tube make_tube(const char *name); void tube_dref(tube t); void tube_iref(tube t); tube tube_find(const char *name); tube tube_find_or_make(const char *name); #define TUBE_ASSIGN(a,b) (tube_dref(a), (a) = (b), tube_iref(a)) Conn *make_conn(int fd, char start_state, tube use, tube watch); int count_cur_conns(void); uint count_tot_conns(void); int count_cur_producers(void); int count_cur_workers(void); extern size_t primes[]; extern size_t job_data_size_limit; void prot_init(void); int64 prottick(Server *s); Conn *remove_waiting_conn(Conn *c); void enqueue_reserved_jobs(Conn *c); void enter_drain_mode(int sig); void h_accept(const int fd, const short which, Server* srv); void prot_remove_tube(tube t); int prot_replay(Server *s, job list); int make_server_socket(char *host_addr, char *port); struct Conn { Server *srv; Socket sock; char state; char type; Conn *next; tube use; int64 tickat; // time at which to do more work int tickpos; // position in srv->conns job soonest_job; // memoization of the soonest job int rw; // currently want: 'r', 'w', or 'h' int pending_timeout; char halfclosed; char cmd[LINE_BUF_SIZE]; // this string is NOT NUL-terminated int cmd_len; int cmd_read; char *reply; int reply_len; int reply_sent; char reply_buf[LINE_BUF_SIZE]; // this string IS NUL-terminated // How many bytes of in_job->body have been read so far. If in_job is NULL // while in_job_read is nonzero, we are in bit bucket mode and // in_job_read's meaning is inverted -- then it counts the bytes that // remain to be thrown away. int in_job_read; job in_job; // a job to be read from the client job out_job; int out_job_sent; struct ms watch; struct job reserved_jobs; // linked list header }; int connless(Conn *a, Conn *b); void connrec(Conn *c, int i); void connwant(Conn *c, int rw); void connsched(Conn *c); void connclose(Conn *c); void connsetproducer(Conn *c); void connsetworker(Conn *c); job connsoonestjob(Conn *c); int conndeadlinesoon(Conn *c); int conn_ready(Conn *c); #define conn_waiting(c) ((c)->type & CONN_TYPE_WAITING) enum { Filesizedef = (10 << 20) }; struct Wal { int filesize; int use; char *dir; File *head; File *cur; File *tail; int nfile; int next; int64 resv; // bytes reserved int64 alive; // bytes in use int64 nmig; // migrations int64 nrec; // records written ever int wantsync; int64 syncrate; int64 lastsync; int nocomp; // disable binlog compaction? }; int waldirlock(Wal*); void walinit(Wal*, job list); int walwrite(Wal*, job); void walmaint(Wal*); int walresvput(Wal*, job); int walresvupdate(Wal*, job); void walgc(Wal*); struct File { File *next; uint refs; int seq; int iswopen; // is open for writing int fd; int free; int resv; char *path; Wal *w; struct job jlist; // jobs written in this file }; int fileinit(File*, Wal*, int); Wal* fileadd(File*, Wal*); void fileincref(File*); void filedecref(File*); void fileaddjob(File*, job); void filermjob(File*, job); int fileread(File*, job list); void filewopen(File*); void filewclose(File*); int filewrjobshort(File*, job); int filewrjobfull(File*, job); #define Portdef "11300" struct Server { char *port; char *addr; char *user; Wal wal; Socket sock; Heap conns; }; void srvserve(Server *srv); void srvaccept(Server *s, int ev); beanstalkd-1.11/doc/000077500000000000000000000000001350566611300143175ustar00rootroot00000000000000beanstalkd-1.11/doc/beanstalkd.1000066400000000000000000000070661350566611300165220ustar00rootroot00000000000000.\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . .TH "BEANSTALKD" "1" "April 2012" "" "" . .SH "NAME" \fBbeanstalkd\fR \- simple, fast work queue . .SH "SYNOPSIS" \fBbeanstalkd\fR [\fIoptions\fR] . .SH "DESCRIPTION" \fBBeanstalkd\fR is a simple work\-queue service\. Its interface is generic, though it was originally designed for reducing the latency of page views in high\-volume web applications by running time\-consuming tasks asynchronously\. . .P When started, \fBbeanstalkd\fR opens a socket (or uses a file descriptor provided by the init(1) system, see \fIENVIRONMENT\fR) and listens for incoming connections\. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done\. See file \fBdoc/protocol\.txt\fR in the \fBbeanstalkd\fR distribution for a thorough description of the meaning and format of the \fBbeanstalkd\fR protocol\. . .SH "OPTIONS" . .TP \fB\-b\fR \fIpath\fR Use a binlog to keep jobs on persistent storage in directory \fIpath\fR\. Upon startup, \fBbeanstalkd\fR will recover any binlog that is present in \fIpath\fR, then, during normal operation, append new jobs and changes in state to the binlog\. . .TP \fB\-c\fR Perform online, incremental compaction of binlog files\. Negates \fB\-n\fR\. This is the default behavior\. . .IP (Do not use this option, except to negate \fB\-n\fR\. Both \fB\-c\fR and \fB\-n\fR will likely be removed in a future \fBbeanstalkd\fR release\.) . .TP \fB\-f\fR \fIms\fR Call fsync(2) at most once every \fIms\fR milliseconds\. Larger values for \fIms\fR reduce disk activity and improve speed at the cost of safety\. A power failure could result in the loss of up to \fIms\fR milliseconds of history\. . .IP A \fIms\fR value of 0 will cause \fBbeanstalkd\fR to call fsync every time it writes to the binlog\. . .IP (This option has no effect without \fB\-b\fR\.) . .TP \fB\-F\fR Never call fsync(2)\. Equivalent to \fB\-f\fR with an infinite \fIms\fR value\. . .IP This is the default behavior\. . .IP (This option has no effect without \fB\-b\fR\.) . .TP \fB\-h\fR Show a brief help message and exit\. . .TP \fB\-l\fR \fIaddr\fR Listen on address \fIaddr\fR (default is 0\.0\.0\.0)\. . .IP (Option \fB\-l\fR has no effect if sd\-daemon(5) socket activation is being used\. See also \fIENVIRONMENT\fR\.) . .TP \fB\-n\fR Turn off binlog compaction, negating \fB\-c\fR\. . .IP (Do not use this option\. Both \fB\-c\fR and \fB\-n\fR will likely be removed in a future \fBbeanstalkd\fR release\.) . .TP \fB\-p\fR \fIport\fR Listen on TCP port \fIport\fR (default is 11300)\. . .IP (Option \fB\-p\fR has no effect if sd\-daemon(5) socket activation is being used\. See also \fIENVIRONMENT\fR\.) . .TP \fB\-s\fR \fIbytes\fR The size in bytes of each binlog file\. . .IP (This option has no effect without \fB\-b\fR\.) . .TP \fB\-u\fR \fIuser\fR Become the user \fIuser\fR and its primary group\. . .TP \fB\-V\fR Increase verbosity\. May be used more than once to produce more verbose output\. The output format is subject to change\. . .TP \fB\-v\fR Print the version string and exit\. . .TP \fB\-z\fR \fIbytes\fR The maximum size in bytes of a job\. . .SH "ENVIRONMENT" . .TP \fBLISTEN_PID\fR, \fBLISTEN_FDS\fR These variables can be set by init(1)\. See sd_listen_fds(3) for details\. . .SH "SEE ALSO" sd\-daemon(5), sd_listen_fds(5) . .P Files \fBREADME\fR and \fBdoc/protocol\.txt\fR in the \fBbeanstalkd\fR distribution\. . .P \fIhttps://beanstalkd\.github\.io/\fR . .SH "AUTHOR" \fBBeanstalkd\fR is written and maintained by Keith Rarick with the help of many others\. beanstalkd-1.11/doc/beanstalkd.1.html000066400000000000000000000200011350566611300174450ustar00rootroot00000000000000 beanstalkd(1) - simple, fast work queue
  1. beanstalkd(1)
  3. beanstalkd(1)

NAME

beanstalkd - simple, fast work queue

SYNOPSIS

beanstalkd [options]

DESCRIPTION

Beanstalkd is a simple work-queue service. Its interface is generic, though it was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously.

When started, beanstalkd opens a socket (or uses a file descriptor provided by the init(1) system, see ENVIRONMENT) and listens for incoming connections. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done. See file doc/protocol.txt in the beanstalkd distribution for a thorough description of the meaning and format of the beanstalkd protocol.

OPTIONS

-b path

Use a binlog to keep jobs on persistent storage in directory path. Upon startup, beanstalkd will recover any binlog that is present in path, then, during normal operation, append new jobs and changes in state to the binlog.

-c

Perform online, incremental compaction of binlog files. Negates -n. This is the default behavior.

(Do not use this option, except to negate -n. Both -c and -n will likely be removed in a future beanstalkd release.)

-f ms

Call fsync(2) at most once every ms milliseconds. Larger values for ms reduce disk activity and improve speed at the cost of safety. A power failure could result in the loss of up to ms milliseconds of history.

A ms value of 0 will cause beanstalkd to call fsync every time it writes to the binlog.

(This option has no effect without -b.)

-F

Never call fsync(2). Equivalent to -f with an infinite ms value.

This is the default behavior.

(This option has no effect without -b.)

-h

Show a brief help message and exit.

-l addr

Listen on address addr (default is 0.0.0.0).

(Option -l has no effect if sd-daemon(5) socket activation is being used. See also ENVIRONMENT.)

-n

Turn off binlog compaction, negating -c.

(Do not use this option. Both -c and -n will likely be removed in a future beanstalkd release.)

-p port

Listen on TCP port port (default is 11300).

(Option -p has no effect if sd-daemon(5) socket activation is being used. See also ENVIRONMENT.)

-s bytes

The size in bytes of each binlog file.

(This option has no effect without -b.)

-u user

Become the user user and its primary group.

-V

Increase verbosity. May be used more than once to produce more verbose output. The output format is subject to change.

-v

Print the version string and exit.

-z bytes

The maximum size in bytes of a job.

ENVIRONMENT

LISTEN_PID, LISTEN_FDS
These variables can be set by init(1). See sd_listen_fds(3) for details.

SEE ALSO

sd-daemon(5), sd_listen_fds(5)

Files README and doc/protocol.txt in the beanstalkd distribution.

https://beanstalkd.github.io/

AUTHOR

Beanstalkd is written and maintained by Keith Rarick with the help of many others.

  2. April 2012
  3. beanstalkd(1)
beanstalkd-1.11/doc/beanstalkd.ronn000066400000000000000000000061541350566611300173330ustar00rootroot00000000000000beanstalkd(1) -- simple, fast work queue ======================================== ## SYNOPSIS `beanstalkd` [] ## DESCRIPTION `Beanstalkd` is a simple work-queue service. Its interface is generic, though it was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. When started, `beanstalkd` opens a socket (or uses a file descriptor provided by the init(1) system, see [ENVIRONMENT][]) and listens for incoming connections. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done. See file `doc/protocol.txt` in the `beanstalkd` distribution for a thorough description of the meaning and format of the `beanstalkd` protocol. ## OPTIONS * `-b` : Use a binlog to keep jobs on persistent storage in directory . Upon startup, `beanstalkd` will recover any binlog that is present in , then, during normal operation, append new jobs and changes in state to the binlog. * `-c`: Perform online, incremental compaction of binlog files. Negates `-n`. This is the default behavior. (Do not use this option, except to negate `-n`. Both `-c` and `-n` will likely be removed in a future `beanstalkd` release.) * `-f` : Call fsync(2) at most once every milliseconds. Larger values for reduce disk activity and improve speed at the cost of safety. A power failure could result in the loss of up to milliseconds of history. A value of 0 will cause `beanstalkd` to call fsync every time it writes to the binlog. (This option has no effect without `-b`.) * `-F`: Never call fsync(2). Equivalent to `-f` with an infinite value. This is the default behavior. (This option has no effect without `-b`.) * `-h`: Show a brief help message and exit. * `-l` : Listen on address (default is 0.0.0.0). (Option `-l` has no effect if sd-daemon(5) socket activation is being used. See also [ENVIRONMENT][].) * `-n`: Turn off binlog compaction, negating `-c`. (Do not use this option. Both `-c` and `-n` will likely be removed in a future `beanstalkd` release.) * `-p` : Listen on TCP port (default is 11300). (Option `-p` has no effect if sd-daemon(5) socket activation is being used. See also [ENVIRONMENT][].) * `-s` : The size in bytes of each binlog file. (This option has no effect without `-b`.) * `-u` : Become the user and its primary group. * `-V`: Increase verbosity. May be used more than once to produce more verbose output. The output format is subject to change. * `-v`: Print the version string and exit. * `-z` : The maximum size in bytes of a job. ## ENVIRONMENT * `LISTEN_PID`, `LISTEN_FDS`: These variables can be set by init(1). See sd_listen_fds(3) for details. ## SEE ALSO sd-daemon(5), sd_listen_fds(5) Files `README` and `doc/protocol.txt` in the `beanstalkd` distribution. ## AUTHOR `Beanstalkd` is written and maintained by Keith Rarick with the help of many others. beanstalkd-1.11/doc/protocol.txt000066400000000000000000000606071350566611300167320ustar00rootroot00000000000000= Beanstalk Protocol = Protocol -------- The beanstalk protocol runs over TCP using ASCII encoding. Clients connect, send commands and data, wait for responses, and close the connection. For each connection, the server processes commands serially in the order in which they were received and sends responses in the same order. All integers in the protocol are formatted in decimal and (unless otherwise indicated) nonnegative. Names, in this protocol, are ASCII strings. They may contain letters (A-Z and a-z), numerals (0-9), hyphen ("-"), plus ("+"), slash ("/"), semicolon (";"), dot ("."), dollar-sign ("$"), underscore ("_"), and parentheses ("(" and ")"), but they may not begin with a hyphen. They are terminated by white space (either a space char or end of line). Each name must be at least one character long. The protocol contains two kinds of data: text lines and unstructured chunks of data. Text lines are used for client commands and server responses. Chunks are used to transfer job bodies and stats information. Each job body is an opaque sequence of bytes. The server never inspects or modifies a job body and always sends it back in its original form. It is up to the clients to agree on a meaningful interpretation of job bodies. The client may issue the "quit" command, or simply close the TCP connection when it no longer has use for the server. However, beanstalkd performs very well with a large number of open connections, so it is usually better for the client to keep its connection open and reuse it as much as possible. This also avoids the overhead of establishing new TCP connections. If a client violates the protocol (such as by sending a request that is not well-formed or a command that does not exist) or if the server has an error, the server will reply with one of the following error messages: - "OUT_OF_MEMORY\r\n" The server cannot allocate enough memory for the job. The client should try again later. - "INTERNAL_ERROR\r\n" This indicates a bug in the server. It should never happen. If it does happen, please report it at http://groups.google.com/group/beanstalk-talk. - "BAD_FORMAT\r\n" The client sent a command line that was not well-formed. This can happen if the line does not end with \r\n, if non-numeric characters occur where an integer is expected, if the wrong number of arguments are present, or if the command line is mal-formed in any other way. - "UNKNOWN_COMMAND\r\n" The client sent a command that the server does not know. These error responses will not be listed in this document for individual commands in the following sections, but they are implicitly included in the description of all commands. Clients should be prepared to receive an error response after any command. As a last resort, if the server has a serious error that prevents it from continuing service to the current client, the server will close the connection. Job Lifecycle ------------- A job in beanstalk gets created by a client with the "put" command. During its life it can be in one of four states: "ready", "reserved", "delayed", or "buried". After the put command, a job typically starts out ready. It waits in the ready queue until a worker comes along and runs the "reserve" command. If this job is next in the queue, it will be reserved for the worker. The worker will execute the job; when it is finished the worker will send a "delete" command to delete the job. Here is a picture of the typical job lifecycle: put reserve delete -----> [READY] ---------> [RESERVED] --------> *poof* Here is a picture with more possibilities: put with delay release with delay ----------------> [DELAYED] <------------. | | | (time passes) | | | put v reserve | delete -----------------> [READY] ---------> [RESERVED] --------> *poof* ^ ^ | | | \ release | | | `-------------' | | | | kick | | | | bury | [BURIED] <---------------' | | delete `--------> *poof* The system has one or more tubes. Each tube consists of a ready queue and a delay queue. Each job spends its entire life in one tube. Consumers can show interest in tubes by sending the "watch" command; they can show disinterest by sending the "ignore" command. This set of interesting tubes is said to be a consumer's "watch list". When a client reserves a job, it may come from any of the tubes in its watch list. When a client connects, its watch list is initially just the tube named "default". If it submits jobs without having sent a "use" command, they will live in the tube named "default". Tubes are created on demand whenever they are referenced. If a tube is empty (that is, it contains no ready, delayed, or buried jobs) and no client refers to it, it will be deleted. Producer Commands ----------------- The "put" command is for any process that wants to insert a job into the queue. It comprises a command line followed by the job body: put \r\n \r\n It inserts a job into the client's currently used tube (see the "use" command below). - is an integer < 2**32. Jobs with smaller priority values will be scheduled before jobs with larger priorities. The most urgent priority is 0; the least urgent priority is 4,294,967,295. - is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. - -- time to run -- is an integer number of seconds to allow a worker to run this job. This time is counted from the moment a worker reserves this job. If the worker does not delete, release, or bury the job within seconds, the job will time out and the server will release the job. The minimum ttr is 1. If the client sends 0, the server will silently increase the ttr to 1. - is an integer indicating the size of the job body, not including the trailing "\r\n". This value must be less than max-job-size (default: 2**16). - is the job body -- a sequence of bytes of length from the previous line. After sending the command line and body, the client waits for a reply, which may be: - "INSERTED \r\n" to indicate success. - is the integer id of the new job - "BURIED \r\n" if the server ran out of memory trying to grow the priority queue data structure. - is the integer id of the new job - "EXPECTED_CRLF\r\n" The job body must be followed by a CR-LF pair, that is, "\r\n". These two bytes are not counted in the job size given by the client in the put command line. - "JOB_TOO_BIG\r\n" The client has requested to put a job with a body larger than max-job-size bytes. - "DRAINING\r\n" This means that the server has been put into "drain mode" and is no longer accepting new jobs. The client should try another server or disconnect and try again later. To put the server in drain mode, send the SIGUSR1 signal to the process. The "use" command is for producers. Subsequent put commands will put jobs into the tube specified by this command. If no use command has been issued, jobs will be put into the tube named "default". use \r\n - is a name at most 200 bytes. It specifies the tube to use. If the tube does not exist, it will be created. The only reply is: USING \r\n - is the name of the tube now being used. Worker Commands --------------- A process that wants to consume jobs from the queue uses "reserve", "delete", "release", and "bury". The first worker command, "reserve", looks like this: reserve\r\n Alternatively, you can specify a timeout as follows: reserve-with-timeout \r\n This will return a newly-reserved job. If no job is available to be reserved, beanstalkd will wait to send a response until one becomes available. Once a job is reserved for the client, the client has limited time to run (TTR) the job before the job times out. When the job times out, the server will put the job back into the ready queue. Both the TTR and the actual time left can be found in response to the stats-job command. If more than one job is ready, beanstalkd will choose the one with the smallest priority value. Within each priority, it will choose the one that was received first. A timeout value of 0 will cause the server to immediately return either a response or TIMED_OUT. A positive value of timeout will limit the amount of time the client will block on the reserve request until a job becomes available. During the TTR of a reserved job, the last second is kept by the server as a safety margin, during which the client will not be made to wait for another job. If the client issues a reserve command during the safety margin, or if the safety margin arrives while the client is waiting on a reserve command, the server will respond with: DEADLINE_SOON\r\n This gives the client a chance to delete or release its reserved job before the server automatically releases it. TIMED_OUT\r\n If a non-negative timeout was specified and the timeout exceeded before a job became available, or if the client's connection is half-closed, the server will respond with TIMED_OUT. Otherwise, the only other response to this command is a successful reservation in the form of a text line followed by the job body: RESERVED \r\n \r\n - is the job id -- an integer unique to this job in this instance of beanstalkd. - is an integer indicating the size of the job body, not including the trailing "\r\n". - is the job body -- a sequence of bytes of length from the previous line. This is a verbatim copy of the bytes that were originally sent to the server in the put command for this job. The delete command removes a job from the server entirely. It is normally used by the client when the job has successfully run to completion. A client can delete jobs that it has reserved, ready jobs, delayed jobs, and jobs that are buried. The delete command looks like this: delete \r\n - is the job id to delete. The client then waits for one line of response, which may be: - "DELETED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not either reserved by the client, ready, or buried. This could happen if the job timed out before the client sent the delete command. The release command puts a reserved job back into the ready queue (and marks its state as "ready") to be run by any client. It is normally used when the job fails because of a transitory error. It looks like this: release \r\n - is the job id to release. - is a new priority to assign to the job. - is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. The client expects one line of response, which may be: - "RELEASED\r\n" to indicate success. - "BURIED\r\n" if the server ran out of memory trying to grow the priority queue data structure. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The bury command puts a job into the "buried" state. Buried jobs are put into a FIFO linked list and will not be touched by the server again until a client kicks them with the "kick" command. The bury command looks like this: bury \r\n - is the job id to release. - is a new priority to assign to the job. There are two possible responses: - "BURIED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The "touch" command allows a worker to request more time to work on a job. This is useful for jobs that potentially take a long time, but you still want the benefits of a TTR pulling a job away from an unresponsive worker. A worker may periodically tell the server that it's still alive and processing a job (e.g. it may do this on DEADLINE_SOON). The command postpones the auto release of a reserved job until TTR seconds from when the command is issued. The touch command looks like this: touch \r\n - is the ID of a job reserved by the current connection. There are two possible responses: - "TOUCHED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The "watch" command adds the named tube to the watch list for the current connection. A reserve command will take a job from any of the tubes in the watch list. For each new connection, the watch list initially consists of one tube, named "default". watch \r\n - is a name at most 200 bytes. It specifies a tube to add to the watch list. If the tube doesn't exist, it will be created. The reply is: WATCHING \r\n - is the integer number of tubes currently in the watch list. The "ignore" command is for consumers. It removes the named tube from the watch list for the current connection. ignore \r\n The reply is one of: - "WATCHING \r\n" to indicate success. - is the integer number of tubes currently in the watch list. - "NOT_IGNORED\r\n" if the client attempts to ignore the only tube in its watch list. Other Commands -------------- The peek commands let the client inspect a job in the system. There are four variations. All but the first operate only on the currently used tube. - "peek \r\n" - return job . - "peek-ready\r\n" - return the next ready job. - "peek-delayed\r\n" - return the delayed job with the shortest delay left. - "peek-buried\r\n" - return the next job in the list of buried jobs. There are two possible responses, either a single line: - "NOT_FOUND\r\n" if the requested job doesn't exist or there are no jobs in the requested state. Or a line followed by a chunk of data, if the command was successful: FOUND \r\n \r\n - is the job id. - is an integer indicating the size of the job body, not including the trailing "\r\n". - is the job body -- a sequence of bytes of length from the previous line. The kick command applies only to the currently used tube. It moves jobs into the ready queue. If there are any buried jobs, it will only kick buried jobs. Otherwise it will kick delayed jobs. It looks like: kick \r\n - is an integer upper bound on the number of jobs to kick. The server will kick no more than jobs. The response is of the form: KICKED \r\n - is an integer indicating the number of jobs actually kicked. The kick-job command is a variant of kick that operates with a single job identified by its job id. If the given job id exists and is in a buried or delayed state, it will be moved to the ready queue of the the same tube where it currently belongs. The syntax is: kick-job \r\n - is the job id to kick. The response is one of: - "NOT_FOUND\r\n" if the job does not exist or is not in a kickable state. This can also happen upon internal errors. - "KICKED\r\n" when the operation succeeded. The stats-job command gives statistical information about the specified job if it exists. Its form is: stats-job \r\n - is a job id. The response is one of: - "NOT_FOUND\r\n" if the job does not exist. - "OK \r\n\r\n" - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented a dictionary. The stats-job data is a YAML file representing a single dictionary of strings to scalars. It contains these keys: - "id" is the job id - "tube" is the name of the tube that contains this job - "state" is "ready" or "delayed" or "reserved" or "buried" - "pri" is the priority value set by the put, release, or bury commands. - "age" is the time in seconds since the put command that created this job. - "delay" is the integer number of seconds to wait before putting this job in the ready queue. - "ttr" -- time to run -- is the integer number of seconds a worker is allowed to run this job. - "time-left" is the number of seconds left until the server puts this job into the ready queue. This number is only meaningful if the job is reserved or delayed. If the job is reserved and this amount of time elapses before its state changes, it is considered to have timed out. - "file" is the number of the earliest binlog file containing this job. If -b wasn't used, this will be 0. - "reserves" is the number of times this job has been reserved. - "timeouts" is the number of times this job has timed out during a reservation. - "releases" is the number of times a client has released this job from a reservation. - "buries" is the number of times this job has been buried. - "kicks" is the number of times this job has been kicked. The stats-tube command gives statistical information about the specified tube if it exists. Its form is: stats-tube \r\n - is a name at most 200 bytes. Stats will be returned for this tube. The response is one of: - "NOT_FOUND\r\n" if the tube does not exist. - "OK \r\n\r\n" - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented a dictionary. The stats-tube data is a YAML file representing a single dictionary of strings to scalars. It contains these keys: - "name" is the tube's name. - "current-jobs-urgent" is the number of ready jobs with priority < 1024 in this tube. - "current-jobs-ready" is the number of jobs in the ready queue in this tube. - "current-jobs-reserved" is the number of jobs reserved by all clients in this tube. - "current-jobs-delayed" is the number of delayed jobs in this tube. - "current-jobs-buried" is the number of buried jobs in this tube. - "total-jobs" is the cumulative count of jobs created in this tube in the current beanstalkd process. - "current-using" is the number of open connections that are currently using this tube. - "current-waiting" is the number of open connections that have issued a reserve command while watching this tube but not yet received a response. - "current-watching" is the number of open connections that are currently watching this tube. - "pause" is the number of seconds the tube has been paused for. - "cmd-delete" is the cumulative number of delete commands for this tube - "cmd-pause-tube" is the cumulative number of pause-tube commands for this tube. - "pause-time-left" is the number of seconds until the tube is un-paused. The stats command gives statistical information about the system as a whole. Its form is: stats\r\n The server will respond: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented a dictionary. The stats data for the system is a YAML file representing a single dictionary of strings to scalars. Entries described as "cumulative" are reset when the beanstalkd process starts; they are not stored on disk with the -b flag. - "current-jobs-urgent" is the number of ready jobs with priority < 1024. - "current-jobs-ready" is the number of jobs in the ready queue. - "current-jobs-reserved" is the number of jobs reserved by all clients. - "current-jobs-delayed" is the number of delayed jobs. - "current-jobs-buried" is the number of buried jobs. - "cmd-put" is the cumulative number of put commands. - "cmd-peek" is the cumulative number of peek commands. - "cmd-peek-ready" is the cumulative number of peek-ready commands. - "cmd-peek-delayed" is the cumulative number of peek-delayed commands. - "cmd-peek-buried" is the cumulative number of peek-buried commands. - "cmd-reserve" is the cumulative number of reserve commands. - "cmd-use" is the cumulative number of use commands. - "cmd-watch" is the cumulative number of watch commands. - "cmd-ignore" is the cumulative number of ignore commands. - "cmd-delete" is the cumulative number of delete commands. - "cmd-release" is the cumulative number of release commands. - "cmd-bury" is the cumulative number of bury commands. - "cmd-kick" is the cumulative number of kick commands. - "cmd-stats" is the cumulative number of stats commands. - "cmd-stats-job" is the cumulative number of stats-job commands. - "cmd-stats-tube" is the cumulative number of stats-tube commands. - "cmd-list-tubes" is the cumulative number of list-tubes commands. - "cmd-list-tube-used" is the cumulative number of list-tube-used commands. - "cmd-list-tubes-watched" is the cumulative number of list-tubes-watched commands. - "cmd-pause-tube" is the cumulative number of pause-tube commands. - "job-timeouts" is the cumulative count of times a job has timed out. - "total-jobs" is the cumulative count of jobs created. - "max-job-size" is the maximum number of bytes in a job. - "current-tubes" is the number of currently-existing tubes. - "current-connections" is the number of currently open connections. - "current-producers" is the number of open connections that have each issued at least one put command. - "current-workers" is the number of open connections that have each issued at least one reserve command. - "current-waiting" is the number of open connections that have issued a reserve command but not yet received a response. - "total-connections" is the cumulative count of connections. - "pid" is the process id of the server. - "version" is the version string of the server. - "rusage-utime" is the cumulative user CPU time of this process in seconds and microseconds. - "rusage-stime" is the cumulative system CPU time of this process in seconds and microseconds. - "uptime" is the number of seconds since this server process started running. - "binlog-oldest-index" is the index of the oldest binlog file needed to store the current jobs. - "binlog-current-index" is the index of the current binlog file being written to. If binlog is not active this value will be 0. - "binlog-max-size" is the maximum size in bytes a binlog file is allowed to get before a new binlog file is opened. - "binlog-records-written" is the cumulative number of records written to the binlog. - "binlog-records-migrated" is the cumulative number of records written as part of compaction. - "id" is a random id string for this server process, generated when each beanstalkd process starts. - "hostname" the hostname of the machine as determined by uname. The list-tubes command returns a list of all existing tubes. Its form is: list-tubes\r\n The response is: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file containing all tube names as a list of strings. The list-tube-used command returns the tube currently being used by the client. Its form is: list-tube-used\r\n The response is: USING \r\n - is the name of the tube being used. The list-tubes-watched command returns a list tubes currently being watched by the client. Its form is: list-tubes-watched\r\n The response is: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file containing watched tube names as a list of strings. The quit command simply closes the connection. Its form is: quit\r\n The pause-tube command can delay any new job being reserved for a given time. Its form is: pause-tube \r\n - is the tube to pause - is an integer number of seconds < 2**32 to wait before reserving any more jobs from the queue There are two possible responses: - "PAUSED\r\n" to indicate success. - "NOT_FOUND\r\n" if the tube does not exist. beanstalkd-1.11/doc/protocol.zh-CN.md000066400000000000000000000371261350566611300174310ustar00rootroot00000000000000## Beanstalkd中文协议 ### 总括 `beanstalkd`协议基于ASCII编码运行在tcp上。客户端连接服务器并发送指令和数据，然后等待响应并关闭连接。对于每个连接，服务器按照接收命令的序列依次处理并响应。所有整型值都非负的十进制数，除非有特别声明。 ### 名称约定 所有名称必须是ASCII码字符串，即包括： * **字母** (A-Z and a-z) * **数字** (0-9) * **连字符** ("-") * **加号** ("+") * **斜线** ("/") * **分号** (";") * **点** (".") * **美元符** ("$") * **下划线** ("_") * **括号** ("*(*" and "*)*") **注意**：名称不能以连字符开始，并且是以空白字符结束，每个名称至少包含一个字符。 ### 错误说明 | 返回的错误 | 描述 | | --------------------------------------- | -------- | | `OUT_OF_MEMORY\r\n` | 服务器没有足够的内存分配给特定的job，客户端应该稍后重试 | | `INTERNAL_ERROR\r\n` | 服务器内部错误，该错误不应该发生，如果发生了，请报告：http://groups.google.com/group/beanstalk-talk. | | `BAD_FORMAT\r\n` | 格式不正确，客户端发送的指令格式出错，有可能不是以\r\n结尾，或者要求整型值等等 | | `UNKNOWN_COMMAND\r\n` | 未知的命令，客户端发送的指令服务器不理解 | ### job的生命周期 一个工作任务job当client使用put命令时创建。在整个生命周期中job可能有四个工作状态：ready，reserved，delayed，buried。在put之后，一个job的典型状态是ready，在ready队列中，它将等待一个worker取出此job并设置为其为reserved状态。worker占有此job并执行，当job执行完毕，worker可以发送一个delete指令删除此job。 | Status | Description | | --------------------| ------------- | | `ready` | 等待被取出并处理 | | `reserved` | 如果job被worker取出，将被此worker预订，worker将执行此job | | `delayed` | 等待特定时间之后，状态再迁移为ready状态 | | `buried` | 等待唤醒，通常在job处理失败时 job典型的生命周期 ``` put reserve delete -----> [READY] ---------> [RESERVED] --------> *poof* ``` job可能的状态迁移 ``` put with delay release with delay ----------------> [DELAYED] <------------. | | kick | (time passes) | | | put v reserve | delete -----------------> [READY] ---------> [RESERVED] --------> *poof* ^ ^ | | | \ release | | | `-------------' | | | | kick | | | | bury | [BURIED] <---------------' | | delete `--------> *poof* ``` ## Tubes 一个服务器有一个或者多个tubes，用来储存统一类型的job。每个tube由一个就绪队列与延迟队列组成。每个job所有的状态迁移在一个tube中完成。consumers消费者可以监控感兴趣的tube，通过发送watch指令。consumers消费者可以取消监控tube，通过发送ignore命令。通过watch list命令返回所有监控的tubes，当客户端预订一个job，此job可能来自任何一个它监控的tube。 当一个客户端连接上服务器时，客户端监控的tube默认为defaut，如果客户端提交job时，没有使用use命令，那么这些job就存于名为default的tube中。 tube按需求创建，无论他们在什么时候被引用到。如果一个tube变为空（即no ready jobs，no delayed jobs，no buried jobs）和没有任何客户端引用，它将会被自动删除。 ### 指令说明（Commands） #### 生产者指令说明（Producer Commands） #### `put` 插入一个job到队列 ``` put \r\n \r\n ``` * `` 整型值，为优先级，可以为0-2^32（4,294,967,295），值越小优先级越高，默认为1024。 * `` 整型值，延迟ready的秒数，在这段时间job为delayed状态。 * `` -- time to run --整型值，允许worker执行的最大秒数，如果worker在这段时间不能delete，release，bury job，那么job超时，服务器将release此job，此job的状态迁移为ready。最小为1秒，如果客户端指定为0将会被重置为1。 * `` 整型值，job body的长度，不包含\r\n，这个值必须小于max-job-size，默认为2^16。 * `` job body 响应 ``` INSERTED \r\n ``` 表示插入job成功，id为新job的任务标识，整型值 ``` BURIED \r\n ``` 如服务器为了增加队列的优先级而，内存不足时返回，id为新job的任务标识，整型值 ``` EXPECTED_CRLF\r\n ``` job body必须以\r\n结尾 ``` JOB_TOO_BIG\r\n ``` job body的长度超过max-job-size ``` DRAINING\r\n ``` 表示服务器资源耗尽，表示服务器已经进入了“drain mode”，服务器再也不能接受连接，客户端应该使用另一个服务器或者断开稍后重试 #### `use` 说明 producer生产者使用，随后使用put命令，将job放置于对应的tube 格式 ``` use \r\n tube tube的名称，最大为200字节，不存在时将自动创建 ``` 响应 ``` USING \r\n tube为正在使用的tube名称 ``` 消费者指令说明（Worker Commands） #### `reserve` 说明 取出（预订）job，待处理。它将返回一个新预订的job，如果没有job，beanstalkd将直到有job时才发送响应。一旦job状态迁移为reserved,取出job的client被限制在指定的时间（如果设置了ttr）完成，否则超时，job状态重装迁移为ready。 格式 ``` reserve\r\n ``` 可选的一个相似的命令 `reserve-with-timeout \r\n` 设置取job的超时时间，timeout设置为0时，服务器立即响应或者TIMED_OUT，积极的设置超时，将会限制客户端阻塞在取job的请求的时间。 ##### 失败响应 ``` DEADLINE_SOON\r\n ``` 在一个预定的任务的运行时间内,最后一秒会被服务器保持为一个安全边际，在此期间，客户端将无法等候另外一个任务。 如果客户端在安全隔离期间发出一个预留命令，或者安全隔离期到了,客户端在等候一个预定命令。 ``` TIMED_OUT\r\n 超时 ``` ##### 成功响应 ``` RESERVED \r\n \r\n ``` 成功取出job，id为job id,整型值，job body的长度，不包含\r\n，data为job body #### `delete` 说明 从队列中删除一个job 格式 ``` delete \r\n ``` id为job id 响应 DELETED\r\n 删除成功 NOT_FOUND\r\n job不存在时，或者job的状态不为ready和buried（这种情况是在job执行超时之前，client发送了delete指令） #### `release` 说明 release指令将一个reserved的job放回ready queue。它通常在job执行失败时使用。 格式 ``` release \r\n ``` id 为job id，pri为job的优先级，delay为延迟ready的秒数 响应 RELEASED\r\n 表明成功 BURIED\r\n 如服务器为了增加队列的优先级而，内存不足时返回 NOT_FOUND\r\n 如果job不存在或者client没有预订此job #### `bury` 说明 将一个job的状态迁移为buried，通过kick命令唤醒 格式 ``` bury \r\n ``` id为job id，pri为优先级 响应 BURIED\r\n 表明成功 NOT_FOUND\r\n 如果job不存在或者client没有预订此job #### `touch` 说明 允许worker请求更多的时间执行job，这个很有用当job需要很长的时间来执行，worker可用周期的告诉服务器它仍然在执行job（可以被DEADLINE_SOON触发） 格式 ``` touch \r\n ``` id为job id 响应 TOUCHED\r\n 表明成功 NOT_FOUND\r\n 如果job不存在或者client没有预订此job #### `watch` 说明 添加监控的tube到watch list列表，reserve指令将会从监控的tube列表获取job，对于每个连接，监控的列表默认为default 格式 ``` watch \r\n ``` tube 为监控的tube名称，名称最大为200字节，如果tube不存在会自动创建 响应 ``` WATCHING \r\n 表明成功 ``` count 整型值，已监控的tube数量 #### `ignore` 说明 从已监控的watch list列表中移出特定的tube 格式 ``` ignore \r\n ``` tube 为移出的tube名称，名称最多为200字节，如果tube不存在会自动创建 响应 ``` WATCHING \r\n 表明成功 ``` count 整型值，已监控的tube数量 NOT_IGNORED\r\n 如果client企图忽略其仅有的tube时的响应 其他指令说明（Other Command） #### `peek` 说明 让client在系统中检查job，有四种形式的命令，其中第一种形式的指令是针对当前使用的tube 格式 ``` peek \r\n 返回id对应的job peek-ready\r\n 返回下一个ready job peek-delayed\r\n 返回下一个延迟剩余时间最短的job peek-buried\r\n 返回下一个在buried列表中的job ``` 响应 NOT_FOUND\r\n 如果job不存在，或者没有对应状态的job ``` FOUND \r\n \r\n ``` id 为对应的job id bytes job body的字节数 data 为job body #### `kick` 说明 此指令应用在当前使用的tube中，它将job的状态迁移为ready或者delayed 格式 ``` kick \r\n ``` bound 整型值，唤醒的job上限 响应 ``` KICKED \r\n ``` count 为真实唤醒的job数量 kick-job 说明 kick指令的一个变体，可以使单个job被唤醒，使一个状态为buried或者delayed的job迁移为ready，所有的状态迁移都在相同的tube中完成 格式 ``` kick-job \r\n ``` id 为job id 响应 NOT_FOUND\r\n 如果job不存在，或者job是不可唤醒的状态 KICKED\r\n 表明成功 #### `stats-job` 说明 统计job的相关信息 格式 ``` stats-job \r\n ``` id 为job id 响应 ``` NOT_FOUND\r\n 如果job不存在 OK \r\n\r\n ``` bytes 为接下来的data区块的长度 data 为YAML file的统计信息 其中YAML file包括的key有： - `id` 表示job id - `tube` 表示tube的名称 - `state` 表示job的当前状态 - `pri` 表示job的优先级 - `age` 表示job创建的时间单位秒 - `delay` 是延迟job放入ready队列的整数秒数 - `ttr` 指允许worker执行job的整数秒数 - `time-left` 表示job的状态迁移为ready的时间，仅在job状态为reserved或者delayed时有意义，当job状态为reserved时表示剩余的超时时间。 - `file` 表示包含此job的binlog序号，如果没有开启它将为0 - `reserves` 表示job被reserved的次数 - `timeouts` 表示job出现超时的次数 - `releases` 表示job被released的次数 - `buries` 表示job被buried的次数 - `kicks` 表示job被kiced的次数 #### `stats-tube` **说明** 统计tube的相关信息 **格式** ``` stats-tube \r\n ``` tube 为对应的tube的名称，最多为200字节 **响应** ``` NOT_FOUND\r\n 如果tube不存在 OK \r\n\r\n ``` bytes 为接下来的data区块的长度 data 为YAML file的统计信息 其中YAML file包括的key有： - `name` 表示tube的名称 - `current-jobs-urgent` 此tube中优先级小于1024状态为ready的job数量 - `current-jobs-ready` 此tube中状态为ready的job数量 - `current-jobs-reserved` 此tube中状态为reserved的job数量 - `current-jobs-delayed` 此tube中状态为delayed的job数量 - `current-jobs-buried` 此tube中状态为buried的job数量 - `total-jobs` 此tube中创建的所有job数量 - `current-using` 使用此tube打开的连接数 - `current-wating` 使用此tube打开连接并且等待响应的连接数 - `current-watching` 打开的连接监控此tube的数量 - `pause` 此tube暂停的秒数 - `cmd-delete` 此tube中总共执行的delete指令的次数 - `cmd-pause-tube` 此tube中总共执行pause-tube指令的次数 - `pause-time-left` 此tube暂停剩余的秒数 #### `stats` **说明** 返回整个消息队列系统的整体信息 **格式** ``` stats\r\n ``` **响应** ``` OK \r\n\r\n ``` bytes 为接下来的data区块的长度 data 为YAML file的统计信息 其中YAML file包括的key有（所有的信息都累积的，自从beanstalkd进程启动以来，这些信息不储存在binlog中）： - `current-jobs-urgent` 优先级小于1024状态为ready的job数量 - `current-jobs-ready` 状态为ready的job数量 - `current-jobs-reserved` 状态为reserved的job数量 - `current-jobs-delayed` 状态为delayed的job数量 - `current-jobs-buried` 状态为buried的job数量 - `cmd-put` 总共执行put指令的次数 - `cmd-peek` 总共执行peek指令的次数 - `cmd-peek-ready` 总共执行peek-ready指令的次数 - `cmd-peek-delayed` 总共执行peek-delayed指令的次数 - `cmd-peek-buried` 总共执行peek-buried指令的次数 - `cmd-reserve` 总共执行reserve指令的次数 - `cmd-use` 总共执行use指令的次数 - `cmd-watch` 总共执行watch指令的次数 - `cmd-ignore` 总共执行ignore指令的次数 - `cmd-release` 总共执行release指令的次数 - `cmd-bury` 总共执行bury指令的次数 - `cmd-kick` 总共执行kick指令的次数 - `cmd-stats` 总共执行stats指令的次数 - `cmd-stats-job` 总共执行stats-job指令的次数 - `cmd-stats-tube` 总共执行stats-tube指令的次数 - `cmd-list-tubes` 总共执行list-tubes指令的次数 - `cmd-list-tube-used` 总共执行list-tube-used指令的次数 - `cmd-list-butes-watched` 总共执行list-tubes-watched指令的次数 - `cmd-pause-tube` 总共执行pause-tube指令的次数 - `job-timeouts` 所有超时的job的总共数量 - `total-jobs` 创建的所有job数量 - `max-job-size` job的数据部分最大长度 - `current-tubes` 当前存在的tube数量 - `current-connections` 当前打开的连接数 - `current-producers` 当前所有的打开的连接中至少执行一次put指令的连接数量 - `current-workers` 当前所有的打开的连接中至少执行一次reserve指令的连接数量 - `current-waiting` 当前所有的打开的连接中执行reserve指令但是未响应的连接数量 - `total-connections` 总共处理的连接数 - `pid` 服务器进程的id - `version` 服务器版本号 - `rusage-utime` 进程总共占用的用户CPU时间 - `rusage-stime` 进程总共占用的系统CPU时间 - `uptime` 服务器进程运行的秒数 - `binlog-oldest-index` 开始储存jobs的binlog索引号 - `binlog-current-index` 当前储存jobs的binlog索引号 - `binlog-max-size binlog`的最大容量 - `binlog-records-written` binlog累积写入的记录数 - `binlog-records-migrated` is the cumulative number of records written as part of compaction. - `id` 一个随机字符串，在beanstalkd进程启动时产生 - `hostname` 主机名 #### `list-tubes` 说明 列表所有存在的tube 格式 ``` list-tubes\r\n ``` 响应 ``` OK \r\n \r\n ``` bytes 为接下来的data区块的长度 data 为YAML file，包含所有的tube名称 #### `list-tube-used` 说明 列表当前client正在use的tube 格式 ``` list-tube-used\r\n ``` 响应 ``` USING \r\n ``` tube 为tube名称 #### `list-tubes-watched` 说明 列表当前client watch的tube 格式 ``` list-tubes-watched\r\n ``` 响应 ``` OK \r\n \r\n ``` bytes 为接下来的data区块的长度 data 为YAML file，包含所有的tube名称 #### `quit` 说明 关闭连接 格式 ``` quit\r\n ``` #### `pause-tube` ##### 说明 此指令针对特定的tube内所有新的job延迟给定的秒数 ##### 格式 ``` pause-tube \r\n ``` ##### 响应 ``` PAUSED\r\n 表示成功 NOT_FOUND\r\n tube不存在 ``` >Translated by PHPBoy :http://www.phpboy.net/ and fzb.me beanstalkd-1.11/file.c000066400000000000000000000277731350566611300146550ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include "dat.h" static int readrec(File*, job, int*); static int readrec5(File*, job, int*); static int readfull(File*, void*, int, int*, char*); static void warnpos(File*, int, char*, ...) __attribute__((format(printf, 3, 4))); FAlloc *falloc = &rawfalloc; enum { Walver5 = 5 }; typedef struct Jobrec5 Jobrec5; struct Jobrec5 { uint64 id; uint32 pri; uint64 delay; // usec uint64 ttr; // usec int32 body_size; uint64 created_at; // usec uint64 deadline_at; // usec uint32 reserve_ct; uint32 timeout_ct; uint32 release_ct; uint32 bury_ct; uint32 kick_ct; byte state; char pad[1]; }; enum { Jobrec5size = offsetof(Jobrec5, pad) }; void fileincref(File *f) { if (!f) return; f->refs++; } void filedecref(File *f) { if (!f) return; f->refs--; if (f->refs < 1) { walgc(f->w); } } void fileaddjob(File *f, job j) { job h; h = &f->jlist; if (!h->fprev) h->fprev = h; j->file = f; j->fprev = h->fprev; j->fnext = h; h->fprev->fnext = j; h->fprev = j; fileincref(f); } void filermjob(File *f, job j) { if (!f) return; if (f != j->file) return; j->fnext->fprev = j->fprev; j->fprev->fnext = j->fnext; j->fnext = 0; j->fprev = 0; j->file = NULL; f->w->alive -= j->walused; j->walused = 0; filedecref(f); } // Fileread reads jobs from f->path into list. // It returns 0 on success, or 1 if any errors occurred. int fileread(File *f, job list) { int err = 0, v; if (!readfull(f, &v, sizeof(v), &err, "version")) { return err; } switch (v) { case Walver: fileincref(f); while (readrec(f, list, &err)); filedecref(f); return err; case Walver5: fileincref(f); while (readrec5(f, list, &err)); filedecref(f); return err; } warnx("%s: unknown version: %d", f->path, v); return 1; } // Readrec reads a record from f->fd into linked list l. // If an error occurs, it sets *err to 1. // Readrec returns the number of records read, either 1 or 0. static int readrec(File *f, job l, int *err) { int r, sz = 0; int namelen; Jobrec jr; job j; tube t; char tubename[MAX_TUBE_NAME_LEN]; r = read(f->fd, &namelen, sizeof(int)); if (r == -1) { twarn("read"); warnpos(f, 0, "error"); *err = 1; return 0; } if (r != sizeof(int)) { return 0; } sz += r; if (namelen >= MAX_TUBE_NAME_LEN) { warnpos(f, -r, "namelen %d exceeds maximum of %d", namelen, MAX_TUBE_NAME_LEN - 1); *err = 1; return 0; } if (namelen < 0) { warnpos(f, -r, "namelen %d is negative", namelen); *err = 1; return 0; } if (namelen) { r = readfull(f, tubename, namelen, err, "tube name"); if (!r) { return 0; } sz += r; } tubename[namelen] = '\0'; r = readfull(f, &jr, sizeof(Jobrec), err, "job struct"); if (!r) { return 0; } sz += r; // are we reading trailing zeroes? if (!jr.id) return 0; j = job_find(jr.id); if (!(j || namelen)) { // We read a short record without having seen a // full record for this job, so the full record // was in an earlier file that has been deleted. // Therefore the job itself has either been // deleted or migrated; either way, this record // should be ignored. return 1; } switch (jr.state) { case Reserved: jr.state = Ready; case Ready: case Buried: case Delayed: if (!j) { if (jr.body_size > job_data_size_limit) { warnpos(f, -r, "job %"PRIu64" is too big (%"PRId32" > %zu)", jr.id, jr.body_size, job_data_size_limit); goto Error; } t = tube_find_or_make(tubename); j = make_job_with_id(jr.pri, jr.delay, jr.ttr, jr.body_size, t, jr.id); j->next = j->prev = j; j->r.created_at = jr.created_at; } j->r = jr; job_insert(l, j); // full record; read the job body if (namelen) { if (jr.body_size != j->r.body_size) { warnpos(f, -r, "job %"PRIu64" size changed", j->r.id); warnpos(f, -r, "was %d, now %d", j->r.body_size, jr.body_size); goto Error; } r = readfull(f, j->body, j->r.body_size, err, "job body"); if (!r) { goto Error; } sz += r; // since this is a full record, we can move // the file pointer and decref the old // file, if any filermjob(j->file, j); fileaddjob(f, j); } j->walused += sz; f->w->alive += sz; return 1; case Invalid: if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 1; } Error: *err = 1; if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 0; } // Readrec5 is like readrec, but it reads a record in "version 5" // of the log format. static int readrec5(File *f, job l, int *err) { int r, sz = 0; size_t namelen; Jobrec5 jr; job j; tube t; char tubename[MAX_TUBE_NAME_LEN]; r = read(f->fd, &namelen, sizeof(namelen)); if (r == -1) { twarn("read"); warnpos(f, 0, "error"); *err = 1; return 0; } if (r != sizeof(namelen)) { return 0; } sz += r; if (namelen >= MAX_TUBE_NAME_LEN) { warnpos(f, -r, "namelen %zu exceeds maximum of %d", namelen, MAX_TUBE_NAME_LEN - 1); *err = 1; return 0; } if (namelen) { r = readfull(f, tubename, namelen, err, "v5 tube name"); if (!r) { return 0; } sz += r; } tubename[namelen] = '\0'; r = readfull(f, &jr, Jobrec5size, err, "v5 job struct"); if (!r) { return 0; } sz += r; // are we reading trailing zeroes? if (!jr.id) return 0; j = job_find(jr.id); if (!(j || namelen)) { // We read a short record without having seen a // full record for this job, so the full record // was in an eariler file that has been deleted. // Therefore the job itself has either been // deleted or migrated; either way, this record // should be ignored. return 1; } switch (jr.state) { case Reserved: jr.state = Ready; case Ready: case Buried: case Delayed: if (!j) { if (jr.body_size > job_data_size_limit) { warnpos(f, -r, "job %"PRIu64" is too big (%"PRId32" > %zu)", jr.id, jr.body_size, job_data_size_limit); goto Error; } t = tube_find_or_make(tubename); j = make_job_with_id(jr.pri, jr.delay, jr.ttr, jr.body_size, t, jr.id); j->next = j->prev = j; j->r.created_at = jr.created_at; } j->r.id = jr.id; j->r.pri = jr.pri; j->r.delay = jr.delay * 1000; // us => ns j->r.ttr = jr.ttr * 1000; // us => ns j->r.body_size = jr.body_size; j->r.created_at = jr.created_at * 1000; // us => ns j->r.deadline_at = jr.deadline_at * 1000; // us => ns j->r.reserve_ct = jr.reserve_ct; j->r.timeout_ct = jr.timeout_ct; j->r.release_ct = jr.release_ct; j->r.bury_ct = jr.bury_ct; j->r.kick_ct = jr.kick_ct; j->r.state = jr.state; job_insert(l, j); // full record; read the job body if (namelen) { if (jr.body_size != j->r.body_size) { warnpos(f, -r, "job %"PRIu64" size changed", j->r.id); warnpos(f, -r, "was %"PRId32", now %"PRId32, j->r.body_size, jr.body_size); goto Error; } r = readfull(f, j->body, j->r.body_size, err, "v5 job body"); if (!r) { goto Error; } sz += r; // since this is a full record, we can move // the file pointer and decref the old // file, if any filermjob(j->file, j); fileaddjob(f, j); } j->walused += sz; f->w->alive += sz; return 1; case Invalid: if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 1; } Error: *err = 1; if (j) { job_remove(j); filermjob(j->file, j); job_free(j); } return 0; } static int readfull(File *f, void *c, int n, int *err, char *desc) { int r; r = read(f->fd, c, n); if (r == -1) { twarn("read"); warnpos(f, 0, "error reading %s", desc); *err = 1; return 0; } if (r != n) { warnpos(f, -r, "unexpected EOF reading %d bytes (got %d): %s", n, r, desc); *err = 1; return 0; } return r; } static void warnpos(File *f, int adj, char *fmt, ...) { int off; va_list ap; off = lseek(f->fd, 0, SEEK_CUR); fprintf(stderr, "%s:%u: ", f->path, off+adj); va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); fputc('\n', stderr); } // Opens f for writing, writes a header, and initializes // f->free and f->resv. // Sets f->iswopen if successful. void filewopen(File *f) { int fd, r; int n; int ver = Walver; fd = open(f->path, O_WRONLY|O_CREAT, 0400); if (fd < 0) { twarn("open %s", f->path); return; } r = falloc(fd, f->w->filesize); if (r) { close(fd); errno = r; twarn("falloc %s", f->path); r = unlink(f->path); if (r) { twarn("unlink %s", f->path); } return; } n = write(fd, &ver, sizeof(int)); if (n < sizeof(int)) { twarn("write %s", f->path); close(fd); return; } f->fd = fd; f->iswopen = 1; fileincref(f); f->free = f->w->filesize - n; f->resv = 0; } static int filewrite(File *f, job j, void *buf, int len) { int r; r = write(f->fd, buf, len); if (r != len) { twarn("write"); return 0; } f->w->resv -= r; f->resv -= r; j->walresv -= r; j->walused += r; f->w->alive += r; return 1; } int filewrjobshort(File *f, job j) { int r, nl; nl = 0; // name len 0 indicates short record r = filewrite(f, j, &nl, sizeof nl) && filewrite(f, j, &j->r, sizeof j->r); if (!r) return 0; if (j->r.state == Invalid) { filermjob(j->file, j); } return r; } int filewrjobfull(File *f, job j) { int nl; fileaddjob(f, j); nl = strlen(j->tube->name); return filewrite(f, j, &nl, sizeof nl) && filewrite(f, j, j->tube->name, nl) && filewrite(f, j, &j->r, sizeof j->r) && filewrite(f, j, j->body, j->r.body_size); } void filewclose(File *f) { if (!f) return; if (!f->iswopen) return; if (f->free) { (void)ftruncate(f->fd, f->w->filesize - f->free); } close(f->fd); f->iswopen = 0; filedecref(f); } int fileinit(File *f, Wal *w, int n) { f->w = w; f->seq = n; f->path = fmtalloc("%s/binlog.%d", w->dir, n); return !!f->path; } // Adds f to the linked list in w, // updating w->tail and w->head as necessary. Wal* fileadd(File *f, Wal *w) { if (w->tail) { w->tail->next = f; } w->tail = f; if (!w->head) { w->head = f; } w->nfile++; return w; } beanstalkd-1.11/freebsd.c000066400000000000000000000000241350566611300153240ustar00rootroot00000000000000#include "darwin.c" beanstalkd-1.11/heap.c000066400000000000000000000035071350566611300146400ustar00rootroot00000000000000#include #include #include #include "dat.h" static void set(Heap *h, int k, void *x) { h->data[k] = x; h->rec(x, k); } static void swap(Heap *h, int a, int b) { void *tmp; tmp = h->data[a]; set(h, a, h->data[b]); set(h, b, tmp); } static int less(Heap *h, int a, int b) { return h->less(h->data[a], h->data[b]); } static void siftdown(Heap *h, int k) { for (;;) { int p = (k-1) / 2; /* parent */ if (k == 0 || less(h, p, k)) { return; } swap(h, k, p); k = p; } } static void siftup(Heap *h, int k) { for (;;) { int l, r, s; l = k*2 + 1; /* left child */ r = k*2 + 2; /* right child */ /* find the smallest of the three */ s = k; if (l < h->len && less(h, l, s)) s = l; if (r < h->len && less(h, r, s)) s = r; if (s == k) { return; /* satisfies the heap property */ } swap(h, k, s); k = s; } } // Heapinsert inserts x into heap h according to h->less. // It returns 1 on success, otherwise 0. int heapinsert(Heap *h, void *x) { int k; if (h->len == h->cap) { void **ndata; int ncap = (h->len+1) * 2; /* allocate twice what we need */ ndata = malloc(sizeof(void*) * ncap); if (!ndata) { return 0; } memcpy(ndata, h->data, sizeof(void*)*h->len); free(h->data); h->data = ndata; h->cap = ncap; } k = h->len; h->len++; set(h, k, x); siftdown(h, k); return 1; } void * heapremove(Heap *h, int k) { void *x; if (k >= h->len) { return 0; } x = h->data[k]; h->len--; set(h, k, h->data[h->len]); siftdown(h, k); siftup(h, k); h->rec(x, -1); return x; } beanstalkd-1.11/job.c000066400000000000000000000116751350566611300145020ustar00rootroot00000000000000#include #include #include #include "dat.h" static uint64 next_id = 1; static int cur_prime = 0; static job all_jobs_init[12289] = {0}; static job *all_jobs = all_jobs_init; static size_t all_jobs_cap = 12289; /* == primes[0] */ static size_t all_jobs_used = 0; static int hash_table_was_oom = 0; static void rehash(int); static int _get_job_hash_index(uint64 job_id) { return job_id % all_jobs_cap; } static void store_job(job j) { int index = 0; index = _get_job_hash_index(j->r.id); j->ht_next = all_jobs[index]; all_jobs[index] = j; all_jobs_used++; /* accept a load factor of 4 */ if (all_jobs_used > (all_jobs_cap << 2)) rehash(1); } static void rehash(int is_upscaling) { job *old = all_jobs; size_t old_cap = all_jobs_cap, old_used = all_jobs_used, i; int old_prime = cur_prime; int d = is_upscaling ? 1 : -1; if (cur_prime + d >= NUM_PRIMES) return; if (cur_prime + d < 0) return; if (is_upscaling && hash_table_was_oom) return; cur_prime += d; all_jobs_cap = primes[cur_prime]; all_jobs = calloc(all_jobs_cap, sizeof(job)); if (!all_jobs) { twarnx("Failed to allocate %zu new hash buckets", all_jobs_cap); hash_table_was_oom = 1; cur_prime = old_prime; all_jobs = old; all_jobs_cap = old_cap; all_jobs_used = old_used; return; } all_jobs_used = 0; hash_table_was_oom = 0; for (i = 0; i < old_cap; i++) { while (old[i]) { job j = old[i]; old[i] = j->ht_next; j->ht_next = NULL; store_job(j); } } if (old != all_jobs_init) { free(old); } } job job_find(uint64 job_id) { job jh = NULL; int index = _get_job_hash_index(job_id); for (jh = all_jobs[index]; jh && jh->r.id != job_id; jh = jh->ht_next); return jh; } job allocate_job(int body_size) { job j; j = malloc(sizeof(struct job) + body_size); if (!j) return twarnx("OOM"), (job) 0; memset(j, 0, sizeof(struct job)); j->r.created_at = nanoseconds(); j->r.body_size = body_size; j->next = j->prev = j; /* not in a linked list */ return j; } job make_job_with_id(uint pri, int64 delay, int64 ttr, int body_size, tube tube, uint64 id) { job j; j = allocate_job(body_size); if (!j) return twarnx("OOM"), (job) 0; if (id) { j->r.id = id; if (id >= next_id) next_id = id + 1; } else { j->r.id = next_id++; } j->r.pri = pri; j->r.delay = delay; j->r.ttr = ttr; store_job(j); TUBE_ASSIGN(j->tube, tube); return j; } static void job_hash_free(job j) { job *slot; slot = &all_jobs[_get_job_hash_index(j->r.id)]; while (*slot && *slot != j) slot = &(*slot)->ht_next; if (*slot) { *slot = (*slot)->ht_next; --all_jobs_used; } // Downscale when the hashmap is too sparse if (all_jobs_used < (all_jobs_cap >> 4)) rehash(0); } void job_free(job j) { if (j) { TUBE_ASSIGN(j->tube, NULL); if (j->r.state != Copy) job_hash_free(j); } free(j); } void job_setheappos(void *j, int pos) { ((job)j)->heap_index = pos; } int job_pri_less(void *ax, void *bx) { job a = ax, b = bx; if (a->r.pri < b->r.pri) return 1; if (a->r.pri > b->r.pri) return 0; return a->r.id < b->r.id; } int job_delay_less(void *ax, void *bx) { job a = ax, b = bx; if (a->r.deadline_at < b->r.deadline_at) return 1; if (a->r.deadline_at > b->r.deadline_at) return 0; return a->r.id < b->r.id; } job job_copy(job j) { job n; if (!j) return NULL; n = malloc(sizeof(struct job) + j->r.body_size); if (!n) return twarnx("OOM"), (job) 0; memcpy(n, j, sizeof(struct job) + j->r.body_size); n->next = n->prev = n; /* not in a linked list */ n->file = NULL; /* copies do not have refcnt on the wal */ n->tube = 0; /* Don't use memcpy for the tube, which we must refcount. */ TUBE_ASSIGN(n->tube, j->tube); /* Mark this job as a copy so it can be appropriately freed later on */ n->r.state = Copy; return n; } const char * job_state(job j) { if (j->r.state == Ready) return "ready"; if (j->r.state == Reserved) return "reserved"; if (j->r.state == Buried) return "buried"; if (j->r.state == Delayed) return "delayed"; return "invalid"; } int job_list_any_p(job head) { return head->next != head || head->prev != head; } job job_remove(job j) { if (!j) return NULL; if (!job_list_any_p(j)) return NULL; /* not in a doubly-linked list */ j->next->prev = j->prev; j->prev->next = j->next; j->prev = j->next = j; return j; } void job_insert(job head, job j) { if (job_list_any_p(j)) return; /* already in a linked list */ j->prev = head->prev; j->next = head; head->prev->next = j; head->prev = j; } /* for unit tests */ size_t get_all_jobs_used() { return all_jobs_used; } beanstalkd-1.11/linux.c000066400000000000000000000032321350566611300150550ustar00rootroot00000000000000#define _XOPEN_SOURCE 600 #include #include #include #include #include #include #include #include "dat.h" #ifndef EPOLLRDHUP #define EPOLLRDHUP 0x2000 #endif static int epfd; /* Allocate disk space. * Expects fd's offset to be 0; may also reset fd's offset to 0. * Returns 0 on success, and a positive errno otherwise. */ int rawfalloc(int fd, int len) { return ftruncate(fd, len); } int sockinit(void) { epfd = epoll_create(1); if (epfd == -1) { twarn("epoll_create"); return -1; } return 0; } int sockwant(Socket *s, int rw) { int op; struct epoll_event ev = {}; if (!s->added && !rw) { return 0; } else if (!s->added && rw) { s->added = 1; op = EPOLL_CTL_ADD; } else if (!rw) { op = EPOLL_CTL_DEL; } else { op = EPOLL_CTL_MOD; } switch (rw) { case 'r': ev.events = EPOLLIN; break; case 'w': ev.events = EPOLLOUT; break; } ev.events |= EPOLLRDHUP | EPOLLPRI; ev.data.ptr = s; return epoll_ctl(epfd, op, s->fd, &ev); } int socknext(Socket **s, int64 timeout) { int r; struct epoll_event ev; r = epoll_wait(epfd, &ev, 1, (int)(timeout/1000000)); if (r == -1 && errno != EINTR) { twarn("epoll_wait"); exit(1); } if (r > 0) { *s = ev.data.ptr; if (ev.events & (EPOLLHUP|EPOLLRDHUP)) { return 'h'; } else if (ev.events & EPOLLIN) { return 'r'; } else if (ev.events & EPOLLOUT) { return 'w'; } } return 0; } beanstalkd-1.11/main.c000066400000000000000000000040441350566611300146440ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include "dat.h" static void su(const char *user) { int r; struct passwd *pwent; errno = 0; pwent = getpwnam(user); if (errno) twarn("getpwnam(\"%s\")", user), exit(32); if (!pwent) twarnx("getpwnam(\"%s\"): no such user", user), exit(33); r = setgid(pwent->pw_gid); if (r == -1) twarn("setgid(%d \"%s\")", pwent->pw_gid, user), exit(34); r = setuid(pwent->pw_uid); if (r == -1) twarn("setuid(%d \"%s\")", pwent->pw_uid, user), exit(34); } static void set_sig_handlers() { int r; struct sigaction sa; sa.sa_handler = SIG_IGN; sa.sa_flags = 0; r = sigemptyset(&sa.sa_mask); if (r == -1) twarn("sigemptyset()"), exit(111); r = sigaction(SIGPIPE, &sa, 0); if (r == -1) twarn("sigaction(SIGPIPE)"), exit(111); sa.sa_handler = enter_drain_mode; r = sigaction(SIGUSR1, &sa, 0); if (r == -1) twarn("sigaction(SIGUSR1)"), exit(111); } int main(int argc, char **argv) { int r; struct job list = {}; progname = argv[0]; setlinebuf(stdout); optparse(&srv, argv+1); if (verbose) { printf("pid %d\n", getpid()); } r = make_server_socket(srv.addr, srv.port); if (r == -1) twarnx("make_server_socket()"), exit(111); srv.sock.fd = r; prot_init(); if (srv.user) su(srv.user); set_sig_handlers(); if (srv.wal.use) { // We want to make sure that only one beanstalkd tries // to use the wal directory at a time. So acquire a lock // now and never release it. if (!waldirlock(&srv.wal)) { twarnx("failed to lock wal dir %s", srv.wal.dir); exit(10); } list.prev = list.next = &list; walinit(&srv.wal, &list); r = prot_replay(&srv, &list); if (!r) { twarnx("failed to replay log"); return 1; } } srvserve(&srv); return 0; } beanstalkd-1.11/ms.c000066400000000000000000000031421350566611300143350ustar00rootroot00000000000000#include #include #include #include "dat.h" void ms_init(ms a, ms_event_fn oninsert, ms_event_fn onremove) { a->used = a->cap = a->last = 0; a->items = NULL; a->oninsert = oninsert; a->onremove = onremove; } static void grow(ms a) { void **nitems; size_t ncap = (a->cap << 1) ? : 1; nitems = malloc(ncap * sizeof(void *)); if (!nitems) return; memcpy(nitems, a->items, a->used * sizeof(void *)); free(a->items); a->items = nitems; a->cap = ncap; } int ms_append(ms a, void *item) { if (a->used >= a->cap) grow(a); if (a->used >= a->cap) return 0; a->items[a->used++] = item; if (a->oninsert) a->oninsert(a, item, a->used - 1); return 1; } static int ms_delete(ms a, size_t i) { void *item; if (i >= a->used) return 0; item = a->items[i]; a->items[i] = a->items[--a->used]; /* it has already been removed now */ if (a->onremove) a->onremove(a, item, i); return 1; } void ms_clear(ms a) { while (ms_delete(a, 0)); free(a->items); ms_init(a, a->oninsert, a->onremove); } int ms_remove(ms a, void *item) { size_t i; for (i = 0; i < a->used; i++) { if (a->items[i] == item) return ms_delete(a, i); } return 0; } int ms_contains(ms a, void *item) { size_t i; for (i = 0; i < a->used; i++) { if (a->items[i] == item) return 1; } return 0; } void * ms_take(ms a) { void *item; if (!a->used) return NULL; a->last = a->last % a->used; item = a->items[a->last]; ms_delete(a, a->last); ++a->last; return item; } beanstalkd-1.11/net.c000066400000000000000000000074131350566611300145110ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include "dat.h" #include "sd-daemon.h" int make_server_socket(char *host, char *port) { int fd = -1, flags, r; struct linger linger = {0, 0}; struct addrinfo *airoot, *ai, hints; /* See if we got a listen fd from systemd. If so, all socket options etc * are already set, so we check that the fd is a TCP listen socket and * return. */ r = sd_listen_fds(1); if (r < 0) { return twarn("sd_listen_fds"), -1; } if (r > 0) { if (r > 1) { twarnx("inherited more than one listen socket;" " ignoring all but the first"); } fd = SD_LISTEN_FDS_START; r = sd_is_socket_inet(fd, 0, SOCK_STREAM, 1, 0); if (r < 0) { errno = -r; twarn("sd_is_socket_inet"); return -1; } if (!r) { twarnx("inherited fd is not a TCP listen socket"); return -1; } return fd; } memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; r = getaddrinfo(host, port, &hints, &airoot); if (r != 0) { twarnx("getaddrinfo(): %s", gai_strerror(r)); return -1; } for(ai = airoot; ai; ai = ai->ai_next) { fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); if (fd == -1) { twarn("socket()"); continue; } flags = fcntl(fd, F_GETFL, 0); if (flags < 0) { twarn("getting flags"); close(fd); continue; } r = fcntl(fd, F_SETFL, flags | O_NONBLOCK); if (r == -1) { twarn("setting O_NONBLOCK"); close(fd); continue; } flags = 1; r = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof flags); if (r == -1) { twarn("setting SO_REUSEADDR on fd %d", fd); close(fd); continue; } r = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &flags, sizeof flags); if (r == -1) { twarn("setting SO_KEEPALIVE on fd %d", fd); close(fd); continue; } r = setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger, sizeof linger); if (r == -1) { twarn("setting SO_LINGER on fd %d", fd); close(fd); continue; } r = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof flags); if (r == -1) { twarn("setting TCP_NODELAY on fd %d", fd); close(fd); continue; } r = bind(fd, ai->ai_addr, ai->ai_addrlen); if (r == -1) { twarn("bind()"); close(fd); continue; } if (verbose) { char hbuf[NI_MAXHOST], pbuf[NI_MAXSERV], *h = host, *p = port; struct sockaddr_in addr; socklen_t addrlen; addrlen = sizeof(addr); r = getsockname(fd, (struct sockaddr *) &addr, &addrlen); if (!r) { r = getnameinfo((struct sockaddr *) &addr, addrlen, hbuf, sizeof(hbuf), pbuf, sizeof(pbuf), NI_NUMERICHOST|NI_NUMERICSERV); if (!r) { h = hbuf; p = pbuf; } } if (ai->ai_family == AF_INET6) { printf("bind %d [%s]:%s\n", fd, h, p); } else { printf("bind %d %s:%s\n", fd, h, p); } } r = listen(fd, 1024); if (r == -1) { twarn("listen()"); close(fd); continue; } break; } freeaddrinfo(airoot); if(ai == NULL) fd = -1; return fd; } beanstalkd-1.11/pkg/000077500000000000000000000000001350566611300143335ustar00rootroot00000000000000beanstalkd-1.11/pkg/README.md000066400000000000000000000003761350566611300156200ustar00rootroot00000000000000How to make a release: 1. Check out master 2. Tag the commit to release devX.Y: git tag dev1.11 3. Write a change log in file News (don't commit it) 4. Run pkg/dist.sh and follow the intructions. 5. Push tags and beanstalkd.github.io 6. Run pkg/mail.sh beanstalkd-1.11/pkg/beanstalkd.spec.in000066400000000000000000000047371350566611300177370ustar00rootroot00000000000000%define beanstalkd_user beanstalkd %define beanstalkd_group %{beanstalkd_user} %define beanstalkd_home %{_localstatedir}/lib/beanstalkd %define beanstalkd_logdir %{_localstatedir}/log/beanstalkd Name: beanstalkd Version: @VERSION@ Release: 0%{?dist} Summary: A simple, fast workqueue service Group: System Environment/Daemons License: GPLv3+ URL: http://xph.us/software/%{name}/ Source0: http://xph.us/dist/%{name}/rel/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires(pre): %{_sbindir}/useradd Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig, /sbin/service Requires(postun): /sbin/service %description Beanstalk is a simple, fast workqueue service. Its interface is generic, but was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. %prep %setup -q if [ ! -e configure ]; then sh buildconf.sh fi %build %configure --disable-rpath --docdir=%{_defaultdocdir}/%{name}-%{version} make %{?_smp_mflags} %install rm -rf $RPM_BUILD_ROOT make install DESTDIR=$RPM_BUILD_ROOT %{__install} -p -D -m 0644 doc/%{name}.1 %{buildroot}%{_mandir}/man1/%{name}.1 %{__install} -p -D -m 0755 scripts/%{name}.init %{buildroot}%{_initrddir}/%{name} %{__install} -p -D -m 0644 scripts/%{name}.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} %clean rm -rf $RPM_BUILD_ROOT %pre %{_sbindir}/useradd -c "beanstalkd user" -s /bin/false -r -m -d %{beanstalkd_home} %{beanstalkd_user} 2>/dev/null || : %post /sbin/chkconfig --add %{name} %preun if [ $1 = 0 ]; then /sbin/service %{name} stop >/dev/null 2>&1 /sbin/chkconfig --del %{name} fi %postun if [ $1 -ge 1 ]; then /sbin/service %{name} condrestart > /dev/null 2>&1 || : fi %files %defattr(-,root,root,-) %doc %{_defaultdocdir}/%{name}-%{version}/protocol.txt %doc README COPYING doc/protocol.txt %{_initrddir}/%{name} %{_bindir}/%{name} %{_mandir}/man1/%{name}.1.gz %config(noreplace) %{_sysconfdir}/sysconfig/%{name} %changelog * Thu Oct 1 2009 Keith Rarick - 1.4-0 - Convert this file to an autoconf template - Tweak the summary and description * Sun Jan 4 2009 Ask Bjørn Hansen - 1.2-0 - 1.2-tobe - Use man page and .init/sysconfig scripts from .tar.gz * Sat Nov 22 2008 Jeremy Hinegardner - 1.1-1 - initial spec creation beanstalkd-1.11/pkg/bloghead.in000066400000000000000000000002761350566611300164350ustar00rootroot00000000000000--- layout: post title: Beanstalkd @VERSION@ Release Notes version: @VERSION@ dist: https://github.com/beanstalkd/beanstalkd/archive/v@VERSION@.tar.gz file: beanstalkd-@VERSION@.tar.gz --- beanstalkd-1.11/pkg/dist.sh000077500000000000000000000024111350566611300156330ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -o pipefail exp() { sed s/@VERSION@/$ver/ | sed s/@PARENT@/$prev/ } clean() { rm -f "$GIT_INDEX_FILE" } mkobj() { git hash-object -w --stdin } die() { echo >&2 "$@" exit 2 } ver=`./vers.sh` case $ver in *+*) die bad ver $ver ;; esac prev=`git describe --abbrev=0 --match=dev* --tags dev$ver^|sed s/^dev//` test -n "$prev" || die no prev ver test -f News || die no News export GIT_INDEX_FILE GIT_INDEX_FILE=`mktemp -t beanstalkd-dist-index` trap clean EXIT git read-tree dev$ver newsobj=`cat News pkg/newstail.in|exp|mkobj` versobj=`echo "printf '$ver'"|mkobj` specobj=`exp $postfile echo "Now, run these commands to update the blog:" echo "(mv $postfile ../beanstalkd.github.io/_posts/ && cd ../beanstalkd.github.io && git add . && git commit -m \"announce release $ver\")" beanstalkd-1.11/pkg/mail.sh000077500000000000000000000005131350566611300156130ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -o pipefail die() { echo >&2 "$@" exit 2 } addr=beanstalk-talk@googlegroups.com ver=`./vers.sh` case $ver in *+*) die bad ver $ver ;; esac (cat < Subject: [ANN] beanstalkd $ver end beanstalkd-1.11/pkg/newstail.in000066400000000000000000000007651350566611300165210ustar00rootroot00000000000000Full list of changes (includes authorship information): Our Urls -------- Download the @VERSION@ tarball directly: Learn all about beanstalk: Talk about beanstalk development or use at: Bugs ---- Please report any bugs to: beanstalkd-1.11/primes.c000066400000000000000000000020071350566611300152140ustar00rootroot00000000000000#include // prime // downscale treshold / upscale treshold size_t primes[] = { 12289, // NA / 3072 24593, // 1537 / 6148 49193, // 3074 / 12298 98387, // 6149 / 24596 196799, // etc 393611, 787243, 1574491, 3148987, 6297979, 12595991, 25191989, 50383981, 100767977, 201535967, 403071937, 806143879, 1612287763, 3224575537UL, #if _LP64 6449151103, 12898302233, 25796604473, 51593208973, 103186417951, 206372835917, 412745671837, 825491343683, 1650982687391, 3301965374803, 6603930749621, 13207861499251, 26415722998507, 52831445997037, 105662891994103, 211325783988211, 422651567976461, 845303135952931, 1690606271905871, 3381212543811743, 6762425087623523, 13524850175247127, 27049700350494287, 54099400700988593, 108198801401977301, 216397602803954641, 432795205607909293, 865590411215818597, 1731180822431637217, #endif }; beanstalkd-1.11/prot.c000066400000000000000000001507661350566611300147210ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "dat.h" /* job body cannot be greater than this many bytes long */ size_t job_data_size_limit = JOB_DATA_SIZE_LIMIT_DEFAULT; #define NAME_CHARS \ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" \ "abcdefghijklmnopqrstuvwxyz" \ "0123456789-+/;.$_()" #define CMD_PUT "put " #define CMD_PEEKJOB "peek " #define CMD_PEEK_READY "peek-ready" #define CMD_PEEK_DELAYED "peek-delayed" #define CMD_PEEK_BURIED "peek-buried" #define CMD_RESERVE "reserve" #define CMD_RESERVE_TIMEOUT "reserve-with-timeout " #define CMD_DELETE "delete " #define CMD_RELEASE "release " #define CMD_BURY "bury " #define CMD_KICK "kick " #define CMD_JOBKICK "kick-job " #define CMD_TOUCH "touch " #define CMD_STATS "stats" #define CMD_JOBSTATS "stats-job " #define CMD_USE "use " #define CMD_WATCH "watch " #define CMD_IGNORE "ignore " #define CMD_LIST_TUBES "list-tubes" #define CMD_LIST_TUBE_USED "list-tube-used" #define CMD_LIST_TUBES_WATCHED "list-tubes-watched" #define CMD_STATS_TUBE "stats-tube " #define CMD_QUIT "quit" #define CMD_PAUSE_TUBE "pause-tube" #define CONSTSTRLEN(m) (sizeof(m) - 1) #define CMD_PEEK_READY_LEN CONSTSTRLEN(CMD_PEEK_READY) #define CMD_PEEK_DELAYED_LEN CONSTSTRLEN(CMD_PEEK_DELAYED) #define CMD_PEEK_BURIED_LEN CONSTSTRLEN(CMD_PEEK_BURIED) #define CMD_PEEKJOB_LEN CONSTSTRLEN(CMD_PEEKJOB) #define CMD_RESERVE_LEN CONSTSTRLEN(CMD_RESERVE) #define CMD_RESERVE_TIMEOUT_LEN CONSTSTRLEN(CMD_RESERVE_TIMEOUT) #define CMD_DELETE_LEN CONSTSTRLEN(CMD_DELETE) #define CMD_RELEASE_LEN CONSTSTRLEN(CMD_RELEASE) #define CMD_BURY_LEN CONSTSTRLEN(CMD_BURY) #define CMD_KICK_LEN CONSTSTRLEN(CMD_KICK) #define CMD_JOBKICK_LEN CONSTSTRLEN(CMD_JOBKICK) #define CMD_TOUCH_LEN CONSTSTRLEN(CMD_TOUCH) #define CMD_STATS_LEN CONSTSTRLEN(CMD_STATS) #define CMD_JOBSTATS_LEN CONSTSTRLEN(CMD_JOBSTATS) #define CMD_USE_LEN CONSTSTRLEN(CMD_USE) #define CMD_WATCH_LEN CONSTSTRLEN(CMD_WATCH) #define CMD_IGNORE_LEN CONSTSTRLEN(CMD_IGNORE) #define CMD_LIST_TUBES_LEN CONSTSTRLEN(CMD_LIST_TUBES) #define CMD_LIST_TUBE_USED_LEN CONSTSTRLEN(CMD_LIST_TUBE_USED) #define CMD_LIST_TUBES_WATCHED_LEN CONSTSTRLEN(CMD_LIST_TUBES_WATCHED) #define CMD_STATS_TUBE_LEN CONSTSTRLEN(CMD_STATS_TUBE) #define CMD_PAUSE_TUBE_LEN CONSTSTRLEN(CMD_PAUSE_TUBE) #define MSG_FOUND "FOUND" #define MSG_NOTFOUND "NOT_FOUND\r\n" #define MSG_RESERVED "RESERVED" #define MSG_DEADLINE_SOON "DEADLINE_SOON\r\n" #define MSG_TIMED_OUT "TIMED_OUT\r\n" #define MSG_DELETED "DELETED\r\n" #define MSG_RELEASED "RELEASED\r\n" #define MSG_BURIED "BURIED\r\n" #define MSG_KICKED "KICKED\r\n" #define MSG_TOUCHED "TOUCHED\r\n" #define MSG_BURIED_FMT "BURIED %"PRIu64"\r\n" #define MSG_INSERTED_FMT "INSERTED %"PRIu64"\r\n" #define MSG_NOT_IGNORED "NOT_IGNORED\r\n" #define MSG_NOTFOUND_LEN CONSTSTRLEN(MSG_NOTFOUND) #define MSG_DELETED_LEN CONSTSTRLEN(MSG_DELETED) #define MSG_TOUCHED_LEN CONSTSTRLEN(MSG_TOUCHED) #define MSG_RELEASED_LEN CONSTSTRLEN(MSG_RELEASED) #define MSG_BURIED_LEN CONSTSTRLEN(MSG_BURIED) #define MSG_KICKED_LEN CONSTSTRLEN(MSG_KICKED) #define MSG_NOT_IGNORED_LEN CONSTSTRLEN(MSG_NOT_IGNORED) #define MSG_OUT_OF_MEMORY "OUT_OF_MEMORY\r\n" #define MSG_INTERNAL_ERROR "INTERNAL_ERROR\r\n" #define MSG_DRAINING "DRAINING\r\n" #define MSG_BAD_FORMAT "BAD_FORMAT\r\n" #define MSG_UNKNOWN_COMMAND "UNKNOWN_COMMAND\r\n" #define MSG_EXPECTED_CRLF "EXPECTED_CRLF\r\n" #define MSG_JOB_TOO_BIG "JOB_TOO_BIG\r\n" #define STATE_WANTCOMMAND 0 #define STATE_WANTDATA 1 #define STATE_SENDJOB 2 #define STATE_SENDWORD 3 #define STATE_WAIT 4 #define STATE_BITBUCKET 5 #define STATE_CLOSE 6 #define OP_UNKNOWN 0 #define OP_PUT 1 #define OP_PEEKJOB 2 #define OP_RESERVE 3 #define OP_DELETE 4 #define OP_RELEASE 5 #define OP_BURY 6 #define OP_KICK 7 #define OP_STATS 8 #define OP_JOBSTATS 9 #define OP_PEEK_BURIED 10 #define OP_USE 11 #define OP_WATCH 12 #define OP_IGNORE 13 #define OP_LIST_TUBES 14 #define OP_LIST_TUBE_USED 15 #define OP_LIST_TUBES_WATCHED 16 #define OP_STATS_TUBE 17 #define OP_PEEK_READY 18 #define OP_PEEK_DELAYED 19 #define OP_RESERVE_TIMEOUT 20 #define OP_TOUCH 21 #define OP_QUIT 22 #define OP_PAUSE_TUBE 23 #define OP_JOBKICK 24 #define TOTAL_OPS 25 #define STATS_FMT "---\n" \ "current-jobs-urgent: %u\n" \ "current-jobs-ready: %u\n" \ "current-jobs-reserved: %u\n" \ "current-jobs-delayed: %u\n" \ "current-jobs-buried: %u\n" \ "cmd-put: %" PRIu64 "\n" \ "cmd-peek: %" PRIu64 "\n" \ "cmd-peek-ready: %" PRIu64 "\n" \ "cmd-peek-delayed: %" PRIu64 "\n" \ "cmd-peek-buried: %" PRIu64 "\n" \ "cmd-reserve: %" PRIu64 "\n" \ "cmd-reserve-with-timeout: %" PRIu64 "\n" \ "cmd-delete: %" PRIu64 "\n" \ "cmd-release: %" PRIu64 "\n" \ "cmd-use: %" PRIu64 "\n" \ "cmd-watch: %" PRIu64 "\n" \ "cmd-ignore: %" PRIu64 "\n" \ "cmd-bury: %" PRIu64 "\n" \ "cmd-kick: %" PRIu64 "\n" \ "cmd-touch: %" PRIu64 "\n" \ "cmd-stats: %" PRIu64 "\n" \ "cmd-stats-job: %" PRIu64 "\n" \ "cmd-stats-tube: %" PRIu64 "\n" \ "cmd-list-tubes: %" PRIu64 "\n" \ "cmd-list-tube-used: %" PRIu64 "\n" \ "cmd-list-tubes-watched: %" PRIu64 "\n" \ "cmd-pause-tube: %" PRIu64 "\n" \ "job-timeouts: %" PRIu64 "\n" \ "total-jobs: %" PRIu64 "\n" \ "max-job-size: %zu\n" \ "current-tubes: %zu\n" \ "current-connections: %u\n" \ "current-producers: %u\n" \ "current-workers: %u\n" \ "current-waiting: %u\n" \ "total-connections: %u\n" \ "pid: %ld\n" \ "version: \"%s\"\n" \ "rusage-utime: %d.%06d\n" \ "rusage-stime: %d.%06d\n" \ "uptime: %u\n" \ "binlog-oldest-index: %d\n" \ "binlog-current-index: %d\n" \ "binlog-records-migrated: %" PRId64 "\n" \ "binlog-records-written: %" PRId64 "\n" \ "binlog-max-size: %d\n" \ "id: %s\n" \ "hostname: %s\n" \ "\r\n" #define STATS_TUBE_FMT "---\n" \ "name: %s\n" \ "current-jobs-urgent: %u\n" \ "current-jobs-ready: %u\n" \ "current-jobs-reserved: %u\n" \ "current-jobs-delayed: %u\n" \ "current-jobs-buried: %u\n" \ "total-jobs: %" PRIu64 "\n" \ "current-using: %u\n" \ "current-watching: %u\n" \ "current-waiting: %u\n" \ "cmd-delete: %" PRIu64 "\n" \ "cmd-pause-tube: %u\n" \ "pause: %" PRIu64 "\n" \ "pause-time-left: %" PRId64 "\n" \ "\r\n" #define STATS_JOB_FMT "---\n" \ "id: %" PRIu64 "\n" \ "tube: %s\n" \ "state: %s\n" \ "pri: %u\n" \ "age: %" PRId64 "\n" \ "delay: %" PRId64 "\n" \ "ttr: %" PRId64 "\n" \ "time-left: %" PRId64 "\n" \ "file: %d\n" \ "reserves: %u\n" \ "timeouts: %u\n" \ "releases: %u\n" \ "buries: %u\n" \ "kicks: %u\n" \ "\r\n" /* this number is pretty arbitrary */ #define BUCKET_BUF_SIZE 1024 static char bucket[BUCKET_BUF_SIZE]; static uint ready_ct = 0; static struct stats global_stat = {0, 0, 0, 0, 0}; static tube default_tube; static int drain_mode = 0; static int64 started_at; enum { NumIdBytes = 8 }; static char id[NumIdBytes * 2 + 1]; // hex-encoded len of NumIdBytes static struct utsname node_info; static uint64 op_ct[TOTAL_OPS], timeout_ct = 0; static Conn *dirty; static const char * op_names[] = { "", CMD_PUT, CMD_PEEKJOB, CMD_RESERVE, CMD_DELETE, CMD_RELEASE, CMD_BURY, CMD_KICK, CMD_STATS, CMD_JOBSTATS, CMD_PEEK_BURIED, CMD_USE, CMD_WATCH, CMD_IGNORE, CMD_LIST_TUBES, CMD_LIST_TUBE_USED, CMD_LIST_TUBES_WATCHED, CMD_STATS_TUBE, CMD_PEEK_READY, CMD_PEEK_DELAYED, CMD_RESERVE_TIMEOUT, CMD_TOUCH, CMD_QUIT, CMD_PAUSE_TUBE, CMD_JOBKICK, }; static job remove_buried_job(job j); static int buried_job_p(tube t) { return job_list_any_p(&t->buried); } static void reply(Conn *c, char *line, int len, int state) { if (!c) return; connwant(c, 'w'); c->next = dirty; dirty = c; c->reply = line; c->reply_len = len; c->reply_sent = 0; c->state = state; if (verbose >= 2) { printf(">%d reply %.*s\n", c->sock.fd, len-2, line); } } static void protrmdirty(Conn *c) { Conn *x, *newdirty = NULL; while (dirty) { x = dirty; dirty = dirty->next; x->next = NULL; if (x != c) { x->next = newdirty; newdirty = x; } } dirty = newdirty; } #define reply_msg(c,m) reply((c),(m),CONSTSTRLEN(m),STATE_SENDWORD) #define reply_serr(c,e) (twarnx("server error: %s",(e)),\ reply_msg((c),(e))) static void reply_line(Conn*, int, const char*, ...) __attribute__((format(printf, 3, 4))); static void reply_line(Conn *c, int state, const char *fmt, ...) { int r; va_list ap; va_start(ap, fmt); r = vsnprintf(c->reply_buf, LINE_BUF_SIZE, fmt, ap); va_end(ap); /* Make sure the buffer was big enough. If not, we have a bug. */ if (r >= LINE_BUF_SIZE) return reply_serr(c, MSG_INTERNAL_ERROR); return reply(c, c->reply_buf, r, state); } static void reply_job(Conn *c, job j, const char *word) { /* tell this connection which job to send */ c->out_job = j; c->out_job_sent = 0; return reply_line(c, STATE_SENDJOB, "%s %"PRIu64" %u\r\n", word, j->r.id, j->r.body_size - 2); } Conn * remove_waiting_conn(Conn *c) { tube t; size_t i; if (!conn_waiting(c)) return NULL; c->type &= ~CONN_TYPE_WAITING; global_stat.waiting_ct--; for (i = 0; i < c->watch.used; i++) { t = c->watch.items[i]; t->stat.waiting_ct--; ms_remove(&t->waiting, c); } return c; } static void reserve_job(Conn *c, job j) { j->r.deadline_at = nanoseconds() + j->r.ttr; global_stat.reserved_ct++; /* stats */ j->tube->stat.reserved_ct++; j->r.reserve_ct++; j->r.state = Reserved; job_insert(&c->reserved_jobs, j); j->reserver = c; c->pending_timeout = -1; if (c->soonest_job && j->r.deadline_at < c->soonest_job->r.deadline_at) { c->soonest_job = j; } return reply_job(c, j, MSG_RESERVED); } static job next_eligible_job(int64 now) { tube t; size_t i; job j = NULL, candidate; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; if (t->pause) { if (t->deadline_at > now) continue; t->pause = 0; } if (t->waiting.used && t->ready.len) { candidate = t->ready.data[0]; if (!j || job_pri_less(candidate, j)) { j = candidate; } } } return j; } static void process_queue() { job j; int64 now = nanoseconds(); while ((j = next_eligible_job(now))) { heapremove(&j->tube->ready, j->heap_index); ready_ct--; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct--; j->tube->stat.urgent_ct--; } reserve_job(remove_waiting_conn(ms_take(&j->tube->waiting)), j); } } static job delay_q_peek() { int i; tube t; job j = NULL, nj; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; if (t->delay.len == 0) { continue; } nj = t->delay.data[0]; if (!j || nj->r.deadline_at < j->r.deadline_at) j = nj; } return j; } static int enqueue_job(Server *s, job j, int64 delay, char update_store) { int r; j->reserver = NULL; if (delay) { j->r.deadline_at = nanoseconds() + delay; r = heapinsert(&j->tube->delay, j); if (!r) return 0; j->r.state = Delayed; } else { r = heapinsert(&j->tube->ready, j); if (!r) return 0; j->r.state = Ready; ready_ct++; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct++; j->tube->stat.urgent_ct++; } } if (update_store) { if (!walwrite(&s->wal, j)) { return 0; } walmaint(&s->wal); } process_queue(); return 1; } static int bury_job(Server *s, job j, char update_store) { int z; if (update_store) { z = walresvupdate(&s->wal, j); if (!z) return 0; j->walresv += z; } job_insert(&j->tube->buried, j); global_stat.buried_ct++; j->tube->stat.buried_ct++; j->r.state = Buried; j->reserver = NULL; j->r.bury_ct++; if (update_store) { if (!walwrite(&s->wal, j)) { return 0; } walmaint(&s->wal); } return 1; } void enqueue_reserved_jobs(Conn *c) { int r; job j; while (job_list_any_p(&c->reserved_jobs)) { j = job_remove(c->reserved_jobs.next); r = enqueue_job(c->srv, j, 0, 0); if (r < 1) bury_job(c->srv, j, 0); global_stat.reserved_ct--; j->tube->stat.reserved_ct--; c->soonest_job = NULL; } } static job delay_q_take() { job j = delay_q_peek(); if (!j) { return 0; } heapremove(&j->tube->delay, j->heap_index); return j; } static int kick_buried_job(Server *s, job j) { int r; int z; z = walresvupdate(&s->wal, j); if (!z) return 0; j->walresv += z; remove_buried_job(j); j->r.kick_ct++; r = enqueue_job(s, j, 0, 1); if (r == 1) return 1; /* ready queue is full, so bury it */ bury_job(s, j, 0); return 0; } static uint get_delayed_job_ct() { tube t; size_t i; uint count = 0; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; count += t->delay.len; } return count; } static int kick_delayed_job(Server *s, job j) { int r; int z; z = walresvupdate(&s->wal, j); if (!z) return 0; j->walresv += z; heapremove(&j->tube->delay, j->heap_index); j->r.kick_ct++; r = enqueue_job(s, j, 0, 1); if (r == 1) return 1; /* ready queue is full, so delay it again */ r = enqueue_job(s, j, j->r.delay, 0); if (r == 1) return 0; /* last resort */ bury_job(s, j, 0); return 0; } /* return the number of jobs successfully kicked */ static uint kick_buried_jobs(Server *s, tube t, uint n) { uint i; for (i = 0; (i < n) && buried_job_p(t); ++i) { kick_buried_job(s, t->buried.next); } return i; } /* return the number of jobs successfully kicked */ static uint kick_delayed_jobs(Server *s, tube t, uint n) { uint i; for (i = 0; (i < n) && (t->delay.len > 0); ++i) { kick_delayed_job(s, (job)t->delay.data[0]); } return i; } static uint kick_jobs(Server *s, tube t, uint n) { if (buried_job_p(t)) return kick_buried_jobs(s, t, n); return kick_delayed_jobs(s, t, n); } static job remove_buried_job(job j) { if (!j || j->r.state != Buried) return NULL; j = job_remove(j); if (j) { global_stat.buried_ct--; j->tube->stat.buried_ct--; } return j; } static job remove_delayed_job(job j) { if (!j || j->r.state != Delayed) return NULL; heapremove(&j->tube->delay, j->heap_index); return j; } static job remove_ready_job(job j) { if (!j || j->r.state != Ready) return NULL; heapremove(&j->tube->ready, j->heap_index); ready_ct--; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct--; j->tube->stat.urgent_ct--; } return j; } static void enqueue_waiting_conn(Conn *c) { tube t; size_t i; global_stat.waiting_ct++; c->type |= CONN_TYPE_WAITING; for (i = 0; i < c->watch.used; i++) { t = c->watch.items[i]; t->stat.waiting_ct++; ms_append(&t->waiting, c); } } static job find_reserved_job_in_conn(Conn *c, job j) { return (j && j->reserver == c && j->r.state == Reserved) ? j : NULL; } static job touch_job(Conn *c, job j) { j = find_reserved_job_in_conn(c, j); if (j) { j->r.deadline_at = nanoseconds() + j->r.ttr; c->soonest_job = NULL; } return j; } static job peek_job(uint64 id) { return job_find(id); } static void check_err(Conn *c, const char *s) { if (errno == EAGAIN) return; if (errno == EINTR) return; if (errno == EWOULDBLOCK) return; twarn("%s", s); c->state = STATE_CLOSE; return; } /* Scan the given string for the sequence "\r\n" and return the line length. * Always returns at least 2 if a match is found. Returns 0 if no match. */ static int scan_line_end(const char *s, int size) { char *match; match = memchr(s, '\r', size - 1); if (!match) return 0; /* this is safe because we only scan size - 1 chars above */ if (match[1] == '\n') return match - s + 2; return 0; } static int cmd_len(Conn *c) { return scan_line_end(c->cmd, c->cmd_read); } /* parse the command line */ static int which_cmd(Conn *c) { #define TEST_CMD(s,c,o) if (strncmp((s), (c), CONSTSTRLEN(c)) == 0) return (o); TEST_CMD(c->cmd, CMD_PUT, OP_PUT); TEST_CMD(c->cmd, CMD_PEEKJOB, OP_PEEKJOB); TEST_CMD(c->cmd, CMD_PEEK_READY, OP_PEEK_READY); TEST_CMD(c->cmd, CMD_PEEK_DELAYED, OP_PEEK_DELAYED); TEST_CMD(c->cmd, CMD_PEEK_BURIED, OP_PEEK_BURIED); TEST_CMD(c->cmd, CMD_RESERVE_TIMEOUT, OP_RESERVE_TIMEOUT); TEST_CMD(c->cmd, CMD_RESERVE, OP_RESERVE); TEST_CMD(c->cmd, CMD_DELETE, OP_DELETE); TEST_CMD(c->cmd, CMD_RELEASE, OP_RELEASE); TEST_CMD(c->cmd, CMD_BURY, OP_BURY); TEST_CMD(c->cmd, CMD_KICK, OP_KICK); TEST_CMD(c->cmd, CMD_JOBKICK, OP_JOBKICK); TEST_CMD(c->cmd, CMD_TOUCH, OP_TOUCH); TEST_CMD(c->cmd, CMD_JOBSTATS, OP_JOBSTATS); TEST_CMD(c->cmd, CMD_STATS_TUBE, OP_STATS_TUBE); TEST_CMD(c->cmd, CMD_STATS, OP_STATS); TEST_CMD(c->cmd, CMD_USE, OP_USE); TEST_CMD(c->cmd, CMD_WATCH, OP_WATCH); TEST_CMD(c->cmd, CMD_IGNORE, OP_IGNORE); TEST_CMD(c->cmd, CMD_LIST_TUBES_WATCHED, OP_LIST_TUBES_WATCHED); TEST_CMD(c->cmd, CMD_LIST_TUBE_USED, OP_LIST_TUBE_USED); TEST_CMD(c->cmd, CMD_LIST_TUBES, OP_LIST_TUBES); TEST_CMD(c->cmd, CMD_QUIT, OP_QUIT); TEST_CMD(c->cmd, CMD_PAUSE_TUBE, OP_PAUSE_TUBE); return OP_UNKNOWN; } /* Copy up to body_size trailing bytes into the job, then the rest into the cmd * buffer. If c->in_job exists, this assumes that c->in_job->body is empty. * This function is idempotent(). */ static void fill_extra_data(Conn *c) { int extra_bytes, job_data_bytes = 0, cmd_bytes; if (!c->sock.fd) return; /* the connection was closed */ if (!c->cmd_len) return; /* we don't have a complete command */ /* how many extra bytes did we read? */ extra_bytes = c->cmd_read - c->cmd_len; /* how many bytes should we put into the job body? */ if (c->in_job) { job_data_bytes = min(extra_bytes, c->in_job->r.body_size); memcpy(c->in_job->body, c->cmd + c->cmd_len, job_data_bytes); c->in_job_read = job_data_bytes; } else if (c->in_job_read) { /* we are in bit-bucket mode, throwing away data */ job_data_bytes = min(extra_bytes, c->in_job_read); c->in_job_read -= job_data_bytes; } /* how many bytes are left to go into the future cmd? */ cmd_bytes = extra_bytes - job_data_bytes; memmove(c->cmd, c->cmd + c->cmd_len + job_data_bytes, cmd_bytes); c->cmd_read = cmd_bytes; c->cmd_len = 0; /* we no longer know the length of the new command */ } static void _skip(Conn *c, int n, char *line, int len) { /* Invert the meaning of in_job_read while throwing away data -- it * counts the bytes that remain to be thrown away. */ c->in_job = 0; c->in_job_read = n; fill_extra_data(c); if (c->in_job_read == 0) return reply(c, line, len, STATE_SENDWORD); c->reply = line; c->reply_len = len; c->reply_sent = 0; c->state = STATE_BITBUCKET; return; } #define skip(c,n,m) (_skip(c,n,m,CONSTSTRLEN(m))) static void enqueue_incoming_job(Conn *c) { int r; job j = c->in_job; c->in_job = NULL; /* the connection no longer owns this job */ c->in_job_read = 0; /* check if the trailer is present and correct */ if (memcmp(j->body + j->r.body_size - 2, "\r\n", 2)) { job_free(j); return reply_msg(c, MSG_EXPECTED_CRLF); } if (verbose >= 2) { printf("<%d job %"PRIu64"\n", c->sock.fd, j->r.id); } if (drain_mode) { job_free(j); return reply_serr(c, MSG_DRAINING); } if (j->walresv) return reply_serr(c, MSG_INTERNAL_ERROR); j->walresv = walresvput(&c->srv->wal, j); if (!j->walresv) return reply_serr(c, MSG_OUT_OF_MEMORY); /* we have a complete job, so let's stick it in the pqueue */ r = enqueue_job(c->srv, j, j->r.delay, 1); if (r < 0) return reply_serr(c, MSG_INTERNAL_ERROR); global_stat.total_jobs_ct++; j->tube->stat.total_jobs_ct++; if (r == 1) return reply_line(c, STATE_SENDWORD, MSG_INSERTED_FMT, j->r.id); /* out of memory trying to grow the queue, so it gets buried */ bury_job(c->srv, j, 0); reply_line(c, STATE_SENDWORD, MSG_BURIED_FMT, j->r.id); } static uint uptime() { return (nanoseconds() - started_at) / 1000000000; } static int fmt_stats(char *buf, size_t size, void *x) { int whead = 0, wcur = 0; Server *srv; struct rusage ru = {{0, 0}, {0, 0}}; srv = x; if (srv->wal.head) { whead = srv->wal.head->seq; } if (srv->wal.cur) { wcur = srv->wal.cur->seq; } getrusage(RUSAGE_SELF, &ru); /* don't care if it fails */ return snprintf(buf, size, STATS_FMT, global_stat.urgent_ct, ready_ct, global_stat.reserved_ct, get_delayed_job_ct(), global_stat.buried_ct, op_ct[OP_PUT], op_ct[OP_PEEKJOB], op_ct[OP_PEEK_READY], op_ct[OP_PEEK_DELAYED], op_ct[OP_PEEK_BURIED], op_ct[OP_RESERVE], op_ct[OP_RESERVE_TIMEOUT], op_ct[OP_DELETE], op_ct[OP_RELEASE], op_ct[OP_USE], op_ct[OP_WATCH], op_ct[OP_IGNORE], op_ct[OP_BURY], op_ct[OP_KICK], op_ct[OP_TOUCH], op_ct[OP_STATS], op_ct[OP_JOBSTATS], op_ct[OP_STATS_TUBE], op_ct[OP_LIST_TUBES], op_ct[OP_LIST_TUBE_USED], op_ct[OP_LIST_TUBES_WATCHED], op_ct[OP_PAUSE_TUBE], timeout_ct, global_stat.total_jobs_ct, job_data_size_limit, tubes.used, count_cur_conns(), count_cur_producers(), count_cur_workers(), global_stat.waiting_ct, count_tot_conns(), (long) getpid(), version, (int) ru.ru_utime.tv_sec, (int) ru.ru_utime.tv_usec, (int) ru.ru_stime.tv_sec, (int) ru.ru_stime.tv_usec, uptime(), whead, wcur, srv->wal.nmig, srv->wal.nrec, srv->wal.filesize, id, node_info.nodename); } /* Read a priority value from the given buffer and place it in pri. * Update end to point to the address after the last character consumed. * Pri and end can be NULL. If they are both NULL, read_pri() will do the * conversion and return the status code but not update any values. This is an * easy way to check for errors. * If end is NULL, read_pri will also check that the entire input string was * consumed and return an error code otherwise. * Return 0 on success, or nonzero on failure. * If a failure occurs, pri and end are not modified. */ static int read_pri(uint *pri, const char *buf, char **end) { char *tend; uint tpri; errno = 0; while (buf[0] == ' ') buf++; if (buf[0] < '0' || '9' < buf[0]) return -1; tpri = strtoul(buf, &tend, 10); if (tend == buf) return -1; if (errno && errno != ERANGE) return -1; if (!end && tend[0] != '\0') return -1; if (pri) *pri = tpri; if (end) *end = tend; return 0; } /* Read a delay value from the given buffer and place it in delay. * The interface and behavior are analogous to read_pri(). */ static int read_delay(int64 *delay, const char *buf, char **end) { int r; uint delay_sec; r = read_pri(&delay_sec, buf, end); if (r) return r; *delay = ((int64) delay_sec) * 1000000000; return 0; } /* Read a timeout value from the given buffer and place it in ttr. * The interface and behavior are the same as in read_delay(). */ static int read_ttr(int64 *ttr, const char *buf, char **end) { return read_delay(ttr, buf, end); } /* Read a tube name from the given buffer moving the buffer to the name start */ static int read_tube_name(char **tubename, char *buf, char **end) { size_t len; while (buf[0] == ' ') buf++; len = strspn(buf, NAME_CHARS); if (len == 0) return -1; if (tubename) *tubename = buf; if (end) *end = buf + len; return 0; } static void wait_for_job(Conn *c, int timeout) { c->state = STATE_WAIT; enqueue_waiting_conn(c); /* Set the pending timeout to the requested timeout amount */ c->pending_timeout = timeout; connwant(c, 'h'); // only care if they hang up c->next = dirty; dirty = c; } typedef int(*fmt_fn)(char *, size_t, void *); static void do_stats(Conn *c, fmt_fn fmt, void *data) { int r, stats_len; /* first, measure how big a buffer we will need */ stats_len = fmt(NULL, 0, data) + 16; c->out_job = allocate_job(stats_len); /* fake job to hold stats data */ if (!c->out_job) return reply_serr(c, MSG_OUT_OF_MEMORY); /* Mark this job as a copy so it can be appropriately freed later on */ c->out_job->r.state = Copy; /* now actually format the stats data */ r = fmt(c->out_job->body, stats_len, data); /* and set the actual body size */ c->out_job->r.body_size = r; if (r > stats_len) return reply_serr(c, MSG_INTERNAL_ERROR); c->out_job_sent = 0; return reply_line(c, STATE_SENDJOB, "OK %d\r\n", r - 2); } static void do_list_tubes(Conn *c, ms l) { char *buf; tube t; size_t i, resp_z; /* first, measure how big a buffer we will need */ resp_z = 6; /* initial "---\n" and final "\r\n" */ for (i = 0; i < l->used; i++) { t = l->items[i]; resp_z += 3 + strlen(t->name); /* including "- " and "\n" */ } c->out_job = allocate_job(resp_z); /* fake job to hold response data */ if (!c->out_job) return reply_serr(c, MSG_OUT_OF_MEMORY); /* Mark this job as a copy so it can be appropriately freed later on */ c->out_job->r.state = Copy; /* now actually format the response */ buf = c->out_job->body; buf += snprintf(buf, 5, "---\n"); for (i = 0; i < l->used; i++) { t = l->items[i]; buf += snprintf(buf, 4 + strlen(t->name), "- %s\n", t->name); } buf[0] = '\r'; buf[1] = '\n'; c->out_job_sent = 0; return reply_line(c, STATE_SENDJOB, "OK %zu\r\n", resp_z - 2); } static int fmt_job_stats(char *buf, size_t size, job j) { int64 t; int64 time_left; int file = 0; t = nanoseconds(); if (j->r.state == Reserved || j->r.state == Delayed) { time_left = (j->r.deadline_at - t) / 1000000000; } else { time_left = 0; } if (j->file) { file = j->file->seq; } return snprintf(buf, size, STATS_JOB_FMT, j->r.id, j->tube->name, job_state(j), j->r.pri, (t - j->r.created_at) / 1000000000, j->r.delay / 1000000000, j->r.ttr / 1000000000, time_left, file, j->r.reserve_ct, j->r.timeout_ct, j->r.release_ct, j->r.bury_ct, j->r.kick_ct); } static int fmt_stats_tube(char *buf, size_t size, tube t) { uint64 time_left; if (t->pause > 0) { time_left = (t->deadline_at - nanoseconds()) / 1000000000; } else { time_left = 0; } return snprintf(buf, size, STATS_TUBE_FMT, t->name, t->stat.urgent_ct, t->ready.len, t->stat.reserved_ct, t->delay.len, t->stat.buried_ct, t->stat.total_jobs_ct, t->using_ct, t->watching_ct, t->stat.waiting_ct, t->stat.total_delete_ct, t->stat.pause_ct, t->pause / 1000000000, time_left); } static void maybe_enqueue_incoming_job(Conn *c) { job j = c->in_job; /* do we have a complete job? */ if (c->in_job_read == j->r.body_size) return enqueue_incoming_job(c); /* otherwise we have incomplete data, so just keep waiting */ c->state = STATE_WANTDATA; } /* j can be NULL */ static job remove_this_reserved_job(Conn *c, job j) { j = job_remove(j); if (j) { global_stat.reserved_ct--; j->tube->stat.reserved_ct--; j->reserver = NULL; } c->soonest_job = NULL; return j; } static job remove_reserved_job(Conn *c, job j) { return remove_this_reserved_job(c, find_reserved_job_in_conn(c, j)); } static int name_is_ok(const char *name, size_t max) { size_t len = strlen(name); return len > 0 && len <= max && strspn(name, NAME_CHARS) == len && name[0] != '-'; } void prot_remove_tube(tube t) { ms_remove(&tubes, t); } static void dispatch_cmd(Conn *c) { int r, i, timeout = -1; int z; uint count; job j = 0; byte type; char *size_buf, *delay_buf, *ttr_buf, *pri_buf, *end_buf, *name; uint pri, body_size; int64 delay, ttr; uint64 id; tube t = NULL; /* NUL-terminate this string so we can use strtol and friends */ c->cmd[c->cmd_len - 2] = '\0'; /* check for possible maliciousness */ if (strlen(c->cmd) != c->cmd_len - 2) { return reply_msg(c, MSG_BAD_FORMAT); } type = which_cmd(c); if (verbose >= 2) { printf("<%d command %s\n", c->sock.fd, op_names[type]); } switch (type) { case OP_PUT: r = read_pri(&pri, c->cmd + 4, &delay_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_delay(&delay, delay_buf, &ttr_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_ttr(&ttr, ttr_buf, &size_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); errno = 0; body_size = strtoul(size_buf, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; if (body_size > job_data_size_limit) { /* throw away the job body and respond with JOB_TOO_BIG */ return skip(c, body_size + 2, MSG_JOB_TOO_BIG); } /* don't allow trailing garbage */ if (end_buf[0] != '\0') return reply_msg(c, MSG_BAD_FORMAT); connsetproducer(c); if (ttr < 1000000000) { ttr = 1000000000; } c->in_job = make_job(pri, delay, ttr, body_size + 2, c->use); /* OOM? */ if (!c->in_job) { /* throw away the job body and respond with OUT_OF_MEMORY */ twarnx("server error: " MSG_OUT_OF_MEMORY); return skip(c, body_size + 2, MSG_OUT_OF_MEMORY); } fill_extra_data(c); /* it's possible we already have a complete job */ maybe_enqueue_incoming_job(c); break; case OP_PEEK_READY: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_READY_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; if (c->use->ready.len) { j = job_copy(c->use->ready.data[0]); } if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_PEEK_DELAYED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_DELAYED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; if (c->use->delay.len) { j = job_copy(c->use->delay.data[0]); } if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_PEEK_BURIED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_BURIED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; j = job_copy(buried_job_p(c->use)? j = c->use->buried.next : NULL); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_PEEKJOB: errno = 0; id = strtoull(c->cmd + CMD_PEEKJOB_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; /* So, peek is annoying, because some other connection might free the * job while we are still trying to write it out. So we copy it and * then free the copy when it's done sending. */ j = job_copy(peek_job(id)); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); reply_job(c, j, MSG_FOUND); break; case OP_RESERVE_TIMEOUT: errno = 0; timeout = strtol(c->cmd + CMD_RESERVE_TIMEOUT_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); case OP_RESERVE: /* FALLTHROUGH */ /* don't allow trailing garbage */ if (type == OP_RESERVE && c->cmd_len != CMD_RESERVE_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; connsetworker(c); if (conndeadlinesoon(c) && !conn_ready(c)) { return reply_msg(c, MSG_DEADLINE_SOON); } /* try to get a new job for this guy */ wait_for_job(c, timeout); process_queue(); break; case OP_DELETE: errno = 0; id = strtoull(c->cmd + CMD_DELETE_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = job_find(id); j = remove_reserved_job(c, j) ? : remove_ready_job(j) ? : remove_buried_job(j) ? : remove_delayed_job(j); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); j->tube->stat.total_delete_ct++; j->r.state = Invalid; r = walwrite(&c->srv->wal, j); walmaint(&c->srv->wal); job_free(j); if (!r) return reply_serr(c, MSG_INTERNAL_ERROR); reply(c, MSG_DELETED, MSG_DELETED_LEN, STATE_SENDWORD); break; case OP_RELEASE: errno = 0; id = strtoull(c->cmd + CMD_RELEASE_LEN, &pri_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); r = read_pri(&pri, pri_buf, &delay_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_delay(&delay, delay_buf, NULL); if (r) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = remove_reserved_job(c, job_find(id)); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); /* We want to update the delay deadline on disk, so reserve space for * that. */ if (delay) { z = walresvupdate(&c->srv->wal, j); if (!z) return reply_serr(c, MSG_OUT_OF_MEMORY); j->walresv += z; } j->r.pri = pri; j->r.delay = delay; j->r.release_ct++; r = enqueue_job(c->srv, j, delay, !!delay); if (r < 0) return reply_serr(c, MSG_INTERNAL_ERROR); if (r == 1) { return reply(c, MSG_RELEASED, MSG_RELEASED_LEN, STATE_SENDWORD); } /* out of memory trying to grow the queue, so it gets buried */ bury_job(c->srv, j, 0); reply(c, MSG_BURIED, MSG_BURIED_LEN, STATE_SENDWORD); break; case OP_BURY: errno = 0; id = strtoull(c->cmd + CMD_BURY_LEN, &pri_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); r = read_pri(&pri, pri_buf, NULL); if (r) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = remove_reserved_job(c, job_find(id)); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); j->r.pri = pri; r = bury_job(c->srv, j, 1); if (!r) return reply_serr(c, MSG_INTERNAL_ERROR); reply(c, MSG_BURIED, MSG_BURIED_LEN, STATE_SENDWORD); break; case OP_KICK: errno = 0; count = strtoul(c->cmd + CMD_KICK_LEN, &end_buf, 10); if (end_buf == c->cmd + CMD_KICK_LEN) { return reply_msg(c, MSG_BAD_FORMAT); } if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; i = kick_jobs(c->srv, c->use, count); return reply_line(c, STATE_SENDWORD, "KICKED %u\r\n", i); case OP_JOBKICK: errno = 0; id = strtoull(c->cmd + CMD_JOBKICK_LEN, &end_buf, 10); if (errno) return twarn("strtoull"), reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = job_find(id); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); if ((j->r.state == Buried && kick_buried_job(c->srv, j)) || (j->r.state == Delayed && kick_delayed_job(c->srv, j))) { reply(c, MSG_KICKED, MSG_KICKED_LEN, STATE_SENDWORD); } else { return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); } break; case OP_TOUCH: errno = 0; id = strtoull(c->cmd + CMD_TOUCH_LEN, &end_buf, 10); if (errno) return twarn("strtoull"), reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = touch_job(c, job_find(id)); if (j) { reply(c, MSG_TOUCHED, MSG_TOUCHED_LEN, STATE_SENDWORD); } else { return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); } break; case OP_STATS: /* don't allow trailing garbage */ if (c->cmd_len != CMD_STATS_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; do_stats(c, fmt_stats, c->srv); break; case OP_JOBSTATS: errno = 0; id = strtoull(c->cmd + CMD_JOBSTATS_LEN, &end_buf, 10); if (errno) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; j = peek_job(id); if (!j) return reply(c, MSG_NOTFOUND, MSG_NOTFOUND_LEN, STATE_SENDWORD); if (!j->tube) return reply_serr(c, MSG_INTERNAL_ERROR); do_stats(c, (fmt_fn) fmt_job_stats, j); break; case OP_STATS_TUBE: name = c->cmd + CMD_STATS_TUBE_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; t = tube_find(name); if (!t) return reply_msg(c, MSG_NOTFOUND); do_stats(c, (fmt_fn) fmt_stats_tube, t); t = NULL; break; case OP_LIST_TUBES: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBES_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; do_list_tubes(c, &tubes); break; case OP_LIST_TUBE_USED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBE_USED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; reply_line(c, STATE_SENDWORD, "USING %s\r\n", c->use->name); break; case OP_LIST_TUBES_WATCHED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBES_WATCHED_LEN + 2) { return reply_msg(c, MSG_BAD_FORMAT); } op_ct[type]++; do_list_tubes(c, &c->watch); break; case OP_USE: name = c->cmd + CMD_USE_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; TUBE_ASSIGN(t, tube_find_or_make(name)); if (!t) return reply_serr(c, MSG_OUT_OF_MEMORY); c->use->using_ct--; TUBE_ASSIGN(c->use, t); TUBE_ASSIGN(t, NULL); c->use->using_ct++; reply_line(c, STATE_SENDWORD, "USING %s\r\n", c->use->name); break; case OP_WATCH: name = c->cmd + CMD_WATCH_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; TUBE_ASSIGN(t, tube_find_or_make(name)); if (!t) return reply_serr(c, MSG_OUT_OF_MEMORY); r = 1; if (!ms_contains(&c->watch, t)) r = ms_append(&c->watch, t); TUBE_ASSIGN(t, NULL); if (!r) return reply_serr(c, MSG_OUT_OF_MEMORY); reply_line(c, STATE_SENDWORD, "WATCHING %zu\r\n", c->watch.used); break; case OP_IGNORE: name = c->cmd + CMD_IGNORE_LEN; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); op_ct[type]++; t = NULL; for (i = 0; i < c->watch.used; i++) { t = c->watch.items[i]; if (strncmp(t->name, name, MAX_TUBE_NAME_LEN) == 0) break; t = NULL; } if (t && c->watch.used < 2) return reply_msg(c, MSG_NOT_IGNORED); if (t) ms_remove(&c->watch, t); /* may free t if refcount => 0 */ t = NULL; reply_line(c, STATE_SENDWORD, "WATCHING %zu\r\n", c->watch.used); break; case OP_QUIT: c->state = STATE_CLOSE; break; case OP_PAUSE_TUBE: op_ct[type]++; r = read_tube_name(&name, c->cmd + CMD_PAUSE_TUBE_LEN, &delay_buf); if (r) return reply_msg(c, MSG_BAD_FORMAT); r = read_delay(&delay, delay_buf, NULL); if (r) return reply_msg(c, MSG_BAD_FORMAT); *delay_buf = '\0'; if (!name_is_ok(name, 200)) return reply_msg(c, MSG_BAD_FORMAT); t = tube_find(name); if (!t) return reply_msg(c, MSG_NOTFOUND); // Always pause for a positive amount of time, to make sure // that waiting clients wake up when the deadline arrives. if (delay == 0) { delay = 1; } t->deadline_at = nanoseconds() + delay; t->pause = delay; t->stat.pause_ct++; reply_line(c, STATE_SENDWORD, "PAUSED\r\n"); break; default: return reply_msg(c, MSG_UNKNOWN_COMMAND); } } /* There are three reasons this function may be called. We need to check for * all of them. * * 1. A reserved job has run out of time. * 2. A waiting client's reserved job has entered the safety margin. * 3. A waiting client's requested timeout has occurred. * * If any of these happen, we must do the appropriate thing. */ static void conn_timeout(Conn *c) { int r, should_timeout = 0; job j; /* Check if the client was trying to reserve a job. */ if (conn_waiting(c) && conndeadlinesoon(c)) should_timeout = 1; /* Check if any reserved jobs have run out of time. We should do this * whether or not the client is waiting for a new reservation. */ while ((j = connsoonestjob(c))) { if (j->r.deadline_at >= nanoseconds()) break; /* This job is in the middle of being written out. If we return it to * the ready queue, someone might free it before we finish writing it * out to the socket. So we'll copy it here and free the copy when it's * done sending. */ if (j == c->out_job) { c->out_job = job_copy(c->out_job); } timeout_ct++; /* stats */ j->r.timeout_ct++; r = enqueue_job(c->srv, remove_this_reserved_job(c, j), 0, 0); if (r < 1) bury_job(c->srv, j, 0); /* out of memory, so bury it */ connsched(c); } if (should_timeout) { return reply_msg(remove_waiting_conn(c), MSG_DEADLINE_SOON); } else if (conn_waiting(c) && c->pending_timeout >= 0) { c->pending_timeout = -1; return reply_msg(remove_waiting_conn(c), MSG_TIMED_OUT); } } void enter_drain_mode(int sig) { drain_mode = 1; } static void do_cmd(Conn *c) { dispatch_cmd(c); fill_extra_data(c); } static void reset_conn(Conn *c) { connwant(c, 'r'); c->next = dirty; dirty = c; /* was this a peek or stats command? */ if (c->out_job && c->out_job->r.state == Copy) job_free(c->out_job); c->out_job = NULL; c->reply_sent = 0; /* now that we're done, reset this */ c->state = STATE_WANTCOMMAND; } static void conn_data(Conn *c) { int r, to_read; job j; struct iovec iov[2]; switch (c->state) { case STATE_WANTCOMMAND: r = read(c->sock.fd, c->cmd + c->cmd_read, LINE_BUF_SIZE - c->cmd_read); if (r == -1) return check_err(c, "read()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->cmd_read += r; /* we got some bytes */ c->cmd_len = cmd_len(c); /* find the EOL */ /* yay, complete command line */ if (c->cmd_len) return do_cmd(c); /* c->cmd_read > LINE_BUF_SIZE can't happen */ /* command line too long? */ if (c->cmd_read == LINE_BUF_SIZE) { c->cmd_read = 0; /* discard the input so far */ return reply_msg(c, MSG_BAD_FORMAT); } /* otherwise we have an incomplete line, so just keep waiting */ break; case STATE_BITBUCKET: /* Invert the meaning of in_job_read while throwing away data -- it * counts the bytes that remain to be thrown away. */ to_read = min(c->in_job_read, BUCKET_BUF_SIZE); r = read(c->sock.fd, bucket, to_read); if (r == -1) return check_err(c, "read()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->in_job_read -= r; /* we got some bytes */ /* (c->in_job_read < 0) can't happen */ if (c->in_job_read == 0) { return reply(c, c->reply, c->reply_len, STATE_SENDWORD); } break; case STATE_WANTDATA: j = c->in_job; r = read(c->sock.fd, j->body + c->in_job_read, j->r.body_size -c->in_job_read); if (r == -1) return check_err(c, "read()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->in_job_read += r; /* we got some bytes */ /* (j->in_job_read > j->r.body_size) can't happen */ maybe_enqueue_incoming_job(c); break; case STATE_SENDWORD: r= write(c->sock.fd, c->reply + c->reply_sent, c->reply_len - c->reply_sent); if (r == -1) return check_err(c, "write()"); if (r == 0) { c->state = STATE_CLOSE; return; } c->reply_sent += r; /* we got some bytes */ /* (c->reply_sent > c->reply_len) can't happen */ if (c->reply_sent == c->reply_len) return reset_conn(c); /* otherwise we sent an incomplete reply, so just keep waiting */ break; case STATE_SENDJOB: j = c->out_job; iov[0].iov_base = (void *)(c->reply + c->reply_sent); iov[0].iov_len = c->reply_len - c->reply_sent; /* maybe 0 */ iov[1].iov_base = j->body + c->out_job_sent; iov[1].iov_len = j->r.body_size - c->out_job_sent; r = writev(c->sock.fd, iov, 2); if (r == -1) return check_err(c, "writev()"); if (r == 0) { c->state = STATE_CLOSE; return; } /* update the sent values */ c->reply_sent += r; if (c->reply_sent >= c->reply_len) { c->out_job_sent += c->reply_sent - c->reply_len; c->reply_sent = c->reply_len; } /* (c->out_job_sent > j->r.body_size) can't happen */ /* are we done? */ if (c->out_job_sent == j->r.body_size) { if (verbose >= 2) { printf(">%d job %"PRIu64"\n", c->sock.fd, j->r.id); } return reset_conn(c); } /* otherwise we sent incomplete data, so just keep waiting */ break; case STATE_WAIT: if (c->halfclosed) { c->pending_timeout = -1; return reply_msg(remove_waiting_conn(c), MSG_TIMED_OUT); } break; } } #define want_command(c) ((c)->sock.fd && ((c)->state == STATE_WANTCOMMAND)) #define cmd_data_ready(c) (want_command(c) && (c)->cmd_read) static void update_conns() { int r; Conn *c; while (dirty) { c = dirty; dirty = dirty->next; c->next = NULL; r = sockwant(&c->sock, c->rw); if (r == -1) { twarn("sockwant"); connclose(c); } } } static void h_conn(const int fd, const short which, Conn *c) { if (fd != c->sock.fd) { twarnx("Argh! event fd doesn't match conn fd."); close(fd); connclose(c); update_conns(); return; } if (which == 'h') { c->halfclosed = 1; } conn_data(c); while (cmd_data_ready(c) && (c->cmd_len = cmd_len(c))) do_cmd(c); if (c->state == STATE_CLOSE) { protrmdirty(c); connclose(c); } update_conns(); } static void prothandle(Conn *c, int ev) { h_conn(c->sock.fd, ev, c); } int64 prottick(Server *s) { int r; job j; int64 now; int i; tube t; int64 period = 0x34630B8A000LL; /* 1 hour in nanoseconds */ int64 d; now = nanoseconds(); while ((j = delay_q_peek())) { d = j->r.deadline_at - now; if (d > 0) { period = min(period, d); break; } j = delay_q_take(); r = enqueue_job(s, j, 0, 0); if (r < 1) bury_job(s, j, 0); /* out of memory, so bury it */ } for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; d = t->deadline_at - now; if (t->pause && d <= 0) { t->pause = 0; process_queue(); } else if (d > 0) { period = min(period, d); } } while (s->conns.len) { Conn *c = s->conns.data[0]; d = c->tickat - now; if (d > 0) { period = min(period, d); break; } heapremove(&s->conns, 0); conn_timeout(c); } update_conns(); return period; } void h_accept(const int fd, const short which, Server *s) { Conn *c; int cfd, flags, r; socklen_t addrlen; struct sockaddr_in6 addr; addrlen = sizeof addr; cfd = accept(fd, (struct sockaddr *)&addr, &addrlen); if (cfd == -1) { if (errno != EAGAIN && errno != EWOULDBLOCK) twarn("accept()"); update_conns(); return; } if (verbose) { printf("accept %d\n", cfd); } flags = fcntl(cfd, F_GETFL, 0); if (flags < 0) { twarn("getting flags"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } r = fcntl(cfd, F_SETFL, flags | O_NONBLOCK); if (r < 0) { twarn("setting O_NONBLOCK"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } c = make_conn(cfd, STATE_WANTCOMMAND, default_tube, default_tube); if (!c) { twarnx("make_conn() failed"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } c->srv = s; c->sock.x = c; c->sock.f = (Handle)prothandle; c->sock.fd = cfd; r = sockwant(&c->sock, 'r'); if (r == -1) { twarn("sockwant"); close(cfd); if (verbose) { printf("close %d\n", cfd); } update_conns(); return; } update_conns(); } void prot_init() { started_at = nanoseconds(); memset(op_ct, 0, sizeof(op_ct)); int dev_random = open("/dev/urandom", O_RDONLY); if (dev_random < 0) { twarn("open /dev/urandom"); exit(50); } int i, r; byte rand_data[NumIdBytes]; r = read(dev_random, &rand_data, NumIdBytes); if (r != NumIdBytes) { twarn("read /dev/urandom"); exit(50); } for (i = 0; i < NumIdBytes; i++) { sprintf(id + (i * 2), "%02x", rand_data[i]); } close(dev_random); if (uname(&node_info) == -1) { warn("uname"); exit(50); } ms_init(&tubes, NULL, NULL); TUBE_ASSIGN(default_tube, tube_find_or_make("default")); if (!default_tube) twarnx("Out of memory during startup!"); } // For each job in list, inserts the job into the appropriate data // structures and adds it to the log. // // Returns 1 on success, 0 on failure. int prot_replay(Server *s, job list) { job j, nj; int64 t, delay; int r, z; for (j = list->next ; j != list ; j = nj) { nj = j->next; job_remove(j); z = walresvupdate(&s->wal, j); if (!z) { twarnx("failed to reserve space"); return 0; } delay = 0; switch (j->r.state) { case Buried: bury_job(s, j, 0); break; case Delayed: t = nanoseconds(); if (t < j->r.deadline_at) { delay = j->r.deadline_at - t; } /* fall through */ default: r = enqueue_job(s, j, delay, 0); if (r < 1) twarnx("error recovering job %"PRIu64, j->r.id); } } return 1; } beanstalkd-1.11/sd-daemon.c000066400000000000000000000261701350566611300155730ustar00rootroot00000000000000/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ /*** Copyright 2010 Lennart Poettering Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ***/ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include #include #include #include "sd-daemon.h" int sd_listen_fds(int unset_environment) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) return 0; #else int r, fd; const char *e; char *p = NULL; unsigned long l; if (!(e = getenv("LISTEN_PID"))) { r = 0; goto finish; } errno = 0; l = strtoul(e, &p, 10); if (errno != 0) { r = -errno; goto finish; } if (!p || *p || l <= 0) { r = -EINVAL; goto finish; } /* Is this for us? */ if (getpid() != (pid_t) l) { r = 0; goto finish; } if (!(e = getenv("LISTEN_FDS"))) { r = 0; goto finish; } errno = 0; l = strtoul(e, &p, 10); if (errno != 0) { r = -errno; goto finish; } if (!p || *p) { r = -EINVAL; goto finish; } for (fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + (int) l; fd ++) { int flags; if ((flags = fcntl(fd, F_GETFD)) < 0) { r = -errno; goto finish; } if (flags & FD_CLOEXEC) continue; if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) { r = -errno; goto finish; } } r = (int) l; finish: if (unset_environment) { unsetenv("LISTEN_PID"); unsetenv("LISTEN_FDS"); } return r; #endif } int sd_is_fifo(int fd, const char *path) { struct stat st_fd; if (fd < 0) return -EINVAL; memset(&st_fd, 0, sizeof(st_fd)); if (fstat(fd, &st_fd) < 0) return -errno; if (!S_ISFIFO(st_fd.st_mode)) return 0; if (path) { struct stat st_path; memset(&st_path, 0, sizeof(st_path)); if (stat(path, &st_path) < 0) { if (errno == ENOENT || errno == ENOTDIR) return 0; return -errno; } return st_path.st_dev == st_fd.st_dev && st_path.st_ino == st_fd.st_ino; } return 1; } static int sd_is_socket_internal(int fd, int type, int listening) { struct stat st_fd; if (fd < 0 || type < 0) return -EINVAL; if (fstat(fd, &st_fd) < 0) return -errno; if (!S_ISSOCK(st_fd.st_mode)) return 0; if (type != 0) { int other_type = 0; socklen_t l = sizeof(other_type); if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &other_type, &l) < 0) return -errno; if (l != sizeof(other_type)) return -EINVAL; if (other_type != type) return 0; } if (listening >= 0) { int accepting = 0; socklen_t l = sizeof(accepting); if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &accepting, &l) < 0) return -errno; if (l != sizeof(accepting)) return -EINVAL; if (!accepting != !listening) return 0; } return 1; } union sockaddr_union { struct sockaddr sa; struct sockaddr_in in4; struct sockaddr_in6 in6; struct sockaddr_un un; struct sockaddr_storage storage; }; int sd_is_socket(int fd, int family, int type, int listening) { int r; if (family < 0) return -EINVAL; if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) return r; if (family > 0) { union sockaddr_union sockaddr; socklen_t l; memset(&sockaddr, 0, sizeof(sockaddr)); l = sizeof(sockaddr); if (getsockname(fd, &sockaddr.sa, &l) < 0) return -errno; if (l < sizeof(sa_family_t)) return -EINVAL; return sockaddr.sa.sa_family == family; } return 1; } int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) { union sockaddr_union sockaddr; socklen_t l; int r; if (family != 0 && family != AF_INET && family != AF_INET6) return -EINVAL; if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) return r; memset(&sockaddr, 0, sizeof(sockaddr)); l = sizeof(sockaddr); if (getsockname(fd, &sockaddr.sa, &l) < 0) return -errno; if (l < sizeof(sa_family_t)) return -EINVAL; if (sockaddr.sa.sa_family != AF_INET && sockaddr.sa.sa_family != AF_INET6) return 0; if (family > 0) if (sockaddr.sa.sa_family != family) return 0; if (port > 0) { if (sockaddr.sa.sa_family == AF_INET) { if (l < sizeof(struct sockaddr_in)) return -EINVAL; return htons(port) == sockaddr.in4.sin_port; } else { if (l < sizeof(struct sockaddr_in6)) return -EINVAL; return htons(port) == sockaddr.in6.sin6_port; } } return 1; } int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) { union sockaddr_union sockaddr; socklen_t l; int r; if ((r = sd_is_socket_internal(fd, type, listening)) <= 0) return r; memset(&sockaddr, 0, sizeof(sockaddr)); l = sizeof(sockaddr); if (getsockname(fd, &sockaddr.sa, &l) < 0) return -errno; if (l < sizeof(sa_family_t)) return -EINVAL; if (sockaddr.sa.sa_family != AF_UNIX) return 0; if (path) { if (length <= 0) length = strlen(path); if (length <= 0) /* Unnamed socket */ return l == sizeof(sa_family_t); if (path[0]) /* Normal path socket */ return (l >= sizeof(sa_family_t) + length + 1) && memcmp(path, sockaddr.un.sun_path, length+1) == 0; else /* Abstract namespace socket */ return (l == sizeof(sa_family_t) + length) && memcmp(path, sockaddr.un.sun_path, length) == 0; } return 1; } int sd_notify(int unset_environment, const char *state) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) || !defined(SOCK_CLOEXEC) return 0; #else int fd = -1, r; struct msghdr msghdr; struct iovec iovec; union sockaddr_union sockaddr; const char *e; if (!state) { r = -EINVAL; goto finish; } if (!(e = getenv("NOTIFY_SOCKET"))) return 0; /* Must be an abstract socket, or an absolute path */ if ((e[0] != '@' && e[0] != '/') || e[1] == 0) { r = -EINVAL; goto finish; } if ((fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0) { r = -errno; goto finish; } memset(&sockaddr, 0, sizeof(sockaddr)); sockaddr.sa.sa_family = AF_UNIX; strncpy(sockaddr.un.sun_path, e, sizeof(sockaddr.un.sun_path)); if (sockaddr.un.sun_path[0] == '@') sockaddr.un.sun_path[0] = 0; memset(&iovec, 0, sizeof(iovec)); iovec.iov_base = (char*) state; iovec.iov_len = strlen(state); memset(&msghdr, 0, sizeof(msghdr)); msghdr.msg_name = &sockaddr; msghdr.msg_namelen = sizeof(sa_family_t) + strlen(e); if (msghdr.msg_namelen > sizeof(struct sockaddr_un)) msghdr.msg_namelen = sizeof(struct sockaddr_un); msghdr.msg_iov = &iovec; msghdr.msg_iovlen = 1; if (sendmsg(fd, &msghdr, MSG_NOSIGNAL) < 0) { r = -errno; goto finish; } r = 1; finish: if (unset_environment) unsetenv("NOTIFY_SOCKET"); if (fd >= 0) close(fd); return r; #endif } int sd_notifyf(int unset_environment, const char *format, ...) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) return 0; #else va_list ap; char *p = NULL; int r; va_start(ap, format); r = vasprintf(&p, format, ap); va_end(ap); if (r < 0 || !p) return -ENOMEM; r = sd_notify(unset_environment, p); free(p); return r; #endif } int sd_booted(void) { #if defined(DISABLE_SYSTEMD) || !defined(__linux__) return 0; #else struct stat a, b; /* We simply test whether the systemd cgroup hierarchy is * mounted */ if (lstat("/sys/fs/cgroup", &a) < 0) return 0; if (lstat("/sys/fs/cgroup/systemd", &b) < 0) return 0; return a.st_dev != b.st_dev; #endif } beanstalkd-1.11/sd-daemon.h000066400000000000000000000240271350566611300155770ustar00rootroot00000000000000/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ #ifndef foosddaemonhfoo #define foosddaemonhfoo /*** Copyright 2010 Lennart Poettering Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ***/ #include #include #ifdef __cplusplus extern "C" { #endif /* Reference implementation of a few systemd related interfaces for writing daemons. These interfaces are trivial to implement. To simplify porting we provide this reference implementation. Applications are welcome to reimplement the algorithms described here if they do not want to include these two source files. The following functionality is provided: - Support for logging with log levels on stderr - File descriptor passing for socket-based activation - Daemon startup and status notification - Detection of systemd boots You may compile this with -DDISABLE_SYSTEMD to disable systemd support. This makes all those calls NOPs that are directly related to systemd (i.e. only sd_is_xxx() will stay useful). Since this is drop-in code we don't want any of our symbols to be exported in any case. Hence we declare hidden visibility for all of them. You may find an up-to-date version of these source files online: http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.h http://cgit.freedesktop.org/systemd/plain/src/sd-daemon.c This should compile on non-Linux systems, too, but with the exception of the sd_is_xxx() calls all functions will become NOPs. See sd-daemon(7) for more information. */ #if __GNUC__ >= 4 #define _sd_printf_attr_(a,b) __attribute__ ((format (printf, a, b))) #else #define _sd_printf_attr_(a,b) #endif #if (__GNUC__ >= 4) && !defined(SD_EXPORT_SYMBOLS) #define _sd_hidden_ __attribute__ ((visibility("hidden"))) #else #define _sd_hidden_ #endif /* Log levels for usage on stderr: fprintf(stderr, SD_NOTICE "Hello World!\n"); This is similar to printk() usage in the kernel. */ #define SD_EMERG "<0>" /* system is unusable */ #define SD_ALERT "<1>" /* action must be taken immediately */ #define SD_CRIT "<2>" /* critical conditions */ #define SD_ERR "<3>" /* error conditions */ #define SD_WARNING "<4>" /* warning conditions */ #define SD_NOTICE "<5>" /* normal but significant condition */ #define SD_INFO "<6>" /* informational */ #define SD_DEBUG "<7>" /* debug-level messages */ /* The first passed file descriptor is fd 3 */ #define SD_LISTEN_FDS_START 3 /* Returns how many file descriptors have been passed, or a negative errno code on failure. Optionally, removes the $LISTEN_FDS and $LISTEN_PID file descriptors from the environment (recommended, but problematic in threaded environments). If r is the return value of this function you'll find the file descriptors passed as fds SD_LISTEN_FDS_START to SD_LISTEN_FDS_START+r-1. Returns a negative errno style error code on failure. This function call ensures that the FD_CLOEXEC flag is set for the passed file descriptors, to make sure they are not passed on to child processes. If FD_CLOEXEC shall not be set, the caller needs to unset it after this call for all file descriptors that are used. See sd_listen_fds(3) for more information. */ int sd_listen_fds(int unset_environment) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is a FIFO in the file system stored under the specified path, 0 otherwise. If path is NULL a path name check will not be done and the call only verifies if the file descriptor refers to a FIFO. Returns a negative errno style error code on failure. See sd_is_fifo(3) for more information. */ int sd_is_fifo(int fd, const char *path) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is a socket of the specified family (AF_INET, ...) and type (SOCK_DGRAM, SOCK_STREAM, ...), 0 otherwise. If family is 0 a socket family check will not be done. If type is 0 a socket type check will not be done and the call only verifies if the file descriptor refers to a socket. If listening is > 0 it is verified that the socket is in listening mode. (i.e. listen() has been called) If listening is == 0 it is verified that the socket is not in listening mode. If listening is < 0 no listening mode check is done. Returns a negative errno style error code on failure. See sd_is_socket(3) for more information. */ int sd_is_socket(int fd, int family, int type, int listening) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is an Internet socket, of the specified family (either AF_INET or AF_INET6) and the specified type (SOCK_DGRAM, SOCK_STREAM, ...), 0 otherwise. If version is 0 a protocol version check is not done. If type is 0 a socket type check will not be done. If port is 0 a socket port check will not be done. The listening flag is used the same way as in sd_is_socket(). Returns a negative errno style error code on failure. See sd_is_socket_inet(3) for more information. */ int sd_is_socket_inet(int fd, int family, int type, int listening, uint16_t port) _sd_hidden_; /* Helper call for identifying a passed file descriptor. Returns 1 if the file descriptor is an AF_UNIX socket of the specified type (SOCK_DGRAM, SOCK_STREAM, ...) and path, 0 otherwise. If type is 0 a socket type check will not be done. If path is NULL a socket path check will not be done. For normal AF_UNIX sockets set length to 0. For abstract namespace sockets set length to the length of the socket name (including the initial 0 byte), and pass the full socket path in path (including the initial 0 byte). The listening flag is used the same way as in sd_is_socket(). Returns a negative errno style error code on failure. See sd_is_socket_unix(3) for more information. */ int sd_is_socket_unix(int fd, int type, int listening, const char *path, size_t length) _sd_hidden_; /* Informs systemd about changed daemon state. This takes a number of newline separated environment-style variable assignments in a string. The following variables are known: READY=1 Tells systemd that daemon startup is finished (only relevant for services of Type=notify). The passed argument is a boolean "1" or "0". Since there is little value in signalling non-readiness the only value daemons should send is "READY=1". STATUS=... Passes a single-line status string back to systemd that describes the daemon state. This is free-from and can be used for various purposes: general state feedback, fsck-like programs could pass completion percentages and failing programs could pass a human readable error message. Example: "STATUS=Completed 66% of file system check..." ERRNO=... If a daemon fails, the errno-style error code, formatted as string. Example: "ERRNO=2" for ENOENT. BUSERROR=... If a daemon fails, the D-Bus error-style error code. Example: "BUSERROR=org.freedesktop.DBus.Error.TimedOut" MAINPID=... The main pid of a daemon, in case systemd did not fork off the process itself. Example: "MAINPID=4711" Daemons can choose to send additional variables. However, it is recommened to prefix variable names not listed above with X_. Returns a negative errno-style error code on failure. Returns > 0 if systemd could be notified, 0 if it couldn't possibly because systemd is not running. Example: When a daemon finished starting up, it could issue this call to notify systemd about it: sd_notify(0, "READY=1"); See sd_notifyf() for more complete examples. See sd_notify(3) for more information. */ int sd_notify(int unset_environment, const char *state) _sd_hidden_; /* Similar to sd_notify() but takes a format string. Example 1: A daemon could send the following after initialization: sd_notifyf(0, "READY=1\n" "STATUS=Processing requests...\n" "MAINPID=%lu", (unsigned long) getpid()); Example 2: A daemon could send the following shortly before exiting, on failure: sd_notifyf(0, "STATUS=Failed to start up: %s\n" "ERRNO=%i", strerror(errno), errno); See sd_notifyf(3) for more information. */ int sd_notifyf(int unset_environment, const char *format, ...) _sd_printf_attr_(2,3) _sd_hidden_; /* Returns > 0 if the system was booted with systemd. Returns < 0 on error. Returns 0 if the system was not booted with systemd. Note that all of the functions above handle non-systemd boots just fine. You should NOT protect them with a call to this function. Also note that this function checks whether the system, not the user session is controlled by systemd. However the functions above work for both session and system services. See sd_booted(3) for more information. */ int sd_booted(void) _sd_hidden_; #ifdef __cplusplus } #endif #endif beanstalkd-1.11/serv.c000066400000000000000000000017261350566611300147030ustar00rootroot00000000000000#include #include #include #include "dat.h" struct Server srv = { Portdef, NULL, NULL, { Filesizedef, }, }; void srvserve(Server *s) { int r; Socket *sock; int64 period; if (sockinit() == -1) { twarnx("sockinit"); exit(1); } s->sock.x = s; s->sock.f = (Handle)srvaccept; s->conns.less = (Less)connless; s->conns.rec = (Record)connrec; r = listen(s->sock.fd, 1024); if (r == -1) { twarn("listen"); return; } r = sockwant(&s->sock, 'r'); if (r == -1) { twarn("sockwant"); exit(2); } for (;;) { period = prottick(s); int rw = socknext(&sock, period); if (rw == -1) { twarnx("socknext"); exit(1); } if (rw) { sock->f(sock->x, rw); } } } void srvaccept(Server *s, int ev) { h_accept(s->sock.fd, ev, s); } beanstalkd-1.11/testheap.c000066400000000000000000000130701350566611300155340ustar00rootroot00000000000000#include #include #include #include #include #include "ct/ct.h" #include "dat.h" void cttestheap_insert_one() { Heap h = {0}; job j; h.less = job_pri_less; h.rec = job_setheappos; j = make_job(1, 0, 1, 0, 0); assertf(j, "allocate job"); heapinsert(&h, j); assertf(h.len == 1, "h should contain one item."); assertf(j->heap_index == 0, "should match"); } void cttestheap_insert_and_remove_one() { Heap h = {0}; int r; job j, j1; h.less = job_pri_less; h.rec = job_setheappos; j1 = make_job(1, 0, 1, 0, 0); assertf(j1, "allocate job"); r = heapinsert(&h, j1); assertf(r, "insert should succeed"); j = heapremove(&h, 0); assertf(j == j1, "j1 should come back out"); assertf(h.len == 0, "h should be empty."); printf("j->heap_index is %zu\n", j->heap_index); assertf(j->heap_index == -1, "j's heap index should be invalid"); } void cttestheap_priority() { Heap h = {0}; int r; job j, j1, j2, j3; h.less = job_pri_less; h.rec = job_setheappos; j1 = make_job(1, 0, 1, 0, 0); j2 = make_job(2, 0, 1, 0, 0); j3 = make_job(3, 0, 1, 0, 0); assertf(j1, "allocate job"); assertf(j2, "allocate job"); assertf(j3, "allocate job"); r = heapinsert(&h, j2); assertf(r, "insert should succeed"); assertf(j2->heap_index == 0, "should match"); r = heapinsert(&h, j3); assertf(r, "insert should succeed"); assertf(j2->heap_index == 0, "should match"); assertf(j3->heap_index == 1, "should match"); r = heapinsert(&h, j1); assertf(r, "insert should succeed"); assertf(j1->heap_index == 0, "should match"); assertf(j2->heap_index == 2, "should match"); assertf(j3->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j1, "j1 should come out first."); assertf(j2->heap_index == 0, "should match"); assertf(j3->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j2, "j2 should come out second."); assertf(j3->heap_index == 0, "should match"); j = heapremove(&h, 0); assertf(j == j3, "j3 should come out third."); } void cttestheap_fifo_property() { Heap h = {0}; int r; job j, j3a, j3b, j3c; h.less = job_pri_less; h.rec = job_setheappos; j3a = make_job(3, 0, 1, 0, 0); j3b = make_job(3, 0, 1, 0, 0); j3c = make_job(3, 0, 1, 0, 0); assertf(j3a, "allocate job"); assertf(j3b, "allocate job"); assertf(j3c, "allocate job"); r = heapinsert(&h, j3a); assertf(r, "insert should succeed"); assertf(h.data[0] == j3a, "j3a should be in pos 0"); assertf(j3a->heap_index == 0, "should match"); r = heapinsert(&h, j3b); assertf(r, "insert should succeed"); assertf(h.data[1] == j3b, "j3b should be in pos 1"); assertf(j3a->heap_index == 0, "should match"); assertf(j3b->heap_index == 1, "should match"); r = heapinsert(&h, j3c); assertf(r, "insert should succeed"); assertf(h.data[2] == j3c, "j3c should be in pos 2"); assertf(j3a->heap_index == 0, "should match"); assertf(j3b->heap_index == 1, "should match"); assertf(j3c->heap_index == 2, "should match"); j = heapremove(&h, 0); assertf(j == j3a, "j3a should come out first."); assertf(j3b->heap_index == 0, "should match"); assertf(j3c->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j3b, "j3b should come out second."); assertf(j3c->heap_index == 0, "should match"); j = heapremove(&h, 0); assertf(j == j3c, "j3c should come out third."); } void cttestheap_many_jobs() { Heap h = {0}; uint last_pri; int r, i, n = 20; job j; h.less = job_pri_less; h.rec = job_setheappos; for (i = 0; i < n; i++) { j = make_job(1 + rand() % 8192, 0, 1, 0, 0); assertf(j, "allocation"); r = heapinsert(&h, j); assertf(r, "heapinsert"); } last_pri = 0; for (i = 0; i < n; i++) { j = heapremove(&h, 0); assertf(j->r.pri >= last_pri, "should come out in order"); last_pri = j->r.pri; } } void cttestheap_remove_k() { Heap h = {0}; uint last_pri; int r, i, c, n = 20; job j; h.less = job_pri_less; h.rec = job_setheappos; for (c = 0; c < 50; c++) { for (i = 0; i < n; i++) { j = make_job(1 + rand() % 8192, 0, 1, 0, 0); assertf(j, "allocation"); r = heapinsert(&h, j); assertf(r, "heapinsert"); } /* remove one from the middle */ heapremove(&h, 25); /* now make sure the rest are still a valid heap */ last_pri = 0; for (i = 1; i < n; i++) { j = heapremove(&h, 0); assertf(j->r.pri >= last_pri, "should come out in order"); last_pri = j->r.pri; } } } void ctbenchheapinsert(int n) { job *j; int i; j = calloc(n, sizeof *j); assert(j); for (i = 0; i < n; i++) { j[i] = make_job(1, 0, 1, 0, 0); assert(j[i]); j[i]->r.pri = -j[i]->r.id; } Heap h = {0}; h.less = job_pri_less; h.rec = job_setheappos; ctresettimer(); for (i = 0; i < n; i++) { heapinsert(&h, j[i]); } } void ctbenchheapremove(int n) { Heap h = {0}; job j; int i; h.less = job_pri_less; h.rec = job_setheappos; for (i = 0; i < n; i++) { j = make_job(1, 0, 1, 0, 0); assertf(j, "allocate job"); heapinsert(&h, j); } ctresettimer(); for (i = 0; i < n; i++) { heapremove(&h, 0); } } beanstalkd-1.11/testjobs.c000066400000000000000000000055751350566611300155670ustar00rootroot00000000000000#include #include #include #include #include #include "ct/ct.h" #include "dat.h" static tube default_tube; void cttestjob_creation() { job j; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job(1, 0, 1, 0, default_tube); assertf(j->r.pri == 1, "priority should match"); } void cttestjob_cmp_pris() { job a, b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(1 << 27, 0, 1, 0, default_tube); assertf(job_pri_less(a, b), "should be less"); } void cttestjob_cmp_ids() { job a, b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(1, 0, 1, 0, default_tube); b->r.id <<= 49; assertf(job_pri_less(a, b), "should be less"); } void cttestjob_large_pris() { job a, b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(-5, 0, 1, 0, default_tube); assertf(job_pri_less(a, b), "should be less"); a = make_job(-5, 0, 1, 0, default_tube); b = make_job(1, 0, 1, 0, default_tube); assertf(!job_pri_less(a, b), "should not be less"); } void cttestjob_hash_free() { job j; uint64 jid = 83; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job_with_id(0, 0, 1, 0, default_tube, jid); job_free(j); assertf(!job_find(jid), "job should be missing"); } void cttestjob_hash_free_next() { job a, b; uint64 aid = 97, bid = 12386; TUBE_ASSIGN(default_tube, make_tube("default")); b = make_job_with_id(0, 0, 1, 0, default_tube, bid); a = make_job_with_id(0, 0, 1, 0, default_tube, aid); assertf(a->ht_next == b, "b should be chained to a"); job_free(b); assertf(a->ht_next == NULL, "job should be missing"); } void cttestjob_all_jobs_used() { job j, x; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job(0, 0, 1, 0, default_tube); assertf(get_all_jobs_used() == 1, "should match"); x = allocate_job(10); assertf(get_all_jobs_used() == 1, "should match"); job_free(x); assertf(get_all_jobs_used() == 1, "should match"); job_free(j); assertf(get_all_jobs_used() == 0, "should match"); } void cttestjob_100_000_jobs() { int i; TUBE_ASSIGN(default_tube, make_tube("default")); for (i = 0; i < 100000; i++) { make_job(0, 0, 1, 0, default_tube); } assertf(get_all_jobs_used() == 100000, "should match"); for (i = 1; i <= 100000; i++) { job_free(job_find(i)); } fprintf(stderr, "get_all_jobs_used() => %zu\n", get_all_jobs_used()); assertf(get_all_jobs_used() == 0, "should match"); } void ctbenchmakejob(int n) { int i; TUBE_ASSIGN(default_tube, make_tube("default")); for (i = 0; i < n; i++) { make_job(0, 0, 1, 0, default_tube); } } beanstalkd-1.11/testserv.c000066400000000000000000000672451350566611300156130ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ct/ct.h" #include "dat.h" static int srvpid, port, fd, size; static int64 timeout = 5000000000LL; // 5s static byte fallocpat[3]; static int wrapfalloc(int fd, int size) { static int c = 0; printf("\nwrapfalloc: fd=%d size=%d\n", fd, size); if (c >= sizeof(fallocpat) || !fallocpat[c++]) { return ENOSPC; } return rawfalloc(fd, size); } static void muststart(char *a0, char *a1, char *a2, char *a3, char *a4) { srvpid = fork(); if (srvpid < 0) { twarn("fork"); exit(1); } if (srvpid > 0) { printf("%s %s %s %s %s\n", a0, a1, a2, a3, a4); printf("start server pid=%d\n", srvpid); usleep(100000); // .1s; time for the child to bind to its port return; } /* now in child */ execlp(a0, a0, a1, a2, a3, a4, NULL); } static int mustdiallocal(int port) { int r, fd; struct sockaddr_in addr = {}; addr.sin_family = AF_INET; addr.sin_port = htons(port); r = inet_aton("127.0.0.1", &addr.sin_addr); if (!r) { errno = EINVAL; twarn("inet_aton"); exit(1); } fd = socket(PF_INET, SOCK_STREAM, 0); if (fd == -1) { twarn("socket"); exit(1); } r = connect(fd, (struct sockaddr*)&addr, sizeof addr); if (r == -1) { twarn("connect"); exit(1); } return fd; } #define SERVER() (progname=__func__, mustforksrv()) static int mustforksrv() { int r, len, port, ok; struct sockaddr_in addr; srv.sock.fd = make_server_socket("127.0.0.1", "0"); if (srv.sock.fd == -1) { puts("mustforksrv failed"); exit(1); } len = sizeof(addr); r = getsockname(srv.sock.fd, (struct sockaddr*)&addr, (socklen_t*)&len); if (r == -1 || len > sizeof(addr)) { puts("mustforksrv failed"); exit(1); } port = ntohs(addr.sin_port); srvpid = fork(); if (srvpid < 0) { twarn("fork"); exit(1); } if (srvpid > 0) { printf("start server port=%d pid=%d\n", port, srvpid); return port; } /* now in child */ prot_init(); if (srv.wal.use) { struct job list = {}; // We want to make sure that only one beanstalkd tries // to use the wal directory at a time. So acquire a lock // now and never release it. if (!waldirlock(&srv.wal)) { twarnx("failed to lock wal dir %s", srv.wal.dir); exit(10); } list.prev = list.next = &list; walinit(&srv.wal, &list); ok = prot_replay(&srv, &list); if (!ok) { twarnx("failed to replay log"); exit(11); } } srvserve(&srv); /* does not return */ exit(1); /* satisfy the compiler */ } static char * readline(int fd) { int r, i = 0; char c = 0, p = 0; static char buf[1024]; fd_set rfd; struct timeval tv; printf("<%d ", fd); fflush(stdout); for (;;) { FD_ZERO(&rfd); FD_SET(fd, &rfd); tv.tv_sec = timeout / 1000000000; tv.tv_usec = (timeout/1000) % 1000000; r = select(fd+1, &rfd, NULL, NULL, &tv); switch (r) { case 1: break; case 0: fputs("timeout", stderr); exit(8); case -1: perror("select"); exit(1); default: fputs("unknown error", stderr); exit(3); } r = read(fd, &c, 1); if (r == -1) { perror("write"); exit(1); } if (i >= sizeof(buf)-1) { fputs("response too big", stderr); exit(4); } putc(c, stdout); fflush(stdout); buf[i++] = c; if (p == '\r' && c == '\n') { break; } p = c; } buf[i] = '\0'; return buf; } static void ckresp(int fd, char *exp) { char *line; line = readline(fd); assertf(strcmp(exp, line) == 0, "\"%s\" != \"%s\"", exp, line); } static void ckrespsub(int fd, char *sub) { char *line; line = readline(fd); assertf(strstr(line, sub), "\"%s\" not in \"%s\"", sub, line); } static void writefull(int fd, char *s, int n) { int c; for (; n; n -= c) { c = write(fd, s, n); if (c == -1) { perror("write"); exit(1); } s += c; } } static void mustsend(int fd, char *s) { writefull(fd, s, strlen(s)); printf(">%d %s", fd, s); fflush(stdout); } static int filesize(char *path) { int r; struct stat s; r = stat(path, &s); if (r == -1) { twarn("stat"); exit(1); } return s.st_size; } static int exist(char *path) { int r; struct stat s; r = stat(path, &s); return r != -1; } void cttestpause() { int64 s; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "INSERTED 1\r\n"); s = nanoseconds(); mustsend(fd, "pause-tube default 1\r\n"); ckresp(fd, "PAUSED\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 1\r\n"); ckresp(fd, "x\r\n"); assert(nanoseconds() - s >= 1000000000); // 1s } void cttestunderscore() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use x_y\r\n"); ckresp(fd, "USING x_y\r\n"); } void cttest2cmdpacket() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use a\r\nuse b\r\n"); ckresp(fd, "USING a\r\n"); ckresp(fd, "USING b\r\n"); } void cttesttoobig() { job_data_size_limit = 10; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 11\r\n"); mustsend(fd, "delete 9999\r\n"); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "JOB_TOO_BIG\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttestdeleteready() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestmultitube() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use abc\r\n"); ckresp(fd, "USING abc\r\n"); mustsend(fd, "put 999999 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "use def\r\n"); ckresp(fd, "USING def\r\n"); mustsend(fd, "put 99 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "watch abc\r\n"); ckresp(fd, "WATCHING 2\r\n"); mustsend(fd, "watch def\r\n"); ckresp(fd, "WATCHING 3\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 2 0\r\n"); } void cttestnonegativedelay() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 512 -1 100 0\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); } void cttestomittimeleft() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 5 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntime-left: 0\n"); } void cttestsmalldelay() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 1 1 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void ctteststatstube() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use tubea\r\n"); ckresp(fd, "USING tubea\r\n"); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nname: tubea\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-urgent: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-ready: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-reserved: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-delayed: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-buried: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntotal-jobs: 1\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-using: 1\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-watching: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-waiting: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-delete: 1\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-pause-tube: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause: 0\n"); mustsend(fd, "stats-tube tubea\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause-time-left: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nname: default\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-urgent: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-ready: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-reserved: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-delayed: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-jobs-buried: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntotal-jobs: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-using: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-watching: 1\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncurrent-waiting: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-delete: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ncmd-pause-tube: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause: 0\n"); mustsend(fd, "stats-tube default\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npause-time-left: 0\n"); } void cttestttrlarge() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 120 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 4294 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "put 0 0 4295 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 3\r\n"); mustsend(fd, "put 0 0 4296 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 4\r\n"); mustsend(fd, "put 0 0 4297 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 5\r\n"); mustsend(fd, "put 0 0 5000 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 6\r\n"); mustsend(fd, "put 0 0 21600 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 7\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 120\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4294\n"); mustsend(fd, "stats-job 3\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4295\n"); mustsend(fd, "stats-job 4\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4296\n"); mustsend(fd, "stats-job 5\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 4297\n"); mustsend(fd, "stats-job 6\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 5000\n"); mustsend(fd, "stats-job 7\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 21600\n"); } void cttestttrsmall() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 1\n"); } void cttestzerodelay() { port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 1 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttestreservewithtimeout2conn() { int fd0, fd1; job_data_size_limit = 10; port = SERVER(); fd0 = mustdiallocal(port); fd1 = mustdiallocal(port); mustsend(fd0, "watch foo\r\n"); ckresp(fd0, "WATCHING 2\r\n"); mustsend(fd0, "reserve-with-timeout 1\r\n"); mustsend(fd1, "watch foo\r\n"); ckresp(fd1, "WATCHING 2\r\n"); timeout = 1100000000; // 1.1s ckresp(fd0, "TIMED_OUT\r\n"); } void cttestunpausetube() { int fd0, fd1; port = SERVER(); fd0 = mustdiallocal(port); fd1 = mustdiallocal(port); mustsend(fd0, "put 0 0 0 0\r\n"); mustsend(fd0, "\r\n"); ckresp(fd0, "INSERTED 1\r\n"); mustsend(fd0, "pause-tube default 86400\r\n"); ckresp(fd0, "PAUSED\r\n"); mustsend(fd1, "reserve\r\n"); mustsend(fd0, "pause-tube default 0\r\n"); ckresp(fd0, "PAUSED\r\n"); // ckresp will time out if this takes too long, so the // test will not pass. ckresp(fd1, "RESERVED 1 0\r\n"); ckresp(fd1, "\r\n"); } void cttestbinlogemptyexit() { srv.wal.dir = ctdir(); srv.wal.use = 1; job_data_size_limit = 10; port = SERVER(); kill(srvpid, 9); waitpid(srvpid, NULL, 0); port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttestbinlogbury() { srv.wal.dir = ctdir(); srv.wal.use = 1; job_data_size_limit = 10; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 0\r\n"); ckresp(fd, "\r\n"); mustsend(fd, "bury 1 0\r\n"); ckresp(fd, "BURIED\r\n"); } void cttestbinlogbasic() { srv.wal.dir = ctdir(); srv.wal.use = 1; job_data_size_limit = 10; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 0\r\n"); mustsend(fd, "\r\n"); ckresp(fd, "INSERTED 1\r\n"); kill(srvpid, 9); waitpid(srvpid, NULL, 0); port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestbinlogsizelimit() { int i = 0; char *b2; int gotsize; size = 1024; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); b2 = fmtalloc("%s/binlog.2", ctdir()); while (!exist(b2)) { mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, fmtalloc("INSERTED %d\r\n", ++i)); } gotsize = filesize(fmtalloc("%s/binlog.1", ctdir())); assertf(gotsize == size, "binlog.1 %d != %d", gotsize, size); gotsize = filesize(b2); assertf(gotsize == size, "binlog.2 %d != %d", gotsize, size); } void cttestbinlogallocation() { int i = 0; size = 601; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); for (i = 1; i <= 96; i++) { mustsend(fd, "put 0 0 120 22\r\n"); mustsend(fd, "job payload xxxxxxxxxx\r\n"); ckresp(fd, fmtalloc("INSERTED %d\r\n", i)); } for (i = 1; i <= 96; i++) { mustsend(fd, fmtalloc("delete %d\r\n", i)); ckresp(fd, "DELETED\r\n"); } } void cttestbinlogread() { srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "use test\r\n"); ckresp(fd, "USING test\r\n"); mustsend(fd, "put 0 0 120 4\r\n"); mustsend(fd, "test\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 120 4\r\n"); mustsend(fd, "tes1\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "watch test\r\n"); ckresp(fd, "WATCHING 2\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 4\r\n"); ckresp(fd, "test\r\n"); mustsend(fd, "release 1 1 1\r\n"); ckresp(fd, "RELEASED\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 2 4\r\n"); ckresp(fd, "tes1\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "DELETED\r\n"); kill(srvpid, 9); waitpid(srvpid, NULL, 0); port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "watch test\r\n"); ckresp(fd, "WATCHING 2\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 4\r\n"); ckresp(fd, "test\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "NOT_FOUND\r\n"); } void cttestbinlogdiskfull() { size = 1000; falloc = &wrapfalloc; fallocpat[0] = 1; fallocpat[2] = 1; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 3\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 4\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "OUT_OF_MEMORY\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 6\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 7\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 8\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 9\r\n"); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 3\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 4\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 6\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 7\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 8\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 9\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestbinlogdiskfulldelete() { size = 1000; falloc = &wrapfalloc; fallocpat[0] = 1; fallocpat[1] = 1; srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = size; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 3\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 4\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 5\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 6\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 7\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "INSERTED 8\r\n"); mustsend(fd, "put 0 0 100 50\r\n"); mustsend(fd, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"); ckresp(fd, "OUT_OF_MEMORY\r\n"); assert(exist(fmtalloc("%s/binlog.1", ctdir()))); mustsend(fd, "delete 1\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 3\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 4\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 5\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 6\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 7\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "delete 8\r\n"); ckresp(fd, "DELETED\r\n"); } void cttestbinlogv5() { char portstr[10]; if (system("which beanstalkd-1.4.6") != 0) { puts("beanstalkd 1.4.6 not found, skipping"); exit(0); } progname=__func__; port = (rand()&0xfbff) + 1024; sprintf(portstr, "%d", port); muststart("beanstalkd-1.4.6", "-b", ctdir(), "-p", portstr); fd = mustdiallocal(port); mustsend(fd, "use test\r\n"); ckresp(fd, "USING test\r\n"); mustsend(fd, "put 1 2 3 4\r\n"); mustsend(fd, "test\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 4 3 2 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 2\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 3\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 4\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 3\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); kill(srvpid, 9); waitpid(srvpid, NULL, 0); srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.syncrate = 0; srv.wal.wantsync = 1; port = SERVER(); fd = mustdiallocal(port); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 1\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 2\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 3\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 1\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nid: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntube: test\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nstate: delayed\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\npri: 4\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ndelay: 3\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nttr: 2\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreserves: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\ntimeouts: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nreleases: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nburies: 0\n"); mustsend(fd, "stats-job 2\r\n"); ckrespsub(fd, "OK "); ckrespsub(fd, "\nkicks: 0\n"); } static void benchputdeletesize(int n, int size) { port = SERVER(); fd = mustdiallocal(port); char buf[50], put[50]; char body[size+1]; memset(body, 'a', size); body[size] = 0; ctsetbytes(size); sprintf(put, "put 0 0 0 %d\r\n", size); int i; for (i = 0; i < n; i++) { mustsend(fd, put); mustsend(fd, body); mustsend(fd, "\r\n"); ckrespsub(fd, "INSERTED "); sprintf(buf, "delete %d\r\n", i+1); mustsend(fd, buf); ckresp(fd, "DELETED\r\n"); } } void ctbenchputdelete8byte(int n) { benchputdeletesize(n, 8); } void ctbenchputdelete1k(int n) { benchputdeletesize(n, 1024); } void ctbenchputdelete8k(int n) { benchputdeletesize(n, 8192); } beanstalkd-1.11/testutil.c000066400000000000000000000065351350566611300156040ustar00rootroot00000000000000#include #include #include #include #include #include #include "ct/ct.h" #include "dat.h" void cttestallocf() { char *got; got = fmtalloc("hello, %s %d", "world", 5); assertf(strcmp("hello, world 5", got) == 0, "got \"%s\"", got); } void cttestoptnone() { char *args[] = { NULL, }; optparse(&srv, args); assert(strcmp(srv.port, Portdef) == 0); assert(srv.addr == NULL); assert(job_data_size_limit == JOB_DATA_SIZE_LIMIT_DEFAULT); assert(srv.wal.filesize == Filesizedef); assert(srv.wal.nocomp == 0); assert(srv.wal.wantsync == 0); assert(srv.user == NULL); assert(srv.wal.dir == NULL); assert(srv.wal.use == 0); assert(verbose == 0); } static void success(void) { _exit(0); } void cttestoptminus() { char *args[] = { "-", NULL, }; atexit(success); optparse(&srv, args); assertf(0, "optparse failed to call exit"); } void cttestoptp() { char *args[] = { "-p1234", NULL, }; optparse(&srv, args); assert(strcmp(srv.port, "1234") == 0); } void cttestoptl() { char *args[] = { "-llocalhost", NULL, }; optparse(&srv, args); assert(strcmp(srv.addr, "localhost") == 0); } void cttestoptlseparate() { char *args[] = { "-l", "localhost", NULL, }; optparse(&srv, args); assert(strcmp(srv.addr, "localhost") == 0); } void cttestoptz() { char *args[] = { "-z1234", NULL, }; optparse(&srv, args); assert(job_data_size_limit == 1234); } void cttestopts() { char *args[] = { "-s1234", NULL, }; optparse(&srv, args); assert(srv.wal.filesize == 1234); } void cttestoptc() { char *args[] = { "-n", "-c", NULL, }; optparse(&srv, args); assert(srv.wal.nocomp == 0); } void cttestoptn() { char *args[] = { "-n", NULL, }; optparse(&srv, args); assert(srv.wal.nocomp == 1); } void cttestoptf() { char *args[] = { "-f1234", NULL, }; optparse(&srv, args); assert(srv.wal.syncrate == 1234000000); assert(srv.wal.wantsync == 1); } void cttestoptF() { char *args[] = { "-f1234", "-F", NULL, }; optparse(&srv, args); assert(srv.wal.wantsync == 0); } void cttestoptu() { char *args[] = { "-ukr", NULL, }; optparse(&srv, args); assert(strcmp(srv.user, "kr") == 0); } void cttestoptb() { char *args[] = { "-bfoo", NULL, }; optparse(&srv, args); assert(strcmp(srv.wal.dir, "foo") == 0); assert(srv.wal.use == 1); } void cttestoptV() { char *args[] = { "-V", NULL, }; optparse(&srv, args); assert(verbose == 1); } void cttestoptV_V() { char *args[] = { "-V", "-V", NULL, }; optparse(&srv, args); assert(verbose == 2); } void cttestoptVVV() { char *args[] = { "-VVV", NULL, }; optparse(&srv, args); assert(verbose == 3); } void cttestoptVnVu() { char *args[] = { "-VnVukr", NULL, }; optparse(&srv, args); assert(verbose == 2); assert(srv.wal.nocomp == 1); assert(strcmp(srv.user, "kr") == 0); } beanstalkd-1.11/time.c000066400000000000000000000004651350566611300146610ustar00rootroot00000000000000#include #include #include #include "dat.h" int64 nanoseconds(void) { int r; struct timeval tv; r = gettimeofday(&tv, 0); if (r != 0) return warnx("gettimeofday"), -1; // can't happen return ((int64)tv.tv_sec)*1000000000 + ((int64)tv.tv_usec)*1000; } beanstalkd-1.11/tube.c000066400000000000000000000032461350566611300146620ustar00rootroot00000000000000#include #include #include #include "dat.h" struct ms tubes; tube make_tube(const char *name) { tube t; t = new(struct tube); if (!t) return NULL; t->name[MAX_TUBE_NAME_LEN - 1] = '\0'; strncpy(t->name, name, MAX_TUBE_NAME_LEN - 1); if (t->name[MAX_TUBE_NAME_LEN - 1] != '\0') twarnx("truncating tube name"); t->ready.less = job_pri_less; t->delay.less = job_delay_less; t->ready.rec = job_setheappos; t->delay.rec = job_setheappos; t->buried = (struct job) { }; t->buried.prev = t->buried.next = &t->buried; ms_init(&t->waiting, NULL, NULL); return t; } static void tube_free(tube t) { prot_remove_tube(t); free(t->ready.data); free(t->delay.data); ms_clear(&t->waiting); free(t); } void tube_dref(tube t) { if (!t) return; if (t->refs < 1) return twarnx("refs is zero for tube: %s", t->name); --t->refs; if (t->refs < 1) tube_free(t); } void tube_iref(tube t) { if (!t) return; ++t->refs; } static tube make_and_insert_tube(const char *name) { int r; tube t = NULL; t = make_tube(name); if (!t) return NULL; /* We want this global tube list to behave like "weak" refs, so don't * increment the ref count. */ r = ms_append(&tubes, t); if (!r) return tube_dref(t), (tube) 0; return t; } tube tube_find(const char *name) { tube t; size_t i; for (i = 0; i < tubes.used; i++) { t = tubes.items[i]; if (strncmp(t->name, name, MAX_TUBE_NAME_LEN) == 0) return t; } return NULL; } tube tube_find_or_make(const char *name) { return tube_find(name) ? : make_and_insert_tube(name); } beanstalkd-1.11/util.c000066400000000000000000000122271350566611300146770ustar00rootroot00000000000000#include #include #include #include #include #include #include "sd-daemon.h" #include "dat.h" const char *progname; static void vwarnx(const char *err, const char *fmt, va_list args) __attribute__((format(printf, 2, 0))); static void vwarnx(const char *err, const char *fmt, va_list args) { fprintf(stderr, "%s: ", progname); if (fmt) { vfprintf(stderr, fmt, args); if (err) fprintf(stderr, ": %s", err); } fputc('\n', stderr); } void warn(const char *fmt, ...) { char *err = strerror(errno); /* must be done first thing */ va_list args; va_start(args, fmt); vwarnx(err, fmt, args); va_end(args); } void warnx(const char *fmt, ...) { va_list args; va_start(args, fmt); vwarnx(NULL, fmt, args); va_end(args); } char* fmtalloc(char *fmt, ...) { int n; char *buf; va_list ap; // find out how much space is needed va_start(ap, fmt); n = vsnprintf(0, 0, fmt, ap) + 1; // include space for trailing NUL va_end(ap); buf = malloc(n); if (buf) { va_start(ap, fmt); vsnprintf(buf, n, fmt, ap); va_end(ap); } return buf; } // Zalloc allocates n bytes of zeroed memory and // returns a pointer to it. // If insufficient memory is available, zalloc returns 0. void* zalloc(int n) { void *p; p = malloc(n); if (p) { memset(p, 0, n); } return p; } static void warn_systemd_ignored_option(char *opt, char *arg) { if (sd_listen_fds(0) > 0) { warnx("inherited listen fd; ignoring option: %s %s", opt, arg); } } static void usage(int code) __attribute__ ((noreturn)); static void usage(int code) { fprintf(stderr, "Use: %s [OPTIONS]\n" "\n" "Options:\n" " -b DIR write-ahead log directory\n" " -f MS fsync at most once every MS milliseconds" " (use -f0 for \"always fsync\")\n" " -F never fsync (default)\n" " -l ADDR listen on address (default is 0.0.0.0)\n" " -p PORT listen on port (default is " Portdef ")\n" " -u USER become user and group\n" " -z BYTES set the maximum job size in bytes (default is %d)\n" " -s BYTES set the size of each write-ahead log file (default is %d)\n" " (will be rounded up to a multiple of 512 bytes)\n" " -c compact the binlog (default)\n" " -n do not compact the binlog\n" " -v show version information\n" " -V increase verbosity\n" " -h show this help\n", progname, JOB_DATA_SIZE_LIMIT_DEFAULT, Filesizedef); exit(code); } static char *flagusage(char *flag) __attribute__ ((noreturn)); static char * flagusage(char *flag) { warnx("flag requires an argument: %s", flag); usage(5); } static size_t parse_size_t(char *str) { char r, x; size_t size; r = sscanf(str, "%zu%c", &size, &x); if (1 != r) { warnx("invalid size: %s", str); usage(5); } return size; } void optparse(Server *s, char **argv) { int64 ms; char *arg, c, *tmp; # define EARGF(x) (*arg ? (tmp=arg,arg="",tmp) : *argv ? *argv++ : (x)) while ((arg = *argv++) && *arg++ == '-' && *arg) { while ((c = *arg++)) { switch (c) { case 'p': s->port = EARGF(flagusage("-p")); warn_systemd_ignored_option("-p", s->port); break; case 'l': s->addr = EARGF(flagusage("-l")); warn_systemd_ignored_option("-l", s->addr); break; case 'z': job_data_size_limit = parse_size_t(EARGF(flagusage("-z"))); break; case 's': s->wal.filesize = parse_size_t(EARGF(flagusage("-s"))); break; case 'c': s->wal.nocomp = 0; break; case 'n': s->wal.nocomp = 1; break; case 'f': ms = (int64)parse_size_t(EARGF(flagusage("-f"))); s->wal.syncrate = ms * 1000000; s->wal.wantsync = 1; break; case 'F': s->wal.wantsync = 0; break; case 'u': s->user = EARGF(flagusage("-u")); break; case 'b': s->wal.dir = EARGF(flagusage("-b")); s->wal.use = 1; break; case 'h': usage(0); case 'v': printf("beanstalkd %s\n", version); exit(0); case 'V': verbose++; break; default: warnx("unknown flag: %s", arg-2); usage(5); } } } if (arg) { warnx("unknown argument: %s", arg-1); usage(5); } } beanstalkd-1.11/verc.sh000077500000000000000000000001051350566611300150440ustar00rootroot00000000000000#!/bin/sh printf 'const char version[] = "' ./vers.sh printf '";\n' beanstalkd-1.11/vers.sh000077500000000000000000000000161350566611300150650ustar00rootroot00000000000000printf '1.11' beanstalkd-1.11/walg.c000066400000000000000000000221011350566611300146440ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include "dat.h" static int reserve(Wal *w, int n); // Reads w->dir for files matching binlog.NNN, // sets w->next to the next unused number, and // returns the minimum number. // If no files are found, sets w->next to 1 and // returns a large number. static int walscandir(Wal *w) { static char base[] = "binlog."; static const int len = sizeof(base) - 1; DIR *d; struct dirent *e; int min = 1<<30; int max = 0; int n; char *p; d = opendir(w->dir); if (!d) return min; while ((e = readdir(d))) { if (strncmp(e->d_name, base, len) == 0) { n = strtol(e->d_name+len, &p, 10); if (p && *p == '\0') { if (n > max) max = n; if (n < min) min = n; } } } closedir(d); w->next = max + 1; return min; } void walgc(Wal *w) { File *f; while (w->head && !w->head->refs) { f = w->head; w->head = f->next; if (w->tail == f) { w->tail = f->next; // also, f->next == NULL } w->nfile--; unlink(f->path); free(f->path); free(f); } } // returns 1 on success, 0 on error. static int usenext(Wal *w) { File *f; f = w->cur; if (!f->next) { twarnx("there is no next wal file"); return 0; } w->cur = f->next; filewclose(f); return 1; } static int ratio(Wal *w) { int64 n, d; d = w->alive + w->resv; n = (int64)w->nfile * (int64)w->filesize - d; if (!d) return 0; return n / d; } // Returns the number of bytes reserved or 0 on error. static int walresvmigrate(Wal *w, job j) { int z = 0; // reserve only space for the migrated full job record // space for the delete is already reserved z += sizeof(int); z += strlen(j->tube->name); z += sizeof(Jobrec); z += j->r.body_size; return reserve(w, z); } static void moveone(Wal *w) { job j; if (w->head == w->cur || w->head->next == w->cur) { // no point in moving a job return; } j = w->head->jlist.fnext; if (!j || j == &w->head->jlist) { // head holds no jlist; can't happen twarnx("head holds no jlist"); return; } if (!walresvmigrate(w, j)) { // it will not fit, so we'll try again later return; } filermjob(w->head, j); w->nmig++; walwrite(w, j); } static void walcompact(Wal *w) { int r; for (r=ratio(w); r>=2; r--) { moveone(w); } } static void walsync(Wal *w) { int64 now; now = nanoseconds(); if (w->wantsync && now >= w->lastsync+w->syncrate) { w->lastsync = now; if (fsync(w->cur->fd) == -1) { twarn("fsync"); } } } // Walwrite writes j to the log w (if w is enabled). // On failure, walwrite disables w and returns 0; on success, it returns 1. // Unlke walresv*, walwrite should never fail because of a full disk. // If w is disabled, then walwrite takes no action and returns 1. int walwrite(Wal *w, job j) { int r = 0; if (!w->use) return 1; if (w->cur->resv > 0 || usenext(w)) { if (j->file) { r = filewrjobshort(w->cur, j); } else { r = filewrjobfull(w->cur, j); } } if (!r) { filewclose(w->cur); w->use = 0; } w->nrec++; return r; } void walmaint(Wal *w) { if (w->use) { if (!w->nocomp) { walcompact(w); } walsync(w); } } static int makenextfile(Wal *w) { File *f; f = new(File); if (!f) { twarnx("OOM"); return 0; } if (!fileinit(f, w, w->next)) { free(f); twarnx("OOM"); return 0; } filewopen(f); if (!f->iswopen) { free(f->path); free(f); return 0; } w->next++; fileadd(f, w); return 1; } static void moveresv(File *to, File *from, int n) { from->resv -= n; from->free += n; to->resv += n; to->free -= n; } static int needfree(Wal *w, int n) { if (w->tail->free >= n) return n; if (makenextfile(w)) return n; return 0; } // Ensures: // 1. b->resv is congruent to n (mod z). // 2. x->resv is congruent to 0 (mod z) for each future file x. // Assumes (and preserves) that b->resv >= n. // Reserved space is conserved (neither created nor destroyed); // we just move it around to preserve the invariant. // We might have to allocate a new file. // Returns 1 on success, otherwise 0. If there was a failure, // w->tail is not updated. static int balancerest(Wal *w, File *b, int n) { int rest, c, r; static const int z = sizeof(int) + sizeof(Jobrec); if (!b) return 1; rest = b->resv - n; r = rest % z; if (r == 0) return balancerest(w, b->next, 0); c = z - r; if (w->tail->resv >= c && b->free >= c) { moveresv(b, w->tail, c); return balancerest(w, b->next, 0); } if (needfree(w, r) != r) { twarnx("needfree"); return 0; } moveresv(w->tail, b, r); return balancerest(w, b->next, 0); } // Ensures: // 1. w->cur->resv >= n. // 2. w->cur->resv is congruent to n (mod z). // 3. x->resv is congruent to 0 (mod z) for each future file x. // (where z is the size of a delete record in the wal). // Reserved space is conserved (neither created nor destroyed); // we just move it around to preserve the invariant. // We might have to allocate a new file. // Returns 1 on success, otherwise 0. If there was a failure, // w->tail is not updated. static int balance(Wal *w, int n) { int r; // Invariant 1 // (this loop will run at most once) while (w->cur->resv < n) { int m = w->cur->resv; r = needfree(w, m); if (r != m) { twarnx("needfree"); return 0; } moveresv(w->tail, w->cur, m); usenext(w); } // Invariants 2 and 3 return balancerest(w, w->cur, n); } // Returns the number of bytes successfully reserved: either 0 or n. static int reserve(Wal *w, int n) { int r; // return value must be nonzero but is otherwise ignored if (!w->use) return 1; if (w->cur->free >= n) { w->cur->free -= n; w->cur->resv += n; w->resv += n; return n; } r = needfree(w, n); if (r != n) { twarnx("needfree"); return 0; } w->tail->free -= n; w->tail->resv += n; w->resv += n; if (!balance(w, n)) { // error; undo the reservation w->resv -= n; w->tail->resv -= n; w->tail->free += n; return 0; } return n; } // Returns the number of bytes reserved or 0 on error. int walresvput(Wal *w, job j) { int z = 0; // reserve space for the initial job record z += sizeof(int); z += strlen(j->tube->name); z += sizeof(Jobrec); z += j->r.body_size; // plus space for a delete to come later z += sizeof(int); z += sizeof(Jobrec); return reserve(w, z); } // Returns the number of bytes reserved or 0 on error. int walresvupdate(Wal *w, job j) { int z = 0; z +=sizeof(int); z +=sizeof(Jobrec); return reserve(w, z); } // Returns the number of locks acquired: either 0 or 1. int waldirlock(Wal *w) { int r; int fd; struct flock lk; char *path; size_t path_length; path_length = strlen(w->dir) + strlen("/lock") + 1; if ((path = malloc(path_length)) == NULL) { twarn("malloc"); return 0; } r = snprintf(path, path_length, "%s/lock", w->dir); fd = open(path, O_WRONLY|O_CREAT, 0600); free(path); if (fd == -1) { twarn("open"); return 0; } lk.l_type = F_WRLCK; lk.l_whence = SEEK_SET; lk.l_start = 0; lk.l_len = 0; r = fcntl(fd, F_SETLK, &lk); if (r) { twarn("fcntl"); return 0; } // intentionally leak fd, since we never want to close it // and we'll never need it again return 1; } void walread(Wal *w, job list, int min) { File *f; int i, fd; int err = 0; for (i = min; i < w->next; i++) { f = new(File); if (!f) { twarnx("OOM"); exit(1); } if (!fileinit(f, w, i)) { free(f); twarnx("OOM"); exit(1); } fd = open(f->path, O_RDONLY); if (fd < 0) { twarn("open %s", f->path); free(f->path); free(f); continue; } f->fd = fd; fileadd(f, w); err |= fileread(f, list); close(fd); } if (err) { warnx("Errors reading one or more WAL files."); warnx("Continuing. You may be missing data."); } } void walinit(Wal *w, job list) { int min; min = walscandir(w); walread(w, list, min); // first writable file if (!makenextfile(w)) { twarnx("makenextfile"); exit(1); } w->cur = w->tail; }