pax_global_header00006660000000000000000000000064116666507770014537gustar00rootroot0000000000000052 comment=2f73945382982e1e20b41888f486ca92553e1e79 stud-0.3/000077500000000000000000000000001166665077700123605ustar00rootroot00000000000000stud-0.3/LICENSE000066400000000000000000000027701166665077700133730ustar00rootroot00000000000000Copyright 2011 Bump Technologies, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY BUMP TECHNOLOGIES, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BUMP TECHNOLOGIES, INC. OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of Bump Technologies, Inc. stud-0.3/Makefile000066400000000000000000000020121166665077700140130ustar00rootroot00000000000000# [g]make USE_xxxx=1 # # USE_SHARED_CACHE : enable/disable a shared session cache (disabled by default) DESTDIR = PREFIX = /usr/local BINDIR = $(PREFIX)/bin MANDIR = $(PREFIX)/share/man CFLAGS = -O2 -g -std=c99 -fno-strict-aliasing -Wall -W -D_GNU_SOURCE LDFLAGS = -lssl -lcrypto -lev OBJS = stud.o ringbuffer.o all: realall # Shared cache feature ifneq ($(USE_SHARED_CACHE),) CFLAGS += -DUSE_SHARED_CACHE -DUSE_SYSCALL_FUTEX OBJS += shctx.o ebtree/libebtree.a ALL += ebtree ebtree/libebtree.a: $(wildcard ebtree/*.c) make -C ebtree ebtree: @[ -d ebtree ] || ( \ echo "*** Download libebtree at http://1wt.eu/tools/ebtree/" ; \ echo "*** Untar it and make a link named 'ebtree' to point on it"; \ exit 1 ) endif ALL += stud realall: $(ALL) stud: $(OBJS) $(CC) $(LDFLAGS) -o $@ $^ install: $(ALL) install -d $(DESTDIR)$(BINDIR) install stud $(DESTDIR)$(BINDIR) install -d $(DESTDIR)$(MANDIR)/man8 install -m 644 doc/stud.8 $(DESTDIR)$(MANDIR)/man8 clean: rm -f stud $(OBJS) .PHONY: all realall stud-0.3/README.md000066400000000000000000000145631166665077700136500ustar00rootroot00000000000000stud - The Scalable TLS Unwrapping Daemon ========================================= `stud` is a network proxy that terminates TLS/SSL connections and forwards the unencrypted traffic to some backend. It's designed to handle 10s of thousands of connections efficiently on multicore machines. It follows a process-per-core model; a parent process spawns N children who each `accept()` on a common socket to distribute connected clients among them. Within each child, asynchronous socket I/O is conducted across the local connections using `libev` and `OpenSSL`'s nonblocking API. By default, `stud` has an overhead of ~200KB per connection--it preallocates some buffer space for data in flight between frontend and backend. `stud` has very few features--it's designed to be paired with an intelligent backend like haproxy or nginx. It maintains a strict 1:1 connection pattern with this backend handler so that the backend can dictate throttling behavior, maxmium connection behavior, availability of service, etc. `stud` will optionally write the client IP address as the first few octets (depending on IPv4 or IPv6) to the backend--or provide that information using HAProxy's PROXY protocol. In this way, backends who care about the client IP can still access it even though `stud` itself appears to be the connected client. Thanks to a contribution from Emeric at Exceliance (the folks behind HAProxy), a special build of `stud` can be made that utilitizes shared memory to use a common session cache between all child processes. This can speed up large `stud` deployments by avoiding client renegotiation. Releases --------- Please be aware of the policy regarding releases, code stability, and security: * In git, the tip of the master branch should always build on Linux and FreeBSD, and is likely to be as stable as any other changeset. A careful review of patches is conducted before being pushed to github. * Periodically, a version tag will be pushed to github for an old(er) changeset--0.1, 0.2, etc. These tags mark a particular release of `stud` that has seen heavy testing and several weeks of production stability. Conservative users are advised to use a tag. * `stud` has optional builds that utilize shared memory-based SSL contexts to keep a session cache between many child processes. The use of these builds can dramatically speed up SSL handshakes on many-core deployments. However, it's important to acknowledge the inevitable theoretical security tradeoffs associated with the use of this (substantially more complex) binary. Therefore, the deeply paranoid are advised to use only the standard `stud` binary at the cost of some performance. Requirements and Limitations ---------------------------- `stud` requires: libev >= 4 openssl (recent, >=1.0.0 recommended) Stud currently works on Linux, OpenBSD, FreeBSD, and MacOSX. It has been tested the most heavily on Linux/x86_64. While porting it to other POSIX platforms is likely trivial, it hasn't be done yet. Patches welcome! If you're handling a large number of connections, you'll probably want to raise `ulimit -n` before running `stud`. It's very strongly recommended to not run `stud` as root; ideally, it would be run as a user ("stud", perhaps) that does nothing but run `stud`. Stud will setuid (using -u) after binding if you need to bind to a low port (< 1024). Installing ---------- To install `stud`: $ make $ sudo make install Usage ----- The only required argument is a path to a PEM file that contains the certificate (or a chain of certificates) and private key. Detail about the entire set of options can be found by invoking `stud -h`: Encryption Methods: --tls TLSv1 (default) --ssl SSLv3 (implies no TLSv1) -c CIPHER_SUITE set allowed ciphers (default is OpenSSL defaults) -e ENGINE set OpenSSL engine Socket: -b HOST,PORT backend [connect] (default is "127.0.0.1,8000") -f HOST,PORT frontend [bind] (default is "*,8443") Performance: -n CORES number of worker processes (default is 1) -B BACKLOG set listen backlog size (default is 100) Security: -r PATH chroot -u USERNAME set gid/uid after binding the socket Logging: -q be quiet; emit only error messages -s send log message to syslog in addition to stderr/stdout Special: --write-ip write 1 octet with the IP family followed by the IP address in 4 (IPv4) or 16 (IPv6) octets little-endian to backend before the actual data --write-proxy write HaProxy's PROXY (IPv4 or IPv6) protocol line before actual data `stud` uses no configuration file. Serving HTTPS ------------- If you're using `stud` for HTTPS, please make sure to use the `--ssl` option! Diffie–Hellman -------------- To use DH with stud, you will need to add some bytes to your pem file: % openssl dhparam -rand - 1024 >> PEMFILE Be sure to set your cipher suite appropriately: -c DHE-RSA-AES256-SHA Authors ------- `stud` was originally written by Jamie Turner (@jamwt) and is maintained by the Bump (http://bu.mp) server team. It currently (6/11) provides server-side TLS termination for over 40 million Bump users. Contributors: * Colin Percival @cperciva -- early security audit and code review * Frank DENIS @jedisct1 -- port to BSD, IPv6 support, various fixes * Denis Bilenko -- HAProxy PROXY protocol support, chroot/setuid * Joe Damato -- Diffie-Hellman parameter loading * Benjamin Pineau -- Chained cert loading, various fixes, performance tweaks * Carl Perry/Dreamhost -- IPv6 PROXY support * Emeric Brun/Exceliance -- Session resumption and shared-memory session cache * Vladimir Dronnikov -- Logging cleanup * James Golick/BitLove Inc. -- SIGPIPE fixes and child-reaping * Joe Williams -- Syslog support * Jason Cook -- SSL option tweaks (performance) * Artur Bergman -- Socket tweaks (performance) stud-0.3/ringbuffer.c000066400000000000000000000070241166665077700146600ustar00rootroot00000000000000/** * Copyright 2011 Bump Technologies, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are * permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this list of * conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, this list * of conditions and the following disclaimer in the documentation and/or other materials * provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY BUMP TECHNOLOGIES, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BUMP TECHNOLOGIES, INC. OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are those of the * authors and should not be interpreted as representing official policies, either expressed * or implied, of Bump Technologies, Inc. * **/ #include "ringbuffer.h" #include /* Initialize a ringbuffer structure to empty */ void ringbuffer_init(ringbuffer *rb) { rb->head = &rb->slots[0]; rb->tail = &rb->slots[0]; rb->used = 0; int x; for (x=0; xslots[x].next = &(rb->slots[(x + 1) % RING_SLOTS]); } /** READ FUNCTIONS **/ /* Return a char * that represents the current unconsumed buffer */ char * ringbuffer_read_next(ringbuffer *rb, int * length) { assert(rb->used); *length = rb->head->left; return rb->head->ptr; } /* Mark consumption of only part of the read head buffer */ void ringbuffer_read_skip(ringbuffer *rb, int length) { assert(rb->used); rb->head->ptr += length; rb->head->left -= length; } /* Pop a consumed (fully read) head from the buffer */ void ringbuffer_read_pop(ringbuffer *rb) { assert(rb->used); rb->head = rb->head->next; rb->used--; } /** WRITE FUNCTIONS **/ /* Return the tail ptr (current target of new writes) */ char * ringbuffer_write_ptr(ringbuffer *rb) { assert(rb->used < RING_SLOTS); return rb->tail->data; } /* Mark the tail appended for `length` bytes, and move the cursor * to the next slot */ void ringbuffer_write_append(ringbuffer *rb, int length) { assert(rb->used < RING_SLOTS); rb->used++; rb->tail->ptr = rb->tail->data; rb->tail->left = length; rb->tail = rb->tail->next; } /** RING STATE FUNCTIONS **/ /* Used size of the ringbuffer */ int ringbuffer_size(ringbuffer *rb) { return rb->used; } /* Used size of the ringbuffer */ int ringbuffer_capacity(ringbuffer *rb) { (void) rb; return RING_SLOTS; } /* Is the ringbuffer completely empty (implies: no data to be written) */ int ringbuffer_is_empty(ringbuffer *rb) { return rb->used == 0; } /* Is the ringbuffer completely full (implies: no more data should be read) */ int ringbuffer_is_full(ringbuffer *rb) { return rb->used == RING_SLOTS; } stud-0.3/ringbuffer.h000066400000000000000000000050431166665077700146640ustar00rootroot00000000000000/** * Copyright 2011 Bump Technologies, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are * permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this list of * conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, this list * of conditions and the following disclaimer in the documentation and/or other materials * provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY BUMP TECHNOLOGIES, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BUMP TECHNOLOGIES, INC. OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are those of the * authors and should not be interpreted as representing official policies, either expressed * or implied, of Bump Technologies, Inc. * **/ #ifndef RINGBUFFER_H #define RINGBUFFER_H #include /* Tweak these for potential memory/throughput tradeoffs */ #define RING_SLOTS 3 #define RING_DATA_LEN 1024 * 32 typedef struct bufent { char data[RING_DATA_LEN]; char *ptr; size_t left; struct bufent *next; } bufent; typedef struct ringbuffer { bufent slots[RING_SLOTS]; bufent *head; // reads from the head bufent *tail; // writes to the tail size_t used; } ringbuffer; void ringbuffer_init(ringbuffer *rb); char * ringbuffer_read_next(ringbuffer *rb, int * length); void ringbuffer_read_skip(ringbuffer *rb, int length); void ringbuffer_read_pop(ringbuffer *rb); char * ringbuffer_write_ptr(ringbuffer *rb); void ringbuffer_write_append(ringbuffer *rb, int length); int ringbuffer_size(ringbuffer *rb); int ringbuffer_capacity(ringbuffer *rb); int ringbuffer_is_empty(ringbuffer *rb); int ringbuffer_is_full(ringbuffer *rb); #endif /* RINGBUFFER_H */ stud-0.3/shctx.c000066400000000000000000000202161166665077700136560ustar00rootroot00000000000000/* * shctx.c * * Copyright (C) 2011 EXCELIANCE * * Author: Emeric Brun - emeric@exceliance.fr * */ #include #ifdef USE_SYSCALL_FUTEX #include #include #include #else /* USE_SYSCALL_FUTEX */ #include #endif /* USE_SYSCALL_FUTEX */ #include "ebtree/ebmbtree.h" #include "shctx.h" #ifndef SHSESS_MAX_DATA_LEN #define SHSESS_MAX_DATA_LEN 512 #endif struct shared_session { struct ebmb_node key; unsigned char key_data[SSL_MAX_SSL_SESSION_ID_LENGTH]; int data_len; unsigned char data[SHSESS_MAX_DATA_LEN]; struct shared_session *p; struct shared_session *n; }; struct shared_context { #ifdef USE_SYSCALL_FUTEX unsigned int waiters; #else /* USE_SYSCALL_FUTEX */ pthread_mutex_t mutex; #endif struct shared_session active; struct shared_session free; }; /* Static shared context */ static struct shared_context *shctx = NULL; /* Lock functions */ #ifdef USE_SYSCALL_FUTEX static inline unsigned int xchg(unsigned int *ptr, unsigned int x) { __asm volatile("lock xchgl %0,%1" : "=r" (x), "+m" (*ptr) : "0" (x) : "memory"); return x; } static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned int new) { unsigned int ret; __asm volatile("lock cmpxchgl %2,%1" : "=a" (ret), "+m" (*ptr) : "r" (new), "0" (old) : "memory"); return ret; } static inline unsigned char atomic_inc(unsigned int *ptr) { unsigned char ret; __asm volatile("lock incl %0\n" "setne %1\n" : "+m" (*ptr), "=qm" (ret) : : "memory"); return ret; } static inline unsigned char atomic_dec(unsigned int *ptr) { unsigned char ret; __asm volatile("lock decl %0\n" "setne %1\n" : "+m" (*ptr), "=qm" (ret) : : "memory"); return ret; } static inline void shared_context_lock(void) { unsigned int x; x = cmpxchg(&shctx->waiters, 0, 1); if (x) { if (x != 2) x = xchg(&shctx->waiters, 2); while (x) { syscall(SYS_futex, &shctx->waiters, FUTEX_WAIT, 2, NULL, 0, 0); x = xchg(&shctx->waiters, 2); } } } static inline void shared_context_unlock(void) { if (atomic_dec(&shctx->waiters)) { shctx->waiters = 0; syscall(SYS_futex, &shctx->waiters, FUTEX_WAKE, 1, NULL, 0, 0); } } #else /* USE_SYSCALL_FUTEX */ #define shared_context_lock(v) pthread_mutex_lock(&shctx->mutex) #define shared_context_unlock(v) pthread_mutex_unlock(&shctx->mutex) #endif /* List Macros */ #define shsess_unset(s) (s)->n->p = (s)->p; \ (s)->p->n = (s)->n; #define shsess_set_free(s) shsess_unset(s) \ (s)->p = &shctx->free; \ (s)->n = shctx->free.n; \ shctx->free.n->p = s; \ shctx->free.n = s; #define shsess_set_active(s) shsess_unset(s) \ (s)->p = &shctx->active; \ (s)->n = shctx->active.n; \ shctx->active.n->p = s; \ shctx->active.n = s; #define shsess_get_next() (shctx->free.p == shctx->free.n) ? \ shctx->active.p : shctx->free.p; /* Tree Macros */ #define shsess_tree_delete(s) ebmb_delete(&(s)->key); #define shsess_tree_insert(s) (struct shared_session *)ebmb_insert(&shctx->active.key.node.branches, \ &(s)->key, SSL_MAX_SSL_SESSION_ID_LENGTH); #define shsess_tree_lookup(k) (struct shared_session *)ebmb_lookup(&shctx->active.key.node.branches, \ (k), SSL_MAX_SSL_SESSION_ID_LENGTH); /* Other Macros */ #define shsess_set_key(s,k,l) { memcpy((s)->key_data, (k), (l)); \ if ((l) < SSL_MAX_SSL_SESSION_ID_LENGTH) \ memset((s)->key_data+(l), 0, SSL_MAX_SSL_SESSION_ID_LENGTH-(l)); }; /* SSL context callbacks */ /* SSL callback used on new session creation */ int shctx_new_cb(SSL *ssl, SSL_SESSION *sess) { (void)ssl; struct shared_session *shsess; unsigned char data[SHSESS_MAX_DATA_LEN],*p; unsigned int data_len; /* check if session reserved size in aligned buffer is large enougth for the ASN1 encode session */ data_len=i2d_SSL_SESSION(sess, NULL); if(data_len > SHSESS_MAX_DATA_LEN) return 1; /* process ASN1 session encoding before the lock: lower cost */ p = data; i2d_SSL_SESSION(sess, &p); shared_context_lock(); shsess = shsess_get_next(); shsess_tree_delete(shsess); shsess_set_key(shsess, sess->session_id, sess->session_id_length); /* it returns the already existing node or current node if none, never returns null */ shsess = shsess_tree_insert(shsess); /* store ASN1 encoded session into cache */ shsess->data_len = data_len; memcpy(shsess->data, data, data_len); shsess_set_active(shsess); shared_context_unlock(); return 1; /* leave the session in local cache for reuse */ } /* SSL callback used on lookup an existing session cause none found in internal cache */ SSL_SESSION *shctx_get_cb(SSL *ssl, unsigned char *key, int key_len, int *do_copy) { (void)ssl; struct shared_session *shsess; unsigned char data[SHSESS_MAX_DATA_LEN], *p; unsigned char tmpkey[SSL_MAX_SSL_SESSION_ID_LENGTH]; unsigned int data_len; SSL_SESSION *sess; /* allow the session to be freed automatically by openssl */ *do_copy = 0; /* tree key is zeros padded sessionid */ if ( key_len < SSL_MAX_SSL_SESSION_ID_LENGTH ) { memcpy(tmpkey, key, key_len); memset(tmpkey+key_len, 0, SSL_MAX_SSL_SESSION_ID_LENGTH-key_len); key = tmpkey; } /* lock cache */ shared_context_lock(); /* lookup for session */ shsess = shsess_tree_lookup(key); if(!shsess) { /* no session found: unlock cache and exit */ shared_context_unlock(); return NULL; } /* copy ASN1 session data to decode outside the lock */ data_len = shsess->data_len; memcpy(data, shsess->data, shsess->data_len); shsess_set_active(shsess); shared_context_unlock(); /* decode ASN1 session */ p = data; sess = d2i_SSL_SESSION(NULL, (const unsigned char **)&p, data_len); return sess; } /* SSL callback used to signal session is no more used in internal cache */ void shctx_remove_cb(SSL_CTX *ctx, SSL_SESSION *sess) { (void)ctx; struct shared_session *shsess; unsigned char tmpkey[SSL_MAX_SSL_SESSION_ID_LENGTH]; unsigned char *key = sess->session_id; /* tree key is zeros padded sessionid */ if ( sess->session_id_length < SSL_MAX_SSL_SESSION_ID_LENGTH ) { memcpy(tmpkey, sess->session_id, sess->session_id_length); memset(tmpkey+sess->session_id_length, 0, SSL_MAX_SSL_SESSION_ID_LENGTH-sess->session_id_length); key = tmpkey; } shared_context_lock(); /* lookup for session */ shsess = shsess_tree_lookup(key); if ( shsess ) { shsess_set_free(shsess); } /* unlock cache */ shared_context_unlock(); } /* Init shared memory context if not allocated and set SSL context callbacks * size is the max number of stored session * Returns: -1 on alloc failure, size if performs context alloc, and 0 if just perform * callbacks registration */ int shared_context_init(SSL_CTX *ctx, int size) { int ret = 0; if (!shctx) { int i; #ifndef USE_SYSCALL_FUTEX pthread_mutexattr_t attr; #endif /* USE_SYSCALL_FUTEX */ struct shared_session *prev,*cur; shctx = (struct shared_context *)mmap(NULL, sizeof(struct shared_context)+(size*sizeof(struct shared_session)), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (!shctx) return -1; #ifdef USE_SYSCALL_FUTEX shctx->waiters = 0; #else pthread_mutexattr_init(&attr); pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); pthread_mutex_init(&shctx->mutex, &attr); #endif memset(&shctx->active.key, 0, sizeof(struct ebmb_node)); memset(&shctx->free.key, 0, sizeof(struct ebmb_node)); /* No duplicate authorized in tree: */ shctx->active.key.node.branches.b[1] = (void *)1; cur = &shctx->active; cur->n = cur->p = cur; cur = &shctx->free; for ( i = 0 ; i < size ; i++) { prev = cur; cur = (struct shared_session *)((char *)prev + sizeof(struct shared_session)); prev->n = cur; cur->p = prev; } cur->n = &shctx->free; shctx->free.p = cur; ret = size; } /* set SSL internal cache size to external cache / 8 + 123 */ SSL_CTX_sess_set_cache_size(ctx, size >> 3 | 0x3ff); /* Set callbacks */ SSL_CTX_sess_set_new_cb(ctx, shctx_new_cb); SSL_CTX_sess_set_get_cb(ctx, shctx_get_cb); SSL_CTX_sess_set_remove_cb(ctx, shctx_remove_cb); return ret; } stud-0.3/shctx.h000066400000000000000000000007321166665077700136640ustar00rootroot00000000000000/* * shctx.h * * Copyright (C) 2011 EXCELIANCE * * Author: Emeric Brun - emeric@exceliance.fr * */ #ifndef SHCTX_H #define SHCTX_H #include /* Init shared memory context if not allocated and set SSL context callbacks * size is the max number of stored session * Returns: -1 on alloc failure, size if performs context alloc, and 0 if just * perform callbacks registration */ int shared_context_init(SSL_CTX *ctx, int size); #endif /* SHCTX_H */ stud-0.3/stud.8000066400000000000000000000106371166665077700134370ustar00rootroot00000000000000.\" Copyright (c) 2011 Vincent Bernat .\" .\" Redistribution and use in source and binary forms, with or without modification, are .\" permitted provided that the following conditions are met: .\" .\" 1. Redistributions of source code must retain the above copyright notice, this list of .\" conditions and the following disclaimer. .\" .\" 2. Redistributions in binary form must reproduce the above copyright notice, this list .\" of conditions and the following disclaimer in the documentation and/or other materials .\" provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY BUMP TECHNOLOGIES, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED .\" WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND .\" FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BUMP TECHNOLOGIES, INC. OR .\" CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR .\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR .\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON .\" ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING .\" NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF .\" ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .\" The views and conclusions contained in the software and documentation are those of the .\" authors and should not be interpreted as representing official policies, either expressed .\" or implied, of Bump Technologies, Inc. .\" .Dd $Mdocdate: September 23 2011 $ .Dt STUD 8 .Os .Sh NAME .Nm stud .Nd The Scalable TLS Unwrapping Daemon .Sh SYNOPSIS .Nm .Op Fl -tls .Op Fl -ssl .Op Fl c Ar ciphers .Op Fl e Ar engine .Op Fl b Ar host,port .Op Fl f Ar host,port .Op Fl n Ar cores .Op Fl B Ar backlog .Op Fl C Ar cache .Op Fl r Ar path .Op Fl u Ar username .Op Fl qs .Op Fl -write-ip .Op Fl -write-proxy .Ar certificate.pem .Sh DESCRIPTION .Nm is a network proxy that terminates TLS/SSL connections and forwards the unencrypted traffic to some backend. It's designed to handle 10s of thousands of connections efficiently on multicore machines. .Pp .Nm has very few features -- it's designed to be paired with an intelligent backend like haproxy or nginx. It maintains a strict 1:1 connection pattern with this backend handler so that the backend can dictate throttling behavior, maxmium connection behavior, availability of service, etc. .Pp The only required argument is a path to a PEM file that contains the certificate (or a chain of certificates) and private key. It should also contain DH parameter if you wish to use Diffie-Hellman cipher suites. .Pp The options are as follows: .Bl -tag -width Ds .It Fl -tls Use TLSv1 (default). .It Fl -ssl Use only SSLv3 and no TLSv1. .It Fl c Ar ciphers Set allowed ciphers using the same format as .Ic openssl ciphers . For example, you can use .Ar RSA:!COMPLEMENTOFALL . .It Fl e Ar engine Specify an OpenSSL engine by its unique ID. The engine will be used by default for all algorithms. The keyword .Ar auto can be used to load all available engines. .It Fl b Ar host,port Define backend. Default is .Ar 127.0.0.1,8000 . Incoming connections will be unwrapped and sent to this IP and port. .It Fl f Ar host,port Define frontend. Default is .Ar *,8443 . Incoming connections will be accepted to this IP and port and will be sent to the backend defined above. .It Fl n Ar cores Use .Ar cores worker processes. Default is 1. .It Fl B Ar backlog Set listen backlog size. Default is 100. .It Fl C Ar cache Set shared cache size in sessions. By default, no shared cache is used. .It Fl r Ar path Chroot to the given path. By default, no chroot is done. .It Fl u Ar username Set GID/UID after binding the socket. By default, no privilege is dropped. .It Fl q Be quiet. Only emit error messages. .It Fl s Send messages to syslog in addition to .Em stderr and .Em stdout . .It Fl -write-ip Write 1 octet with the IP family followed by the IP address in 4 (IPv4) or 16 (IPv6) octets little-endian to backend before the actual data. .It Fl -write-proxy Write HaProxy's PROXY (IPv4 or IPv6) protocol line before actual data. .El .Sh SEE ALSO .Xr ciphers 1SSL , .Xr dhparam 1SSL , .Xr haproxy 1 .Sh AUTHORS .Nm was originally written by Jamie Turner (@jamwt) and is maintained by the Bump server team. It currently provides server-side TLS termination for over 40 million Bump users. stud-0.3/stud.c000066400000000000000000001130361166665077700135070ustar00rootroot00000000000000/** * Copyright 2011 Bump Technologies, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are * permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this list of * conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, this list * of conditions and the following disclaimer in the documentation and/or other materials * provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY BUMP TECHNOLOGIES, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BUMP TECHNOLOGIES, INC. OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are those of the * authors and should not be interpreted as representing official policies, either expressed * or implied, of Bump Technologies, Inc. * **/ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ringbuffer.h" #include "shctx.h" #ifndef MSG_NOSIGNAL # define MSG_NOSIGNAL 0 #endif #ifndef AI_ADDRCONFIG # define AI_ADDRCONFIG 0 #endif /* Globals */ static struct ev_loop *loop; static struct addrinfo *backaddr; static pid_t master_pid; static ev_io listener; static int listener_socket; static int child_num; static pid_t *child_pids; static SSL_CTX *ssl_ctx; /* Command line Options */ typedef enum { ENC_TLS, ENC_SSL } ENC_TYPE; typedef struct stud_options { ENC_TYPE ETYPE; int WRITE_IP_OCTET; int WRITE_PROXY_LINE; const char* CHROOT; uid_t UID; gid_t GID; const char *FRONT_IP; const char *FRONT_PORT; const char *BACK_IP; const char *BACK_PORT; long NCORES; const char *CERT_FILE; const char *CIPHER_SUITE; const char *ENGINE; int BACKLOG; #ifdef USE_SHARED_CACHE int SHARED_CACHE; #endif int QUIET; int SYSLOG; int TCP_KEEPALIVE; } stud_options; static stud_options OPTIONS = { ENC_TLS, // ETYPE 0, // WRITE_IP_OCTET 0, // WRITE_PROXY_LINE NULL, // CHROOT 0, // UID 0, // GID NULL, // FRONT_IP "8443", // FRONT_PORT "127.0.0.1", // BACK_IP "8000", // BACK_PORT 1, // NCORES NULL, // CERT_FILE NULL, // CIPHER_SUITE NULL, // ENGINE 100, // BACKLOG #ifdef USE_SHARED_CACHE 0, // SHARED_CACHE #endif 0, // QUIET 0, // SYSLOG 3600 // TCP_KEEPALIVE }; static char tcp_proxy_line[128] = ""; /* What agent/state requests the shutdown--for proper half-closed * handling */ typedef enum _SHUTDOWN_REQUESTOR { SHUTDOWN_HARD, SHUTDOWN_DOWN, SHUTDOWN_UP } SHUTDOWN_REQUESTOR; /* * Proxied State * * All state associated with one proxied connection */ typedef struct proxystate { ringbuffer ring_down; /* pushing bytes from client to backend */ ringbuffer ring_up; /* pushing bytes from backend to client */ ev_io ev_r_up; /* Upstream write event */ ev_io ev_w_up; /* Upstream read event */ ev_io ev_r_handshake; /* Downstream write event */ ev_io ev_w_handshake; /* Downstream read event */ ev_io ev_r_down; /* Downstream write event */ ev_io ev_w_down; /* Downstream read event */ int fd_up; /* Upstream (client) socket */ int fd_down; /* Downstream (backend) socket */ int want_shutdown:1; /* Connection is half-shutdown */ int handshaked:1; /* Initial handshake happened */ int renegotiation:1; /* Renegotation is occuring */ SSL *ssl; /* OpenSSL SSL state */ struct sockaddr_storage remote_ip; /* Remote ip returned from `accept` */ } proxystate; #define LOG(...) \ do { \ if (!OPTIONS.QUIET) fprintf(stdout, __VA_ARGS__); \ if (OPTIONS.SYSLOG) syslog(LOG_INFO, __VA_ARGS__); \ } while(0) #define ERR(...) \ do { \ fprintf(stderr, __VA_ARGS__); \ if (OPTIONS.SYSLOG) syslog(LOG_ERR, __VA_ARGS__); \ } while(0) /* set a file descriptor (socket) to non-blocking mode */ static void setnonblocking(int fd) { int flag = 1; assert (ioctl(fd, FIONBIO, &flag) == 0); } /* set a tcp socket to use TCP Keepalive */ static void settcpkeepalive(int fd) { int optval = 1; socklen_t optlen = sizeof(optval); if(setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen) < 0) { ERR("Error activating SO_KEEPALIVE on client socket: %s", strerror(errno)); } optval = OPTIONS.TCP_KEEPALIVE; optlen = sizeof(optval); if(setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, &optval, optlen) < 0) { ERR("Error setting TCP_KEEPIDLE on client socket: %s", strerror(errno)); } } static void fail(const char* s) { perror(s); exit(1); } #ifndef OPENSSL_NO_DH static int init_dh(SSL_CTX *ctx, const char *cert) { DH *dh; BIO *bio; assert(cert); bio = BIO_new_file(cert, "r"); if (!bio) { ERR_print_errors_fp(stderr); return -1; } dh = PEM_read_bio_DHparams(bio, NULL, NULL, NULL); BIO_free(bio); if (!dh) { ERR("{core} Note: no DH parameters found in %s\n", cert); return -1; } LOG("{core} Using DH parameters from %s\n", cert); SSL_CTX_set_tmp_dh(ctx, dh); LOG("{core} DH initialized with %d bit key\n", 8*DH_size(dh)); DH_free(dh); return 0; } #endif /* OPENSSL_NO_DH */ /* This callback function is executed while OpenSSL processes the SSL * handshake and does SSL record layer stuff. It's used to trap * client-initiated renegotiations. */ static void info_callback(const SSL *ssl, int where, int ret) { (void)ret; if (where & SSL_CB_HANDSHAKE_START) { proxystate *ps = (proxystate *)SSL_get_app_data(ssl); if (ps->handshaked) { ps->renegotiation = 1; LOG("{core} SSL renegotiation asked by client\n"); } } } /* Init library and load specified certificate. * Establishes a SSL_ctx, to act as a template for * each connection */ static SSL_CTX * init_openssl() { SSL_library_init(); SSL_load_error_strings(); SSL_CTX *ctx = NULL; long ssloptions = SSL_OP_NO_SSLv2 | SSL_OP_ALL | SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION; if (OPTIONS.ETYPE == ENC_TLS) ctx = SSL_CTX_new(TLSv1_server_method()); else if (OPTIONS.ETYPE == ENC_SSL) ctx = SSL_CTX_new(SSLv23_server_method()); else assert(OPTIONS.ETYPE == ENC_TLS || OPTIONS.ETYPE == ENC_SSL); #ifdef SSL_OP_NO_COMPRESSION ssloptions |= SSL_OP_NO_COMPRESSION; #endif SSL_CTX_set_options(ctx, ssloptions); SSL_CTX_set_info_callback(ctx, info_callback); if (SSL_CTX_use_certificate_chain_file(ctx, OPTIONS.CERT_FILE) <= 0) { ERR_print_errors_fp(stderr); exit(1); } if (SSL_CTX_use_RSAPrivateKey_file(ctx, OPTIONS.CERT_FILE, SSL_FILETYPE_PEM) <= 0) { ERR_print_errors_fp(stderr); exit(1); } #ifndef OPENSSL_NO_DH init_dh(ctx, OPTIONS.CERT_FILE); #endif /* OPENSSL_NO_DH */ if (OPTIONS.ENGINE) { ENGINE *e = NULL; ENGINE_load_builtin_engines(); if (!strcmp(OPTIONS.ENGINE, "auto")) ENGINE_register_all_complete(); else { if ((e = ENGINE_by_id(OPTIONS.ENGINE)) == NULL || !ENGINE_init(e) || !ENGINE_set_default(e, ENGINE_METHOD_ALL)) { ERR_print_errors_fp(stderr); exit(1); } LOG("{core} will use OpenSSL engine %s.\n", ENGINE_get_id(e)); ENGINE_finish(e); ENGINE_free(e); } } if (OPTIONS.CIPHER_SUITE) if (SSL_CTX_set_cipher_list(ctx, OPTIONS.CIPHER_SUITE) != 1) ERR_print_errors_fp(stderr); #ifdef USE_SHARED_CACHE if (OPTIONS.SHARED_CACHE) { if (shared_context_init(ctx, OPTIONS.SHARED_CACHE) < 0) { ERR("Unable to alloc memory for shared cache.\n"); exit(1); } } #endif return ctx; } static void prepare_proxy_line(struct sockaddr* ai_addr) { tcp_proxy_line[0] = 0; char tcp6_address_string[INET6_ADDRSTRLEN]; if (ai_addr->sa_family == AF_INET) { struct sockaddr_in* addr = (struct sockaddr_in*)ai_addr; size_t res = snprintf(tcp_proxy_line, sizeof(tcp_proxy_line), "PROXY %%s %%s %s %%hu %hu\r\n", inet_ntoa(addr->sin_addr), ntohs(addr->sin_port)); assert(res < sizeof(tcp_proxy_line)); } else if (ai_addr->sa_family == AF_INET6 ) { struct sockaddr_in6* addr = (struct sockaddr_in6*)ai_addr; inet_ntop(AF_INET6,&(addr->sin6_addr),tcp6_address_string,INET6_ADDRSTRLEN); size_t res = snprintf(tcp_proxy_line, sizeof(tcp_proxy_line), "PROXY %%s %%s %s %%hu %hu\r\n", tcp6_address_string, ntohs(addr->sin6_port)); assert(res < sizeof(tcp_proxy_line)); } else { ERR("The --write-proxy mode is not implemented for this address family.\n"); exit(1); } } /* Create the bound socket in the parent process */ static int create_main_socket() { struct addrinfo *ai, hints; memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; const int gai_err = getaddrinfo(OPTIONS.FRONT_IP, OPTIONS.FRONT_PORT, &hints, &ai); if (gai_err != 0) { ERR("{getaddrinfo}: [%s]\n", gai_strerror(gai_err)); exit(1); } int s = socket(ai->ai_family, SOCK_STREAM, IPPROTO_TCP); if (s == -1) fail("{socket: main}"); int t = 1; setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &t, sizeof(int)); #ifdef SO_REUSEPORT setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &t, sizeof(int)); #endif setnonblocking(s); if (bind(s, ai->ai_addr, ai->ai_addrlen)) { fail("{bind-socket}"); } #if TCP_DEFER_ACCEPT int timeout = 1; setsockopt(s, IPPROTO_TCP, TCP_DEFER_ACCEPT, &timeout, sizeof(int) ); #endif /* TCP_DEFER_ACCEPT */ prepare_proxy_line(ai->ai_addr); freeaddrinfo(ai); listen(s, OPTIONS.BACKLOG); return s; } /* Initiate a clear-text nonblocking connect() to the backend IP on behalf * of a newly connected upstream (encrypted) client*/ static int create_back_socket() { int s = socket(backaddr->ai_family, SOCK_STREAM, IPPROTO_TCP); if (s == -1) return -1; int flag = 1; int ret = setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag)); if (ret == -1) { perror("Couldn't setsockopt to backend (TCP_NODELAY)\n"); } int t = 1; setnonblocking(s); t = connect(s, backaddr->ai_addr, backaddr->ai_addrlen); if (t == 0 || errno == EINPROGRESS || errno == EINTR) return s; perror("{backend-connect}"); return -1; } /* Only enable a libev ev_io event if the proxied connection still * has both up and down connected */ static void safe_enable_io(proxystate *ps, ev_io *w) { if (!ps->want_shutdown) ev_io_start(loop, w); } /* Only enable a libev ev_io event if the proxied connection still * has both up and down connected */ static void shutdown_proxy(proxystate *ps, SHUTDOWN_REQUESTOR req) { if (ps->want_shutdown || req == SHUTDOWN_HARD) { ev_io_stop(loop, &ps->ev_w_up); ev_io_stop(loop, &ps->ev_r_up); ev_io_stop(loop, &ps->ev_w_handshake); ev_io_stop(loop, &ps->ev_r_handshake); ev_io_stop(loop, &ps->ev_w_down); ev_io_stop(loop, &ps->ev_r_down); close(ps->fd_up); close(ps->fd_down); SSL_set_shutdown(ps->ssl, SSL_SENT_SHUTDOWN); SSL_free(ps->ssl); free(ps); } else { ps->want_shutdown = 1; if (req == SHUTDOWN_DOWN && ringbuffer_is_empty(&ps->ring_up)) shutdown_proxy(ps, SHUTDOWN_HARD); else if (req == SHUTDOWN_UP && ringbuffer_is_empty(&ps->ring_down)) shutdown_proxy(ps, SHUTDOWN_HARD); } } /* Handle various socket errors */ static void handle_socket_errno(proxystate *ps) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) return; if (errno == ECONNRESET) ERR("{backend} Connection reset by peer\n"); else if (errno == ETIMEDOUT) ERR("{backend} Connection to backend timed out\n"); else if (errno == EPIPE) ERR("{backend} Broken pipe to backend (EPIPE)\n"); else perror("{backend} [errno]"); shutdown_proxy(ps, SHUTDOWN_DOWN); } /* Read some data from the backend when libev says data is available-- * write it into the upstream buffer and make sure the write event is * enabled for the upstream socket */ static void back_read(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; int t; proxystate *ps = (proxystate *)w->data; if (ps->want_shutdown) { ev_io_stop(loop, &ps->ev_r_down); return; } int fd = w->fd; char * buf = ringbuffer_write_ptr(&ps->ring_up); t = recv(fd, buf, RING_DATA_LEN, 0); if (t > 0) { ringbuffer_write_append(&ps->ring_up, t); if (ringbuffer_is_full(&ps->ring_up)) ev_io_stop(loop, &ps->ev_r_down); safe_enable_io(ps, &ps->ev_w_up); } else if (t == 0) { LOG("{backend} Connection closed\n"); shutdown_proxy(ps, SHUTDOWN_DOWN); } else { assert(t == -1); handle_socket_errno(ps); } } /* Write some data, previously received on the secure upstream socket, * out of the downstream buffer and onto the backend socket */ static void back_write(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; int t; proxystate *ps = (proxystate *)w->data; int fd = w->fd; int sz; assert(!ringbuffer_is_empty(&ps->ring_down)); char *next = ringbuffer_read_next(&ps->ring_down, &sz); t = send(fd, next, sz, MSG_NOSIGNAL); if (t > 0) { if (t == sz) { ringbuffer_read_pop(&ps->ring_down); safe_enable_io(ps, &ps->ev_r_up); if (ringbuffer_is_empty(&ps->ring_down)) { if (ps->want_shutdown) { shutdown_proxy(ps, SHUTDOWN_HARD); return; // dealloc'd } ev_io_stop(loop, &ps->ev_w_down); } } else { ringbuffer_read_skip(&ps->ring_down, t); } } else { assert(t == -1); handle_socket_errno(ps); } } static void start_handshake(proxystate *ps, int err); /* Continue/complete the asynchronous connect() before starting data transmission * between front/backend */ static void handle_connect(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; int t; proxystate *ps = (proxystate *)w->data; char tcp6_address_string[INET6_ADDRSTRLEN]; size_t written = 0; t = connect(ps->fd_down, backaddr->ai_addr, backaddr->ai_addrlen); if (!t || errno == EISCONN || !errno) { /* INIT */ ev_io_stop(loop, &ps->ev_w_down); ev_io_init(&ps->ev_r_down, back_read, ps->fd_down, EV_READ); ev_io_init(&ps->ev_w_down, back_write, ps->fd_down, EV_WRITE); start_handshake(ps, SSL_ERROR_WANT_READ); /* for client-first handshake */ ev_io_start(loop, &ps->ev_r_down); if (OPTIONS.WRITE_PROXY_LINE) { char *ring_pnt = ringbuffer_write_ptr(&ps->ring_down); assert(ps->remote_ip.ss_family == AF_INET || ps->remote_ip.ss_family == AF_INET6); if(ps->remote_ip.ss_family == AF_INET) { struct sockaddr_in* addr = (struct sockaddr_in*)&ps->remote_ip; written = snprintf(ring_pnt, RING_DATA_LEN, tcp_proxy_line, "TCP4", inet_ntoa(addr->sin_addr), ntohs(addr->sin_port)); } else if (ps->remote_ip.ss_family == AF_INET6) { struct sockaddr_in6* addr = (struct sockaddr_in6*)&ps->remote_ip; inet_ntop(AF_INET6,&(addr->sin6_addr),tcp6_address_string,INET6_ADDRSTRLEN); written = snprintf(ring_pnt, RING_DATA_LEN, tcp_proxy_line, "TCP6", tcp6_address_string, ntohs(addr->sin6_port)); } ringbuffer_write_append(&ps->ring_down, written); ev_io_start(loop, &ps->ev_w_down); } else if (OPTIONS.WRITE_IP_OCTET) { char *ring_pnt = ringbuffer_write_ptr(&ps->ring_down); assert(ps->remote_ip.ss_family == AF_INET || ps->remote_ip.ss_family == AF_INET6); *ring_pnt++ = (unsigned char) ps->remote_ip.ss_family; if (ps->remote_ip.ss_family == AF_INET6) { memcpy(ring_pnt, &((struct sockaddr_in6 *) &ps->remote_ip) ->sin6_addr.s6_addr, 16U); ringbuffer_write_append(&ps->ring_down, 1U + 16U); } else { memcpy(ring_pnt, &((struct sockaddr_in *) &ps->remote_ip) ->sin_addr.s_addr, 4U); ringbuffer_write_append(&ps->ring_down, 1U + 4U); } ev_io_start(loop, &ps->ev_w_down); } } else if (errno == EINPROGRESS || errno == EINTR || errno == EALREADY) { /* do nothing, we'll get phoned home again... */ } else { perror("{backend-connect}"); shutdown_proxy(ps, SHUTDOWN_HARD); } } /* Upon receiving a signal from OpenSSL that a handshake is required, re-wire * the read/write events to hook up to the handshake handlers */ static void start_handshake(proxystate *ps, int err) { ev_io_stop(loop, &ps->ev_r_up); ev_io_stop(loop, &ps->ev_w_up); if (err == SSL_ERROR_WANT_READ) ev_io_start(loop, &ps->ev_r_handshake); else if (err == SSL_ERROR_WANT_WRITE) ev_io_start(loop, &ps->ev_w_handshake); } /* After OpenSSL is done with a handshake, re-wire standard read/write handlers * for data transmission */ static void end_handshake(proxystate *ps) { ev_io_stop(loop, &ps->ev_r_handshake); ev_io_stop(loop, &ps->ev_w_handshake); /* Disable renegotiation (CVE-2009-3555) */ if (ps->ssl->s3) { ps->ssl->s3->flags |= SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS; } ps->handshaked = 1; /* if incoming buffer is not full */ if (!ringbuffer_is_full(&ps->ring_down)) safe_enable_io(ps, &ps->ev_r_up); /* if outgoing buffer is not empty */ if (!ringbuffer_is_empty(&ps->ring_up)) // not safe.. we want to resume stream even during half-closed ev_io_start(loop, &ps->ev_w_up); } /* The libev I/O handler during the OpenSSL handshake phase. Basically, just * let OpenSSL do what it likes with the socket and obey its requests for reads * or writes */ static void client_handshake(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; int t; proxystate *ps = (proxystate *)w->data; t = SSL_do_handshake(ps->ssl); if (t == 1) { end_handshake(ps); } else { int err = SSL_get_error(ps->ssl, t); if (err == SSL_ERROR_WANT_READ) { ev_io_stop(loop, &ps->ev_w_handshake); ev_io_start(loop, &ps->ev_r_handshake); } else if (err == SSL_ERROR_WANT_WRITE) { ev_io_stop(loop, &ps->ev_r_handshake); ev_io_start(loop, &ps->ev_w_handshake); } else if (err == SSL_ERROR_ZERO_RETURN) { LOG("{client} Connection closed (in handshake)\n"); shutdown_proxy(ps, SHUTDOWN_UP); } else { LOG("{client} Unexpected SSL error (in handshake): %d\n", err); shutdown_proxy(ps, SHUTDOWN_UP); } } } /* Handle a socket error condition passed to us from OpenSSL */ static void handle_fatal_ssl_error(proxystate *ps, int err) { if (err == SSL_ERROR_ZERO_RETURN) ERR("{client} Connection closed (in data)\n"); else if (err == SSL_ERROR_SYSCALL) if (errno == 0) ERR("{client} Connection closed (in data)\n"); else perror("{client} [errno] "); else ERR("{client} Unexpected SSL_read error: %d\n", err); shutdown_proxy(ps, SHUTDOWN_UP); } /* Read some data from the upstream secure socket via OpenSSL, * and buffer anything we get for writing to the backend */ static void client_read(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; int t; proxystate *ps = (proxystate *)w->data; if (ps->want_shutdown) { ev_io_stop(loop, &ps->ev_r_up); return; } char * buf = ringbuffer_write_ptr(&ps->ring_down); t = SSL_read(ps->ssl, buf, RING_DATA_LEN); /* Fix CVE-2009-3555. Disable reneg if started by client. */ if (ps->renegotiation) { shutdown_proxy(ps, SHUTDOWN_UP); return; } if (t > 0) { ringbuffer_write_append(&ps->ring_down, t); if (ringbuffer_is_full(&ps->ring_down)) ev_io_stop(loop, &ps->ev_r_up); safe_enable_io(ps, &ps->ev_w_down); } else { int err = SSL_get_error(ps->ssl, t); if (err == SSL_ERROR_WANT_WRITE) { start_handshake(ps, err); } else if (err == SSL_ERROR_WANT_READ) { } /* incomplete SSL data */ else handle_fatal_ssl_error(ps, err); } } /* Write some previously-buffered backend data upstream on the * secure socket using OpenSSL */ static void client_write(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; int t; int sz; proxystate *ps = (proxystate *)w->data; assert(!ringbuffer_is_empty(&ps->ring_up)); char * next = ringbuffer_read_next(&ps->ring_up, &sz); t = SSL_write(ps->ssl, next, sz); if (t > 0) { if (t == sz) { ringbuffer_read_pop(&ps->ring_up); safe_enable_io(ps, &ps->ev_r_down); // can be re-enabled b/c we've popped if (ringbuffer_is_empty(&ps->ring_up)) { if (ps->want_shutdown) { shutdown_proxy(ps, SHUTDOWN_HARD); return; } ev_io_stop(loop, &ps->ev_w_up); } } else { ringbuffer_read_skip(&ps->ring_up, t); } } else { int err = SSL_get_error(ps->ssl, t); if (err == SSL_ERROR_WANT_READ) { start_handshake(ps, err); } else if (err == SSL_ERROR_WANT_WRITE) {} /* incomplete SSL data */ else handle_fatal_ssl_error(ps, err); } } /* libev read handler for the bound socket. Socket is accepted, * the proxystate is allocated and initalized, and we're off the races * connecting to the backend */ static void handle_accept(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; struct sockaddr_storage addr; socklen_t sl = sizeof(addr); int client = accept(w->fd, (struct sockaddr *) &addr, &sl); if (client == -1) { switch (errno) { case EMFILE: ERR("{client} accept() failed; too many open files for this process\n"); break; case ENFILE: ERR("{client} accept() failed; too many open files for this system\n"); break; case 'k': OPTIONS.TCP_KEEPALIVE = atoi(optarg); break; default: assert(errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN); break; } return; } int flag = 1; int ret = setsockopt(client, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag) ); if (ret == -1) { perror("Couldn't setsockopt on client (TCP_NODELAY)\n"); } #ifdef TCP_CWND int cwnd = 10; ret = setsockopt(client, IPPROTO_TCP, TCP_CWND, &cwnd, sizeof(cwnd)); if (ret == -1) { perror("Couldn't setsockopt on client (TCP_CWND)\n"); } #endif setnonblocking(client); settcpkeepalive(client); int back = create_back_socket(); if (back == -1) { close(client); perror("{backend-connect}"); return; } SSL_CTX * ctx = (SSL_CTX *)w->data; SSL *ssl = SSL_new(ctx); long mode = SSL_MODE_ENABLE_PARTIAL_WRITE; #ifdef SSL_MODE_RELEASE_BUFFERS mode |= SSL_MODE_RELEASE_BUFFERS; #endif SSL_set_mode(ssl, mode); SSL_set_accept_state(ssl); SSL_set_fd(ssl, client); proxystate *ps = (proxystate *)malloc(sizeof(proxystate)); ps->fd_up = client; ps->fd_down = back; ps->ssl = ssl; ps->want_shutdown = 0; ps->handshaked = 0; ps->renegotiation = 0; ps->remote_ip = addr; ringbuffer_init(&ps->ring_up); ringbuffer_init(&ps->ring_down); /* set up events */ ev_io_init(&ps->ev_r_up, client_read, client, EV_READ); ev_io_init(&ps->ev_w_up, client_write, client, EV_WRITE); ev_io_init(&ps->ev_r_handshake, client_handshake, client, EV_READ); ev_io_init(&ps->ev_w_handshake, client_handshake, client, EV_WRITE); ev_io_init(&ps->ev_w_down, handle_connect, back, EV_WRITE); ev_io_init(&ps->ev_r_down, back_read, back, EV_READ); ev_io_start(loop, &ps->ev_w_down); ps->ev_r_up.data = ps; ps->ev_w_up.data = ps; ps->ev_r_down.data = ps; ps->ev_w_down.data = ps; ps->ev_r_handshake.data = ps; ps->ev_w_handshake.data = ps; /* Link back proxystate to SSL state */ SSL_set_app_data(ssl, ps); } static void check_ppid(struct ev_loop *loop, ev_timer *w, int revents) { (void) revents; pid_t ppid = getppid(); if (ppid != master_pid) { ERR("{core} Process %d detected parent death, closing listener socket.\n", child_num); ev_timer_stop(loop, w); ev_io_stop(loop, &listener); close(listener_socket); } } /* Set up the child (worker) process including libev event loop, read event * on the bound socket, etc */ static void handle_connections() { LOG("{core} Process %d online\n", child_num); #if defined(CPU_ZERO) && defined(CPU_SET) cpu_set_t cpus; CPU_ZERO(&cpus); CPU_SET(child_num, &cpus); int res = sched_setaffinity(0, sizeof(cpus), &cpus); if (!res) LOG("{core} Successfully attached to CPU #%d\n", child_num); else ERR("{core-warning} Unable to attach to CPU #%d; do you have that many cores?\n", child_num); #endif loop = ev_default_loop(EVFLAG_AUTO); ev_timer timer_ppid_check; ev_timer_init(&timer_ppid_check, check_ppid, 1.0, 1.0); ev_timer_start(loop, &timer_ppid_check); ev_io_init(&listener, handle_accept, listener_socket, EV_READ); listener.data = ssl_ctx; ev_io_start(loop, &listener); ev_loop(loop, 0); ERR("{core} Child %d exiting.\n", child_num); exit(1); } /* Print usage w/error message and exit failure */ static void usage_fail(const char *prog, const char *msg) { if (msg) fprintf(stderr, "%s: %s\n", prog, msg); fprintf(stderr, "usage: %s [OPTION] PEM\n", prog); fprintf(stderr, "Encryption Methods:\n" " --tls TLSv1 (default)\n" " --ssl SSLv3 (implies no TLSv1)\n" " -c CIPHER_SUITE set allowed ciphers (default is OpenSSL defaults)\n" " -e ENGINE set OpenSSL engine\n" "\n" "Socket:\n" " -b HOST,PORT backend [connect] (default is \"127.0.0.1,8000\")\n" " -f HOST,PORT frontend [bind] (default is \"*,8443\")\n" "\n" "Performance:\n" " -n CORES number of worker processes (default is 1)\n" " -B BACKLOG set listen backlog size (default is 100)\n" " -k SECS Change default tcp keepalive on client socket\n" #ifdef USE_SHARED_CACHE " -C SHARED_CACHE set shared cache size in sessions (default no shared cache)\n" #endif "\n" "Security:\n" " -r PATH chroot\n" " -u USERNAME set gid/uid after binding the socket\n" "\n" "Logging:\n" " -q be quiet; emit only error messages\n" " -s send log message to syslog in addition to stderr/stdout\n" "\n" "Special:\n" " --write-ip write 1 octet with the IP family followed by the IP\n" " address in 4 (IPv4) or 16 (IPv6) octets little-endian\n" " to backend before the actual data\n" " --write-proxy write HaProxy's PROXY (IPv4 or IPv6) protocol line\n" " before actual data\n" ); exit(1); } static void parse_host_and_port(const char *prog, const char *name, char *inp, int wildcard_okay, const char **ip, const char **port) { char buf[150]; char *sp; if (strlen(inp) >= sizeof buf) { snprintf(buf, sizeof buf, "invalid option for %s HOST,PORT\n", name); usage_fail(prog, buf); } sp = strchr(inp, ','); if (!sp) { snprintf(buf, sizeof buf, "invalid option for %s HOST,PORT\n", name); usage_fail(prog, buf); } if (!strncmp(inp, "*", sp - inp)) { if (!wildcard_okay) { snprintf(buf, sizeof buf, "wildcard host specification invalid for %s\n", name); usage_fail(prog, buf); } *ip = NULL; } else { *sp = 0; *ip = inp; } *port = sp + 1; } /* Handle command line arguments modifying behavior */ static void parse_cli(int argc, char **argv) { const char *prog = argv[0]; static int tls = 0, ssl = 0; int c; struct passwd* passwd; static struct option long_options[] = { {"tls", 0, &tls, 1}, {"ssl", 0, &ssl, 1}, {"write-ip", 0, &OPTIONS.WRITE_IP_OCTET, 1}, {"write-proxy", 0, &OPTIONS.WRITE_PROXY_LINE, 1}, {0, 0, 0, 0} }; while (1) { int option_index = 0; c = getopt_long(argc, argv, "hf:b:n:c:e:u:r:B:C:k:qs", long_options, &option_index); if (c == -1) break; switch (c) { case 0: break; case 'n': errno = 0; OPTIONS.NCORES = strtol(optarg, NULL, 10); if ((errno == ERANGE && (OPTIONS.NCORES == LONG_MAX || OPTIONS.NCORES == LONG_MIN)) || OPTIONS.NCORES < 1 || OPTIONS.NCORES > 128) { usage_fail(prog, "invalid option for -n CORES; please provide an integer between 1 and 128\n"); } break; case 'b': parse_host_and_port(prog, "-b", optarg, 0, &(OPTIONS.BACK_IP), &(OPTIONS.BACK_PORT)); break; case 'f': parse_host_and_port(prog, "-f", optarg, 1, &(OPTIONS.FRONT_IP), &(OPTIONS.FRONT_PORT)); break; case 'c': OPTIONS.CIPHER_SUITE = optarg; break; case 'e': OPTIONS.ENGINE = optarg; break; case 'u': passwd = getpwnam(optarg); if (!passwd) { if (errno) fail("getpwnam failed"); else ERR("user not found: %s\n", optarg); exit(1); } OPTIONS.UID = passwd->pw_uid; OPTIONS.GID = passwd->pw_gid; break; case 'r': if (optarg && optarg[0] == '/') OPTIONS.CHROOT = optarg; else { ERR("chroot must be absolute path: \"%s\"\n", optarg); exit(1); } break; case 'B': OPTIONS.BACKLOG = atoi(optarg); if ( OPTIONS.BACKLOG <= 0 ) { ERR("listen backlog can not be set to %d\n", OPTIONS.BACKLOG); exit(1); } break; #ifdef USE_SHARED_CACHE case 'C': OPTIONS.SHARED_CACHE = atoi(optarg); if ( OPTIONS.SHARED_CACHE < 0 ) { ERR("shared cache size can not be set to %d\n", OPTIONS.SHARED_CACHE); exit(1); } break; #endif case 'q': OPTIONS.QUIET = 1; break; case 's': OPTIONS.SYSLOG = 1; break; default: usage_fail(prog, NULL); } } /* Post-processing */ if (tls && ssl) usage_fail(prog, "Cannot specify both --tls and --ssl"); if (ssl) OPTIONS.ETYPE = ENC_SSL; // implied.. else, TLS if (OPTIONS.WRITE_IP_OCTET && OPTIONS.WRITE_PROXY_LINE) usage_fail(prog, "Cannot specify both --write-ip and --write-proxy; pick one!"); argc -= optind; argv += optind; if (argc != 1) usage_fail(prog, "exactly one argument is required: path to PEM file with cert/key"); OPTIONS.CERT_FILE = argv[0]; } void change_root() { if (chroot(OPTIONS.CHROOT) == -1) fail("chroot"); if (chdir("/")) fail("chdir"); } void drop_privileges() { if (setgid(OPTIONS.GID)) fail("setgid failed"); if (setuid(OPTIONS.UID)) fail("setuid failed"); } void init_globals() { /* backaddr */ struct addrinfo hints; memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = 0; const int gai_err = getaddrinfo(OPTIONS.BACK_IP, OPTIONS.BACK_PORT, &hints, &backaddr); if (gai_err != 0) { ERR("{getaddrinfo}: [%s]", gai_strerror(gai_err)); exit(1); } /* child_pids */ if ((child_pids = calloc(OPTIONS.NCORES, sizeof(pid_t))) == NULL) fail("calloc"); if (OPTIONS.SYSLOG) openlog("stud", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_DAEMON); } /* Forks COUNT children starting with START_INDEX. * Each child's index is stored in child_num and its pid is stored in child_pids[child_num] * so the parent can manage it later. */ void start_children(int start_index, int count) { for (child_num = start_index; child_num < start_index + count; child_num++) { int pid = fork(); if (pid == -1) { ERR("{core} fork() failed! Goodbye cruel world!\n"); exit(1); } else if (pid == 0) { /* child */ handle_connections(); exit(0); } else { /* parent. Track new child. */ child_pids[child_num] = pid; } } } /* Forks a new child to replace the old, dead, one with the given PID.*/ void replace_child_with_pid(pid_t pid) { int i; /* find old child's slot and put a new child there */ for (i = 0; i < OPTIONS.NCORES; i++) { if (child_pids[i] == pid) { start_children(i, 1); return; } } ERR("Cannot find index for child pid %d", pid); } /* Manage status changes in child processes */ static void do_wait(int __attribute__ ((unused)) signo) { int status; int pid = wait(&status); if (pid == -1) { if (errno == ECHILD) { ERR("{core} All children have exited! Restarting...\n"); start_children(0, OPTIONS.NCORES); } else if (errno == EINTR) { ERR("{core} Interrupted wait\n"); } else { fail("wait"); } } else { if (WIFEXITED(status)) { ERR("{core} Child %d exited with status %d. Replacing...\n", pid, WEXITSTATUS(status)); replace_child_with_pid(pid); } else if (WIFSIGNALED(status)) { ERR("{core} Child %d was terminated by signal %d. Replacing...\n", pid, WTERMSIG(status)); replace_child_with_pid(pid); } } } void init_signals() { struct sigaction act; sigemptyset(&act.sa_mask); act.sa_flags = 0; act.sa_handler = SIG_IGN; /* Avoid getting PIPE signal when writing to a closed file descriptor */ if (sigaction(SIGPIPE, &act, NULL) < 0) fail("sigaction - sigpipe"); /* We don't care if someone stops and starts a child process with kill (1) */ act.sa_flags = SA_NOCLDSTOP; act.sa_handler = do_wait; /* We do care when child processes change status */ if (sigaction(SIGCHLD, &act, NULL) < 0) fail("sigaction - sigchld"); } /* Process command line args, create the bound socket, * spawn child (worker) processes, and respawn if any die */ int main(int argc, char **argv) { parse_cli(argc, argv); init_signals(); init_globals(); listener_socket = create_main_socket(); /* load certificate, pass to handle_connections */ ssl_ctx = init_openssl(); master_pid = getpid(); if (OPTIONS.CHROOT && OPTIONS.CHROOT[0]) change_root(); if (OPTIONS.UID || OPTIONS.GID) drop_privileges(); start_children(0, OPTIONS.NCORES); for (;;) { /* Sleep and let the children work. * Parent will be woken up if a signal arrives */ pause(); } exit(0); /* just a formality; we never get here */ }