pax_global_header00006660000000000000000000000064122501754670014523gustar00rootroot0000000000000052 comment=1dac5c47455dcf0c5ace064035f6e42f56767143 swift-bench-1.0/000077500000000000000000000000001225017546700135745ustar00rootroot00000000000000swift-bench-1.0/.gitignore000066400000000000000000000002321225017546700155610ustar00rootroot00000000000000*.swp dist/ .tox *.egg *.egg-info *.py[co] .DS_Store *.log .testrepository subunit.log build .autogenerated .coverage cover/ coverage.xml doc/source/api/ swift-bench-1.0/.gitreview000066400000000000000000000001201225017546700155730ustar00rootroot00000000000000[gerrit] host=review.openstack.org port=29418 project=openstack/swift-bench.git swift-bench-1.0/.mailmap000066400000000000000000000030321225017546700152130ustar00rootroot00000000000000Greg Holt gholt Greg Holt gholt Greg Holt gholt Greg Holt gholt Greg Holt Greg Holt John Dickinson Michael Barton Michael Barton Michael Barton Mike Barton Clay Gerrard Clay Gerrard Clay Gerrard clayg David Goetz David Goetz Anne Gentle Anne Gentle annegentle Greg Lange Greg Lange Chmouel Boudjnah Joe Arnold Samuel Merritt Victor Rodionov Florian Hines Kun Huang Tom Fifield Tom Fifield swift-bench-1.0/.testr.conf000066400000000000000000000002351225017546700156620ustar00rootroot00000000000000[DEFAULT] test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list swift-bench-1.0/.unittests000077500000000000000000000001631225017546700156420ustar00rootroot00000000000000#!/bin/bash set -e python setup.py testr --coverage RET=$? coverage report -mswiftbench rm -f .coverage exit $RET swift-bench-1.0/AUTHORS000066400000000000000000000021051225017546700146420ustar00rootroot00000000000000Alex Gaynor Anne Gentle Chmouel Boudjnah Chuck Thier Clay Gerrard Dan Prince Darrell Bishop David Goetz Dirk Mueller Felipe Reyes Florian Hines François Charlier Greg Holt Greg Lange Joe Arnold John Dickinson Jon Snitow Juan J. Martinez Ksenia Demina Kun Huang Maru Newby Michael Barton Monty Taylor ning_zhang Peter Portante Samuel Merritt TheSriram Tom Fifield Victor Rodionov Zap Chang ZhiQiang Fan swift-bench-1.0/CHANGELOG000066400000000000000000000001141225017546700150020ustar00rootroot00000000000000swift-bench (1.0) * Initial release, after moving from OpenStack Swift swift-bench-1.0/LICENSE000066400000000000000000000261361225017546700146110ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. swift-bench-1.0/README.rst000066400000000000000000000000311225017546700152550ustar00rootroot00000000000000Swift Benchmarking tool. swift-bench-1.0/bin/000077500000000000000000000000001225017546700143445ustar00rootroot00000000000000swift-bench-1.0/bin/swift-bench000077500000000000000000000172411225017546700165100ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import sys import signal import uuid from optparse import OptionParser from swiftbench.bench import (BenchController, DistributedBenchController, create_containers, delete_containers) from swiftbench.utils import readconf, config_true_value # The defaults should be sufficient to run swift-bench on a SAIO CONF_DEFAULTS = { 'auth': os.environ.get('ST_AUTH', ''), 'user': os.environ.get('ST_USER', ''), 'key': os.environ.get('ST_KEY', ''), 'auth_version': '1.0', 'use_proxy': 'yes', 'put_concurrency': '10', 'get_concurrency': '10', 'del_concurrency': '10', 'concurrency': '', # set all 3 in one shot 'object_sources': '', # set of file contents to read and use for PUTs 'lower_object_size': '10', # bounded random size used if these differ 'upper_object_size': '10', 'object_size': '1', # only if not object_sources and lower == upper 'num_objects': '1000', 'num_gets': '10000', 'delete': 'yes', 'container_name': uuid.uuid4().hex, # really "container name base" 'num_containers': '20', 'url': '', # used when use_proxy = no or overrides auth X-Storage-Url 'account': '', # used when use_proxy = no 'devices': 'sdb1', # space-sep list 'log_level': 'INFO', 'timeout': '10', 'delay': '0', 'bench_clients': [], } SAIO_DEFAULTS = { 'auth': 'http://localhost:8080/auth/v1.0', 'user': 'test:tester', 'key': 'testing', } if __name__ == '__main__': usage = "usage: %prog [OPTIONS] [CONF_FILE]" usage += """\n\nConf file with SAIO defaults: [bench] auth = http://localhost:8080/auth/v1.0 user = test:tester key = testing concurrency = 10 object_size = 1 num_objects = 1000 num_gets = 10000 delete = yes auth_version = 1.0 """ parser = OptionParser(usage=usage) parser.add_option('', '--saio', dest='saio', action='store_true', default=False, help='Run benchmark with SAIO defaults') parser.add_option('-A', '--auth', dest='auth', help='URL for obtaining an auth token') parser.add_option('-U', '--user', dest='user', help='User name for obtaining an auth token') parser.add_option('-K', '--key', dest='key', help='Key for obtaining an auth token') parser.add_option('-b', '--bench-clients', action='append', metavar=':', help=('A string of the form ":" which matches ' 'the arguments supplied to a swift-bench-client ' 'process. This argument must be specified ' 'once per swift-bench-client you want to ' 'utilize.')) parser.add_option('-u', '--url', dest='url', help='Storage URL') parser.add_option('-c', '--concurrency', dest='concurrency', help=('Number of concurrent connections to use. For ' 'finer-grained control, see --get-concurrency, ' '--put-concurrency, and --delete-concurrency.')) parser.add_option('--get-concurrency', dest='get_concurrency', help='Number of concurrent GET requests') parser.add_option('--put-concurrency', dest='put_concurrency', help='Number of concurrent PUT requests') parser.add_option('--delete-concurrency', dest='delete_concurrency', help='Number of concurrent DELETE requests') parser.add_option('-s', '--object-size', dest='object_size', help='Size of objects to PUT (in bytes)') parser.add_option('-l', '--lower-object-size', dest='lower_object_size', help=('Lower size of objects (in bytes); ' '--object-size will be upper-object-size')) parser.add_option('-n', '--num-objects', dest='num_objects', help='Number of objects to PUT') parser.add_option('-g', '--num-gets', dest='num_gets', help='Number of GET operations to perform') parser.add_option('-C', '--num-containers', dest='num_containers', help='Number of containers to distribute objects among') parser.add_option('-x', '--no-delete', dest='delete', action='store_false', help='If set, will not delete the objects created') parser.add_option('-V', '--auth_version', dest='auth_version', help='Authentication version') parser.add_option('-d', '--delay', dest='delay', help='Delay before delete requests in seconds') if len(sys.argv) == 1: parser.print_help() sys.exit(1) options, args = parser.parse_args() if options.saio: CONF_DEFAULTS.update(SAIO_DEFAULTS) if getattr(options, 'lower_object_size', None): if options.object_size <= options.lower_object_size: raise ValueError('--lower-object-size (%s) must be ' '< --object-size (%s)' % (options.lower_object_size, options.object_size)) CONF_DEFAULTS['upper_object_size'] = options.object_size if args: conf = args[0] if not os.path.exists(conf): sys.exit("No such conf file: %s" % conf) conf = readconf(conf, 'bench', log_name='swift-bench', defaults=CONF_DEFAULTS) conf['bench_clients'] = [] else: conf = CONF_DEFAULTS parser.set_defaults(**conf) options, _junk = parser.parse_args() if options.concurrency is not '': options.put_concurrency = options.concurrency options.get_concurrency = options.concurrency options.del_concurrency = options.concurrency options.containers = ['%s_%d' % (options.container_name, i) for i in xrange(int(options.num_containers))] # Normalize boolean option to a config parameter value if config_true_value(str(options.delete).lower()): options.delete = 'yes' else: options.delete = 'no' def sigterm(signum, frame): sys.exit('Termination signal received.') signal.signal(signal.SIGTERM, sigterm) logger = logging.getLogger('swift-bench') logger.propagate = False logger.setLevel({ 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL}.get( options.log_level.lower(), logging.INFO)) loghandler = logging.StreamHandler() logger.addHandler(loghandler) logformat = logging.Formatter( 'swift-bench %(asctime)s %(levelname)s %(message)s') loghandler.setFormatter(logformat) if options.use_proxy: create_containers(logger, options) controller_class = DistributedBenchController if options.bench_clients \ else BenchController controller = controller_class(logger, options) controller.run() if options.use_proxy and options.delete == 'yes': delete_containers(logger, options) swift-bench-1.0/bin/swift-bench-client000077500000000000000000000036231225017546700177630ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import sys import signal from optparse import OptionParser from swiftbench.bench import BenchServer if __name__ == '__main__': usage = "usage: %prog " usage += "\n\nRun a client for distributed swift-bench runs." parser = OptionParser(usage=usage) parser.add_option('-o', '--log-level', dest='log_level', default='info', help='Logging level (debug, info, etc)') if len(sys.argv) != 3: parser.print_help() sys.exit(1) options, args = parser.parse_args() logger = logging.getLogger('swift-bench-client') logger.setLevel({ 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL}.get( options.log_level.lower(), logging.INFO)) loghandler = logging.StreamHandler() logger.addHandler(loghandler) logformat = logging.Formatter( 'swift-bench-client %(asctime)s %(levelname)s %(message)s') loghandler.setFormatter(logformat) def sigterm(signum, frame): sys.exit('Termination signal received.') signal.signal(signal.SIGTERM, sigterm) signal.signal(signal.SIGINT, sigterm) server = BenchServer(logger, args[0], args[1]) server.run() swift-bench-1.0/etc/000077500000000000000000000000001225017546700143475ustar00rootroot00000000000000swift-bench-1.0/etc/swift-bench.conf-sample000066400000000000000000000036571225017546700207210ustar00rootroot00000000000000[bench] # auth = http://localhost:8080/auth/v1.0 # user = test:tester # key = testing # auth_version = 1.0 # log-level = INFO # timeout = 10 # You can configure PUT, GET, and DELETE concurrency independently or set all # three with "concurrency" # put_concurrency = 10 # get_concurrency = 10 # del_concurrency = 10 # concurrency = # A space-sep list of files whose contents will be read and randomly chosen # as the body (object contents) for each PUT. # object_sources = # If object_sources is not set and lower_object_size != upper_object_size, # each PUT will randomly select an object size between the two values. Units # are bytes. # lower_object_size = 10 # upper_object_size = 10 # If object_sources is not set and lower_object_size == upper_object_size, # every object PUT will contain this many bytes. # object_size = 1 # num_objects = 1000 # num_gets = 10000 # num_containers = 20 # The base name for created containers. # container_name = (randomly-chosen uuid4) # Should swift-bench benchmark DELETEing the created objects and then delete # all created containers? # delete = yes # Without use_proxy, swift-bench will talk directly to the backend Swift # servers. Doing that will require "url", "account", and at least one # "devices" entry. # use_proxy = yes # If use_proxy = yes, this will override any returned X-Storage-Url returned # by authenticaion (the account name will still be extracted from # X-Storage-Url though and may NOT be set with the "account" conf var). If # use_proxy = no, this setting is required and used as the X-Storage-Url when # deleting containers and as a source for IP and port for back-end Swift server # connections. The IP and port specified in this setting must have local # storage access to every device specified in "devices". # url = # Only used (and required) when use_proxy = no. # account = # A space-sep list of devices names; only relevant (and required) when # use_proxy = no. # devices = sdb1 swift-bench-1.0/requirements.txt000066400000000000000000000000441225017546700170560ustar00rootroot00000000000000python-swiftclient eventlet>=0.9.15 swift-bench-1.0/setup.cfg000066400000000000000000000002611225017546700154140ustar00rootroot00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 [nosetests] exe = 1 verbosity = 2 detailed-errors = 1 cover-package = swiftbench cover-html = true cover-erase = true swift-bench-1.0/setup.py000066400000000000000000000032141225017546700153060ustar00rootroot00000000000000#!/usr/bin/python # Copyright (c) 2010-2012 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from setuptools import setup, find_packages from swiftbench import __version__ as version name = 'swift-bench' with open('requirements.txt', 'r') as f: requires = [x.strip() for x in f if x.strip()] setup( name=name, version=version, description='Benchmark tool for OpenStack Swift', license='Apache License (2.0)', author='OpenStack', author_email='openstack-dev@lists.openstack.org', url='http://openstack.org', packages=find_packages(exclude=['test', 'bin']), test_suite='nose.collector', classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Environment :: No Input/Output (Daemon)', 'Environment :: OpenStack', ], install_requires=requires, scripts=[ 'bin/swift-bench', 'bin/swift-bench-client', ], ) swift-bench-1.0/swiftbench/000077500000000000000000000000001225017546700157305ustar00rootroot00000000000000swift-bench-1.0/swiftbench/__init__.py000066400000000000000000000000241225017546700200350ustar00rootroot00000000000000__version__ = '1.0' swift-bench-1.0/swiftbench/bench.py000066400000000000000000000446441225017546700173750ustar00rootroot00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import re import sys import uuid import time import random import signal import socket import logging from contextlib import contextmanager from swift import gettext_ as _ from optparse import Values import eventlet import eventlet.pools from eventlet.green.httplib import CannotSendRequest import swiftclient as client from swiftbench.utils import config_true_value try: import simplejson as json except ImportError: import json try: from swift.common import direct_client except ImportError: direct_client = None HTTP_CONFLICT = 409 def _func_on_containers(logger, conf, concurrency_key, func): """Run a function on each container with concurrency.""" bench = Bench(logger, conf, []) pool = eventlet.GreenPool(int(getattr(conf, concurrency_key))) for container in conf.containers: pool.spawn_n(func, bench.url, bench.token, container) pool.waitall() def delete_containers(logger, conf): """Utility function to delete benchmark containers.""" def _deleter(url, token, container): try: client.delete_container(url, token, container) except client.ClientException as e: if e.http_status != HTTP_CONFLICT: logger.warn("Unable to delete container '%s'. " "Got http status '%d'." % (container, e.http_status)) _func_on_containers(logger, conf, 'del_concurrency', _deleter) def create_containers(logger, conf): """Utility function to create benchmark containers.""" _func_on_containers(logger, conf, 'put_concurrency', client.put_container) class SourceFile(object): """ Iterable, file-like object to lazily emit a bunch of zeros in reasonable-size chunks. swift.common.direct_client wants iterables, but swiftclient wants file-like objects where hasattr(thing, 'read') is true. Therefore, this class can do both. """ def __init__(self, size, chunk_size=1024 * 64): self.pos = 0 self.size = size self.chunk_size = chunk_size def __iter__(self): return self def __len__(self): return self.size def next(self): if self.pos >= self.size: raise StopIteration chunk_size = min(self.size - self.pos, self.chunk_size) yield '0' * chunk_size self.pos += chunk_size def read(self, desired_size): chunk_size = min(self.size - self.pos, desired_size) self.pos += chunk_size return '0' * chunk_size class ConnectionPool(eventlet.pools.Pool): def __init__(self, url, size): self.url = url eventlet.pools.Pool.__init__(self, size, size) def create(self): return client.http_connection(self.url) class BenchServer(object): """ A BenchServer binds to an IP/port and listens for bench jobs. A bench job consists of the normal conf "dict" encoded in JSON, terminated with an EOF. The log level is at least INFO, but DEBUG may also be specified in the conf dict. The server will wait forever for jobs, running them one at a time. """ def __init__(self, logger, bind_ip, bind_port): self.logger = logger self.bind_ip = bind_ip self.bind_port = int(bind_port) def run(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.logger.info('Binding to %s:%s', self.bind_ip, self.bind_port) s.bind((self.bind_ip, self.bind_port)) s.listen(20) while True: client, address = s.accept() self.logger.debug('Accepting connection from %s:%s', *address) client_file = client.makefile('rb+', 1) json_data = client_file.read() conf = Values(json.loads(json_data)) self.logger.info( 'Starting run for %s:%s [put/get/del_concurrency: %s/%s/%s, ' 'num_objects: %s, num_gets: %s]', address[0], address[1], conf.put_concurrency, conf.get_concurrency, conf.del_concurrency, conf.num_objects, conf.num_gets) logger = logging.getLogger('bench-server') level = logging.DEBUG if conf.log_level.lower() == 'debug' \ else logging.INFO logger.setLevel(level) loghandler = logging.StreamHandler(client_file) logformat = logging.Formatter( 'bench-server %(asctime)s %(levelname)s %(message)s') loghandler.setFormatter(logformat) logger.addHandler(loghandler) controller = BenchController(logger, conf) try: controller.run() except socket.error: logger.warning('Socket error', exc_info=1) logger.removeHandler(loghandler) client_file.close() client.close() self.logger.info('...bench run completed; waiting for next run.') class Bench(object): def __init__(self, logger, conf, names): self.logger = logger self.aborted = False self.user = conf.user self.key = conf.key self.auth_url = conf.auth self.use_proxy = config_true_value(conf.use_proxy) if not self.use_proxy and direct_client is None: self.logger.critical("You need to have swift installed if you are " "not using the proxy") sys.exit(1) self.auth_version = conf.auth_version self.logger.info("Auth version: %s" % self.auth_version) if self.use_proxy: url, token = client.get_auth(self.auth_url, self.user, self.key, auth_version=self.auth_version) self.token = token self.account = url.split('/')[-1] if conf.url == '': self.url = url else: self.url = conf.url else: self.token = 'SlapChop!' self.account = conf.account self.url = conf.url self.ip, self.port = self.url.split('/')[2].split(':') self.object_size = int(conf.object_size) self.object_sources = conf.object_sources self.lower_object_size = int(conf.lower_object_size) self.upper_object_size = int(conf.upper_object_size) self.files = [] if self.object_sources: self.object_sources = self.object_sources.split() self.files = [file(f, 'rb').read() for f in self.object_sources] self.put_concurrency = int(conf.put_concurrency) self.get_concurrency = int(conf.get_concurrency) self.del_concurrency = int(conf.del_concurrency) self.total_objects = int(conf.num_objects) self.total_gets = int(conf.num_gets) self.timeout = int(conf.timeout) self.devices = conf.devices.split() self.names = names self.conn_pool = ConnectionPool(self.url, max(self.put_concurrency, self.get_concurrency, self.del_concurrency)) def _log_status(self, title): total = time.time() - self.beginbeat self.logger.info(_('%(complete)s %(title)s [%(fail)s failures], ' '%(rate).01f/s'), {'title': title, 'complete': self.complete, 'fail': self.failures, 'rate': (float(self.complete) / total)}) @contextmanager def connection(self): try: hc = self.conn_pool.get() try: yield hc except CannotSendRequest: self.logger.info(_("CannotSendRequest. Skipping...")) try: hc.close() except Exception: pass self.failures += 1 hc = self.conn_pool.create() finally: self.conn_pool.put(hc) def run(self): pool = eventlet.GreenPool(self.concurrency) self.beginbeat = self.heartbeat = time.time() self.heartbeat -= 13 # just to get the first report quicker self.failures = 0 self.complete = 0 for i in xrange(self.total): if self.aborted: break pool.spawn_n(self._run, i) pool.waitall() self._log_status(self.msg + ' **FINAL**') def _run(self, thread): return class DistributedBenchController(object): """ This class manages a distributed swift-bench run. For this Controller class to make sense, the conf.bench_clients list must contain at least one entry. The idea is to split the configured load between one or more swift-bench-client processes, each of which use eventlet for concurrency. We deliberately take a simple, naive approach with these limitations: 1) Concurrency, num_objects, and num_gets are spread evenly between the swift-bench-client processes. With a low concurrency to swift-bench-client count ratio, rounding may result in a greater than desired aggregate concurrency. 2) Each swift-bench-client process runs independently so some may finish up before others, i.e. the target aggregate concurrency is not necessarily present the whole time. This may bias aggregate reported rates lower than a more efficient architecture. 3) Because of #2, some swift-bench-client processes may be running GETs while others are still runinng their PUTs. Because of this potential skew, distributed runs will not isolate one operation at a time like a single swift-bench run will. 3) Reported aggregate rates are simply the sum of each swift-bench-client process reported FINAL number. That's probably inaccurate somehow. """ def __init__(self, logger, conf): self.logger = logger # ... INFO 1000 PUTS **FINAL** [0 failures], 34.9/s self.final_re = re.compile( 'INFO (\d+) (.*) \*\*FINAL\*\* \[(\d+) failures\], (\d+\.\d+)/s') self.clients = conf.bench_clients del conf.bench_clients for key, minval in [('put_concurrency', 1), ('get_concurrency', 1), ('del_concurrency', 1), ('num_objects', 0), ('num_gets', 0)]: setattr(conf, key, max(minval, int(getattr(conf, key)) / len(self.clients))) self.conf = conf def run(self): eventlet.patcher.monkey_patch(socket=True) pool = eventlet.GreenPool(size=len(self.clients)) pile = eventlet.GreenPile(pool) for client in self.clients: pile.spawn(self.do_run, client) results = { 'PUTS': dict(count=0, failures=0, rate=0.0), 'GETS': dict(count=0, failures=0, rate=0.0), 'DEL': dict(count=0, failures=0, rate=0.0), } for result in pile: for k, v in result.iteritems(): target = results[k] target['count'] += int(v['count']) target['failures'] += int(v['failures']) target['rate'] += float(v['rate']) for k in ['PUTS', 'GETS', 'DEL']: v = results[k] self.logger.info('%d %s **FINAL** [%d failures], %.1f/s' % ( v['count'], k, v['failures'], v['rate'])) def do_run(self, client): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ip, port = client.split(':') s.connect((ip, int(port))) s.sendall(json.dumps(self.conf.__dict__)) s.shutdown(socket.SHUT_WR) s_file = s.makefile('rb', 1) result = {} for line in s_file: match = self.final_re.search(line) if match: g = match.groups() result[g[1]] = { 'count': g[0], 'failures': g[2], 'rate': g[3], } else: sys.stderr.write('%s %s' % (client, line)) return result class BenchController(object): def __init__(self, logger, conf): self.logger = logger self.conf = conf self.names = [] self.delete = config_true_value(conf.delete) self.gets = int(conf.num_gets) self.aborted = False self.delay = int(self.conf.delay) def sigint1(self, signum, frame): if self.delete: print >>sys.stderr, ( 'SIGINT received; finishing up and running DELETE.\n' 'Send one more SIGINT to exit *immediately*.') self.aborted = True if self.running and not isinstance(self.running, BenchDELETE): self.running.aborted = True signal.signal(signal.SIGINT, self.sigint2) else: self.sigint2(signum, frame) def sigint2(self, signum, frame): sys.exit('Final SIGINT received.') def run(self): eventlet.patcher.monkey_patch(socket=True) signal.signal(signal.SIGINT, self.sigint1) puts = BenchPUT(self.logger, self.conf, self.names) self.running = puts puts.run() if self.gets and not self.aborted: gets = BenchGET(self.logger, self.conf, self.names) self.running = gets gets.run() if self.delete: if self.delay != 0: self.logger.info('Delay before ' 'DELETE request %s sec' % self.delay) time.sleep(self.delay) dels = BenchDELETE(self.logger, self.conf, self.names) self.running = dels dels.run() class BenchDELETE(Bench): def __init__(self, logger, conf, names): Bench.__init__(self, logger, conf, names) self.concurrency = self.del_concurrency self.total = len(names) self.msg = 'DEL' def _run(self, thread): if time.time() - self.heartbeat >= 15: self.heartbeat = time.time() self._log_status('DEL') device, partition, name, container_name = self.names.pop() with self.connection() as conn: try: if self.use_proxy: client.delete_object(self.url, self.token, container_name, name, http_conn=conn) else: node = {'ip': self.ip, 'port': self.port, 'device': device} direct_client.direct_delete_object(node, partition, self.account, container_name, name) except client.ClientException as e: self.logger.debug(str(e)) self.failures += 1 self.complete += 1 class BenchGET(Bench): def __init__(self, logger, conf, names): Bench.__init__(self, logger, conf, names) self.concurrency = self.get_concurrency self.total = self.total_gets self.msg = 'GETS' def _run(self, thread): if time.time() - self.heartbeat >= 15: self.heartbeat = time.time() self._log_status('GETS') device, partition, name, container_name = random.choice(self.names) with self.connection() as conn: try: if self.use_proxy: client.get_object(self.url, self.token, container_name, name, http_conn=conn) else: node = {'ip': self.ip, 'port': self.port, 'device': device} direct_client.direct_get_object(node, partition, self.account, container_name, name) except client.ClientException as e: self.logger.debug(str(e)) self.failures += 1 self.complete += 1 class BenchPUT(Bench): def __init__(self, logger, conf, names): Bench.__init__(self, logger, conf, names) self.concurrency = self.put_concurrency self.total = self.total_objects self.msg = 'PUTS' self.containers = conf.containers def _run(self, thread): if time.time() - self.heartbeat >= 15: self.heartbeat = time.time() self._log_status('PUTS') name = uuid.uuid4().hex if self.object_sources: source = random.choice(self.files) elif self.upper_object_size > self.lower_object_size: source = SourceFile(random.randint(self.lower_object_size, self.upper_object_size)) else: source = SourceFile(self.object_size) device = random.choice(self.devices) partition = str(random.randint(1, 3000)) container_name = random.choice(self.containers) with self.connection() as conn: try: if self.use_proxy: client.put_object(self.url, self.token, container_name, name, source, content_length=len(source), http_conn=conn) else: node = {'ip': self.ip, 'port': self.port, 'device': device} direct_client.direct_put_object(node, partition, self.account, container_name, name, source, content_length=len(source)) except client.ClientException as e: self.logger.debug(str(e)) self.failures += 1 else: self.names.append((device, partition, name, container_name)) self.complete += 1 swift-bench-1.0/swiftbench/utils.py000066400000000000000000000052411225017546700174440ustar00rootroot00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import sys from ConfigParser import ConfigParser, RawConfigParser # Used when reading config values TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) # NOTE(chmouel): Imported from swift without the modular directory feature. def readconf(conf_path, section_name=None, log_name=None, defaults=None, raw=False): """ Read config file(s) and return config items as a dict :param conf_path: path to config file, or a file-like object (hasattr readline) :param section_name: config section to read (will return all sections if not defined) :param log_name: name to be used with logging (will use section_name if not defined) :param defaults: dict of default values to pre-populate the config with :returns: dict of config items """ if defaults is None: defaults = {} if raw: c = RawConfigParser(defaults) else: c = ConfigParser(defaults) if hasattr(conf_path, 'readline'): c.readfp(conf_path) else: success = c.read(conf_path) if not success: print "Unable to read config from %s" % conf_path sys.exit(1) if section_name: if c.has_section(section_name): conf = dict(c.items(section_name)) else: print "Unable to find %s config section in %s" % \ (section_name, conf_path) sys.exit(1) if "log_name" not in conf: if log_name is not None: conf['log_name'] = log_name else: conf['log_name'] = section_name else: conf = {} for s in c.sections(): conf.update({s: dict(c.items(s))}) if 'log_name' not in conf: conf['log_name'] = log_name conf['__file__'] = conf_path return conf def config_true_value(value): """ Returns True if the value is either True or a string in TRUE_VALUES. Returns False otherwise. """ return value is True or \ (isinstance(value, basestring) and value.lower() in TRUE_VALUES) swift-bench-1.0/test-requirements.txt000066400000000000000000000001551225017546700200360ustar00rootroot00000000000000hacking>=0.5.6,<0.8 coverage>=3.6 discover mock>=1.0 sphinx>=1.1.2 testrepository>=0.0.17 testtools>=0.9.32 swift-bench-1.0/tests/000077500000000000000000000000001225017546700147365ustar00rootroot00000000000000swift-bench-1.0/tests/__init__.py000066400000000000000000000000001225017546700170350ustar00rootroot00000000000000swift-bench-1.0/tests/test_bench.py000066400000000000000000000013751225017546700174340ustar00rootroot00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # TODO(gholt): Tests import unittest class TestBench(unittest.TestCase): def test_placeholder(self): pass if __name__ == '__main__': unittest.main() swift-bench-1.0/tests/test_utils.py000066400000000000000000000100221225017546700175020ustar00rootroot00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import mock import os import tempfile import unittest from StringIO import StringIO from swiftbench import utils class TestUtils(unittest.TestCase): @mock.patch.object(utils, "TRUE_VALUES") def test_config_true_value(self, mocked): utils.TRUE_VALUES = 'hello world'.split() for val in 'hello world HELLO WORLD'.split(): self.assertTrue(utils.config_true_value(val) is True) self.assertTrue(utils.config_true_value(True) is True) self.assertTrue(utils.config_true_value('foo') is False) self.assertTrue(utils.config_true_value(False) is False) def test_readconf(self): conf = '''[section1] foo = bar [section2] log_name = yarr''' # setup a real file fd, temppath = tempfile.mkstemp(dir='/tmp') with os.fdopen(fd, 'wb') as f: f.write(conf) make_filename = lambda: temppath # setup a file stream make_fp = lambda: StringIO(conf) for conf_object_maker in (make_filename, make_fp): conffile = conf_object_maker() result = utils.readconf(conffile) expected = {'__file__': conffile, 'log_name': None, 'section1': {'foo': 'bar'}, 'section2': {'log_name': 'yarr'}} self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section1') expected = {'__file__': conffile, 'log_name': 'section1', 'foo': 'bar'} self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section2').get('log_name') expected = 'yarr' self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section1', log_name='foo').get('log_name') expected = 'foo' self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section1', defaults={'bar': 'baz'}) expected = {'__file__': conffile, 'log_name': 'section1', 'foo': 'bar', 'bar': 'baz'} self.assertEquals(result, expected) self.assertRaises(SystemExit, utils.readconf, temppath, 'section3') os.unlink(temppath) self.assertRaises(SystemExit, utils.readconf, temppath) def test_readconf_raw(self): conf = '''[section1] foo = bar [section2] log_name = %(yarr)s''' # setup a real file fd, temppath = tempfile.mkstemp(dir='/tmp') with os.fdopen(fd, 'wb') as f: f.write(conf) make_filename = lambda: temppath # setup a file stream make_fp = lambda: StringIO(conf) for conf_object_maker in (make_filename, make_fp): conffile = conf_object_maker() result = utils.readconf(conffile, raw=True) expected = {'__file__': conffile, 'log_name': None, 'section1': {'foo': 'bar'}, 'section2': {'log_name': '%(yarr)s'}} self.assertEquals(result, expected) os.unlink(temppath) self.assertRaises(SystemExit, utils.readconf, temppath) if __name__ == '__main__': unittest.main() swift-bench-1.0/tox.ini000066400000000000000000000011451225017546700151100ustar00rootroot00000000000000[tox] envlist = py26,py27,py33,pypy,pep8 [testenv] setenv = VIRTUAL_ENV={envdir} LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=C deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = python setup.py testr --testr-args="{posargs}" [testenv:pep8] commands = flake8 flake8 bin/swift-bench flake8 bin/swift-bench-client [testenv:venv] commands = {posargs} [testenv:cover] commands = python setup.py testr --coverage [tox:jenkins] downloadcache = ~/cache/pip [flake8] ignore = H show-source = True exclude = .venv,.tox,dist,doc,test,*egg