pax_global_header00006660000000000000000000000064146433053330014516gustar00rootroot0000000000000052 comment=fdbc043f8688b2173fe867a12c85e6e5abf034b2 puppetlabs-jruby-utils-ca5d27b/000077500000000000000000000000001464330533300166555ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/.github/000077500000000000000000000000001464330533300202155ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/.github/workflows/000077500000000000000000000000001464330533300222525ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/.github/workflows/lein-test.yaml000066400000000000000000000023241464330533300250430ustar00rootroot00000000000000name: lein_test on: workflow_dispatch: push: branches: - main paths: ['src/**','test/**'] pull_request: types: [opened, reopened, edited, synchronize] paths: ['src/**','test/**'] jobs: run-lein-tests: name: lein test - Java ${{ matrix.java }} runs-on: ubuntu-latest strategy: matrix: java: [ '11', '17' ] steps: - name: Check out repository code uses: actions/checkout@v4 - name: Setup java uses: actions/setup-java@v4 with: distribution: temurin java-version: ${{ matrix.java }} - name: Install Clojure tools uses: DeLaGuardo/setup-clojure@12.5 with: cli: latest # Clojure CLI based on tools.deps lein: latest # Leiningen boot: latest # Boot.clj bb: latest # Babashka clj-kondo: latest # Clj-kondo cljstyle: latest # cljstyle zprint: latest # zprint - name: Run lein tests with update profile run: | set -x set -e lein version echo "Running tests" lein -U test puppetlabs-jruby-utils-ca5d27b/.github/workflows/mend.yml000066400000000000000000000036031464330533300237220ustar00rootroot00000000000000name: mend_scan on: workflow_dispatch: push: branches: - main jobs: build: runs-on: ubuntu-latest steps: - name: connect_twingate uses: twingate/github-action@v1 with: service-key: ${{ secrets.TWINGATE_PUBLIC_REPO_KEY }} - name: checkout repo content uses: actions/checkout@v2 # checkout the repository content to github runner. with: fetch-depth: 1 # install java which is required for mend and clojure - name: setup java uses: actions/setup-java@v3 with: distribution: temurin java-version: 17 # install clojure tools - name: Install Clojure tools uses: DeLaGuardo/setup-clojure@10.1 with: # Install just one or all simultaneously # The value must indicate a particular version of the tool, or use 'latest' # to always provision the latest version cli: latest # Clojure CLI based on tools.deps lein: latest # Leiningen boot: latest # Boot.clj bb: latest # Babashka clj-kondo: latest # Clj-kondo cljstyle: latest # cljstyle zprint: latest # zprint # run lein gen - name: create pom.xml run: lein pom # download mend - name: download_mend run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar - name: run mend run: env WS_INCLUDES=pom.xml java -jar wss-unified-agent.jar env: WS_APIKEY: ${{ secrets.MEND_API_KEY }} WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent WS_USERKEY: ${{ secrets.MEND_TOKEN }} WS_PRODUCTNAME: Puppet Enterprise WS_PROJECTNAME: ${{ github.event.repository.name }} WS_FILESYSTEMSCAN: true WS_CHECKPOLICIES: true WS_FORCEUPDATE: true puppetlabs-jruby-utils-ca5d27b/.gitignore000066400000000000000000000006551464330533300206530ustar00rootroot00000000000000scratch pom.xml *jar /lib/ /classes/ /targets/ .lein-deps-sum .lein-failures target/ log .bundle .vagrant vendor .nrepl-port profiles.clj dev/user.clj .lein-repl-history ext/packaging checkouts # Bundler local state files Gemfile.lock Gemfile.local /.bundle/ # Local test runs drop files into junit and tmp /junit/ /tmp/ acceptance/scripts/hosts.cfg /dev-resources/i18n/bin /resources/locales.clj /resources/**/Messages*.class puppetlabs-jruby-utils-ca5d27b/.gitmodules000066400000000000000000000000001464330533300210200ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/CODEOWNERS000066400000000000000000000002671464330533300202550ustar00rootroot00000000000000# This will cause the puppetserver-maintainers group to be assigned # review of any opened PRs against the branches containing this file. * @puppetlabs/dumpling @puppetlabs/skeletor puppetlabs-jruby-utils-ca5d27b/CONTRIBUTING.md000066400000000000000000000035611464330533300211130ustar00rootroot00000000000000# How to contribute * Make sure you have a [GitHub account](https://github.com/signup/free) * Fork the repository on GitHub ## Making Changes * Create a topic branch from where you want to base your work (this is almost definitely the master branch). * To quickly create a topic branch based on master; `git branch fix/master/my_contribution master` then checkout the new branch with `git checkout fix/master/my_contribution`. * Please avoid working directly on the `master` branch. * Make commits of logical units. * Check for unnecessary whitespace with `git diff --check` before committing. * Make sure your commit messages are in the proper format. ```` Make the example in CONTRIBUTING imperative and concrete Without this patch applied the example commit message in the CONTRIBUTING document is not a concrete example. This is a problem because the contributor is left to imagine what the commit message should look like based on a description rather than an example. This patch fixes the problem by making the example concrete and imperative. The first line is a real life imperative statement. The body describes the behavior without the patch, why this is a problem, and how the patch fixes the problem when applied. ```` * Make sure you have added the necessary tests for your changes. * Run _all_ the tests to assure nothing else was accidentally broken. ## Submitting Changes * Sign the [Contributor License Agreement](http://links.puppetlabs.com/cla). * Push your changes to a topic branch in your fork of the repository. * Submit a pull request to the repository in the puppetlabs organization. # Additional Resources * [Contributor License Agreement](http://links.puppetlabs.com/cla) * [General GitHub documentation](http://help.github.com/) * [GitHub pull request documentation](http://help.github.com/send-pull-requests/) puppetlabs-jruby-utils-ca5d27b/LICENSE000066400000000000000000000013241464330533300176620ustar00rootroot00000000000000 JRuby Utils - A library to help interacting with JRuby from Clojure Copyright (C) 2005-2016 Puppet Labs Inc Puppet Labs can be contacted at: info@puppetlabs.com Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. puppetlabs-jruby-utils-ca5d27b/Makefile000066400000000000000000000000441464330533300203130ustar00rootroot00000000000000include dev-resources/Makefile.i18n puppetlabs-jruby-utils-ca5d27b/README.md000066400000000000000000000037761464330533300201510ustar00rootroot00000000000000# JRuby Utils A library for creating and interacting with a pool of JRuby instances in Clojure. [![Build Status](https://travis-ci.org/puppetlabs/jruby-utils.svg)](https://travis-ci.org/puppetlabs/jruby-utils) ## Usage This is a brief overview of the functionality of this library; TODO add more docs. This library provides three public namespaces: `puppetlabs.services.jruby-pool-manager.jruby-schemas`, `puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service`, and `puppetlabs.services.jruby-pool-manager.jruby-core`. The entry-point into most of the functionality this library provides is the `jruby-pool-manager-service` Trapperkeeper service's `create-pool`, which creates and returns a pool context and asynchronously starts filling the pool. `create-pool` takes a config matching the `JRubyConfig` schema; this config can be created and provided defaults by the `jruby-core/initialize-config` function. The non-optional config settings that you must provide are `ruby-load-path` and `gem-home`. You can provide custom initialization and callback logic by providing lifecycle functions for initializing the scripting container, initializing the pool instance, cleaning up a pool instance, and cleaning up the pool for shutdown. After the pool has been created, use the `borrow` and `return` functions to work with instances. There is a `with-jruby-instance` macro to make this easier. There is also a `with-lock` macro that holds a lock on the pool (so that no borrows can take place) while you execute some logic. In most TK apps where you want to work with JRuby instances, you will want to call `create-pool` in the `init` lifecycle of your service, and then call `jruby-core/flush-pool-for-shutdown!` in the `stop` lifecycle function. ## Running tests To run the clojure unit tests, use: ~~~sh lein test ~~~ ## License See [LICENSE](LICENSE). ## Support We use the [Trapperkeeper project on JIRA](https://tickets.puppetlabs.com/browse/TK) for tickets on this project, although Github issues are welcome too. puppetlabs-jruby-utils-ca5d27b/dev-resources/000077500000000000000000000000001464330533300214435ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/dev-resources/Makefile.i18n000066400000000000000000000141011464330533300236560ustar00rootroot00000000000000# -*- Makefile -*- # This file was generated by the i18n leiningen plugin # Do not edit this file; it will be overwritten the next time you run # lein i18n init # # The name of the package into which the translations bundle will be placed BUNDLE=puppetlabs.jruby_utils # The name of the POT file into which the gettext code strings (msgid) will be placed POT_NAME=jruby-utils.pot # The list of names of packages covered by the translation bundle; # by default it contains a single package - the same where the translations # bundle itself is placed - but this can be overridden - preferably in # the top level Makefile PACKAGES?=$(BUNDLE) LOCALES=$(basename $(notdir $(wildcard locales/*.po))) BUNDLE_DIR=$(subst .,/,$(BUNDLE)) BUNDLE_FILES=$(patsubst %,resources/$(BUNDLE_DIR)/Messages_%.class,$(LOCALES)) FIND_SOURCES=find src -name \*.clj # xgettext before 0.19 does not understand --add-location=file. Even CentOS # 7 ships with an older gettext. We will therefore generate full location # info on those systems, and only file names where xgettext supports it LOC_OPT=$(shell xgettext --add-location=file -f - /dev/null 2>&1 && echo --add-location=file || echo --add-location) LOCALES_CLJ=resources/locales.clj define LOCALES_CLJ_CONTENTS { :locales #{$(patsubst %,"%",$(LOCALES))} :packages [$(patsubst %,"%",$(PACKAGES))] :bundle $(patsubst %,"%",$(BUNDLE).Messages) } endef export LOCALES_CLJ_CONTENTS i18n: msgfmt # Update locales/.pot update-pot: locales/$(POT_NAME) locales/$(POT_NAME): $(shell $(FIND_SOURCES)) | locales @tmp=$$(mktemp $@.tmp.XXXX); \ $(FIND_SOURCES) \ | xgettext --from-code=UTF-8 --language=lisp \ --copyright-holder='Puppet ' \ --package-name="$(BUNDLE)" \ --package-version="$(BUNDLE_VERSION)" \ --msgid-bugs-address="docs@puppet.com" \ -k \ -kmark:1 -ki18n/mark:1 \ -ktrs:1 -ki18n/trs:1 \ -ktru:1 -ki18n/tru:1 \ -ktrun:1,2 -ki18n/trun:1,2 \ -ktrsn:1,2 -ki18n/trsn:1,2 \ $(LOC_OPT) \ --add-comments --sort-by-file \ -o $$tmp -f -; \ sed -i.bak -e 's/charset=CHARSET/charset=UTF-8/' $$tmp; \ sed -i.bak -e 's/POT-Creation-Date: [^\\]*/POT-Creation-Date: /' $$tmp; \ rm -f $$tmp.bak; \ if ! diff -q -I POT-Creation-Date $$tmp $@ >/dev/null 2>&1; then \ mv $$tmp $@; \ else \ rm $$tmp; touch $@; \ fi # Run msgfmt over all .po files to generate Java resource bundles # and create the locales.clj file msgfmt: $(BUNDLE_FILES) $(LOCALES_CLJ) clean-orphaned-bundles # Force rebuild of locales.clj if its contents is not the the desired one. The # shell echo is used to add a trailing newline to match the one from `cat` ifneq ($(shell cat $(LOCALES_CLJ) 2> /dev/null),$(shell echo '$(LOCALES_CLJ_CONTENTS)')) .PHONY: $(LOCALES_CLJ) endif $(LOCALES_CLJ): | resources @echo "Writing $@" @echo "$$LOCALES_CLJ_CONTENTS" > $@ # Remove every resource bundle that wasn't generated from a PO file. # We do this because we used to generate the english bundle directly from the POT. .PHONY: clean-orphaned-bundles clean-orphaned-bundles: @for bundle in resources/$(BUNDLE_DIR)/Messages_*.class; do \ locale=$$(basename "$$bundle" | sed -E -e 's/\$$?1?\.class$$/_class/' | cut -d '_' -f 2;); \ if [ ! -f "locales/$$locale.po" ]; then \ rm "$$bundle"; \ fi \ done resources/$(BUNDLE_DIR)/Messages_%.class: locales/%.po | resources msgfmt --java2 -d resources -r $(BUNDLE).Messages -l $(*F) $< # Use this to initialize translations. Updating the PO files is done # automatically through a CI job that utilizes the scripts in the project's # `bin` file, which themselves come from the `clj-i18n` project. locales/%.po: | locales @if [ ! -f $@ ]; then \ touch $@ && msginit --no-translator -l $(*F) -o $@ -i locales/$(POT_NAME); \ fi resources locales: @mkdir $@ help: $(info $(HELP)) @echo .PHONY: help define HELP This Makefile assists in handling i18n related tasks during development. Files that need to be checked into source control are put into the locales/ directory. They are locales/$(POT_NAME) - the POT file generated by 'make update-pot' locales/$$LANG.po - the translations for $$LANG Only the $$LANG.po files should be edited manually; this is usually done by translators. You can use the following targets: i18n: refresh all the files in locales/ and recompile resources update-pot: extract strings and update locales/$(POT_NAME) locales/LANG.po: create translations for LANG msgfmt: compile the translations into Java classes; this step is needed to make translations available to the Clojure code and produces Java class files in resources/ endef # @todo lutter 2015-04-20: for projects that use libraries with their own # translation, we need to combine all their translations into one big po # file and then run msgfmt over that so that we only have to deal with one # resource bundle puppetlabs-jruby-utils-ca5d27b/dev-resources/README.md000066400000000000000000000003411464330533300227200ustar00rootroot00000000000000This directory should only contain files (sample data, ssl files, etc.) that are referenced by tests. When possible, files should be placed in a package/directory structure that corresponds with the test that is using them. puppetlabs-jruby-utils-ca5d27b/dev-resources/logback-test.xml000066400000000000000000000005771464330533300245550ustar00rootroot00000000000000 %d %-5p [%t] [%c{2}] %m%n puppetlabs-jruby-utils-ca5d27b/dev-resources/puppetlabs/000077500000000000000000000000001464330533300236225ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/dev-resources/puppetlabs/services/000077500000000000000000000000001464330533300254455ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/dev-resources/puppetlabs/services/jruby_pool_manager/000077500000000000000000000000001464330533300313235ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/dev-resources/puppetlabs/services/jruby_pool_manager/jruby_core_test/000077500000000000000000000000001464330533300345255ustar00rootroot00000000000000foo.rb000077500000000000000000000000241464330533300355550ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/dev-resources/puppetlabs/services/jruby_pool_manager/jruby_core_testdef foo "bar" end puppetlabs-jruby-utils-ca5d27b/locales/000077500000000000000000000000001464330533300202775ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/locales/eo.po000066400000000000000000000120331464330533300212410ustar00rootroot00000000000000# Esperanto translations for puppetlabs.jruby_utils package. # Copyright (C) 2017 Puppet # This file is distributed under the same license as the puppetlabs.jruby_utils package. # Automatically generated, 2017. # msgid "" msgstr "" "Project-Id-Version: puppetlabs.jruby_utils \n" "Report-Msgid-Bugs-To: docs@puppet.com\n" "POT-Creation-Date: \n" "PO-Revision-Date: \n" "Last-Translator: Automatically generated\n" "Language-Team: none\n" "Language: eo\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Initializing JRubyInstances with the following settings:" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Priming JRubyInstance {0} of {1}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished creating JRubyInstance {0} of {1}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "There was a problem adding a JRubyInstance to the pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "There was a problem borrowing a JRubyInstance from the pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "An attempt to lock the JRubyPool failed with a timeout" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "There was a problem creating a JRubyInstance for the pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished draining and refilling pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished draining pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Draining and refilling JRuby pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Draining JRuby pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Borrowed all JRuby instances, proceeding with cleanup." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Beginning flush of JRuby pools for shutdown" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished flush of JRuby pools for shutdown" msgstr "" #. Since the drain-and-refill-pool! function takes the pool lock, we know that if we #. receive multiple flush requests before the first one finishes, they will #. be queued up waiting for the lock, which can't be granted until all the instances #. are returned to the pool, which won't be done until sometimes after #. this function exits #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Flush request received; flushing old JRuby instances." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "compat-version is set to `{0}`, which is not an allowed option." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "The available compat-versions are `1.9` and `2.0`" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Cleaned up old JRubyInstance with id {0}." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "JRuby service missing config value 'ruby-load-path'" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Creating JRubyInstance with id {0}." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Unable to borrow JRubyInstance from pool" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Borrowed unrecognized object from pool!: {0}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "" "Flushing JRubyInstance {0} because it has exceeded the maximum number of " "borrows ({1})" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Acquiring lock on JRubyPool..." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Lock acquired" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Lock on JRubyPool released" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "command {0} could not be found in {1}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Attempt to borrow a JRubyInstance from the pool timed out." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Attempted to borrow a JRubyInstance from the pool during a shutdown." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Please try again." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_pool_manager_service.clj msgid "Initializing the JRuby service" msgstr "" puppetlabs-jruby-utils-ca5d27b/locales/jruby-utils.pot000066400000000000000000000131031464330533300233120ustar00rootroot00000000000000# SOME DESCRIPTIVE TITLE. # Copyright (C) YEAR Puppet # This file is distributed under the same license as the puppetlabs.jruby_utils package. # FIRST AUTHOR , YEAR. # #, fuzzy msgid "" msgstr "" "Project-Id-Version: puppetlabs.jruby_utils \n" "X-Git-Ref: 2f5897fa01123f5d930c9e07ae93e8bfacd17811\n" "Report-Msgid-Bugs-To: docs@puppet.com\n" "POT-Creation-Date: \n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/instance_pool.clj msgid "" "Flushing JRubyInstance {0} because it has exceeded its borrow limit of {1}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "There was a problem adding a JRubyInstance to the pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Initializing JRubyInstances with the following settings:" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Priming JRubyInstance {0} of {1}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished creating JRubyInstance {0} of {1}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "There was a problem borrowing a JRubyInstance from the pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "There was a problem creating a JRubyInstance for the pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished draining and refilling pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished draining pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Draining and refilling JRuby pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Draining JRuby pool." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Borrowed all JRuby instances, proceeding with cleanup." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Beginning flush of JRuby pools for shutdown" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Finished flush of JRuby pools for shutdown" msgstr "" #. Since the drain-and-refill-pool! function takes the pool lock, we know that if we #. receive multiple flush requests before the first one finishes, they will #. be queued up waiting for the lock, which can't be granted until all the instances #. are returned to the pool, which won't be done until sometimes after #. this function exits #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj msgid "Flush request received; flushing old JRuby instances." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Writing jruby profiling output to ''{0}''" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Cleaned up old JRubyInstance with id {0}." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "JRuby service missing config value 'ruby-load-path'" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Creating JRubyInstance with id {0}." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Unable to borrow JRubyInstance from pool" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Borrowed unrecognized object from pool!: {0}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "An attempt to lock the JRubyPool failed with a timeout" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "Exception raised while generating thread dump" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_internal.clj msgid "" "JRuby management interface not enabled. Add ''-Djruby.management." "enabled=true'' to JAVA_ARGS to enable thread dumps." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/reference_pool.clj msgid "Finished creating JRuby instance with id {0}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/impl/reference_pool.clj msgid "" "Max borrows reached, but JRubyPool could not be flushed because lock could " "not be acquired. Will try again later." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Acquiring lock on JRubyPool..." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Lock acquired" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Lock on JRubyPool released" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "command {0} could not be found in {1}" msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Attempt to borrow a JRubyInstance from the pool timed out." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Attempted to borrow a JRubyInstance from the pool during a shutdown." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj msgid "Please try again." msgstr "" #: src/clj/puppetlabs/services/jruby_pool_manager/jruby_pool_manager_service.clj msgid "Initializing the JRuby service" msgstr "" puppetlabs-jruby-utils-ca5d27b/project.clj000066400000000000000000000054141464330533300210210ustar00rootroot00000000000000(defproject puppetlabs/jruby-utils "5.2.0" :description "A library for working with JRuby" :url "https://github.com/puppetlabs/jruby-utils" :license {:name "Apache License, Version 2.0" :url "http://www.apache.org/licenses/LICENSE-2.0"} :min-lein-version "2.9.1" :parent-project {:coords [puppetlabs/clj-parent "7.2.3"] :inherit [:managed-dependencies]} :pedantic? :abort :source-paths ["src/clj"] :java-source-paths ["src/java"] :test-paths ["test/unit" "test/integration"] :dependencies [[org.clojure/clojure] [org.clojure/java.jmx] [org.clojure/tools.logging] [clj-commons/fs] [prismatic/schema] [slingshot] [puppetlabs/jruby-deps "9.4.8.0-1"] [puppetlabs/i18n] [puppetlabs/kitchensink] [puppetlabs/trapperkeeper] [puppetlabs/ring-middleware]] :deploy-repositories [["releases" {:url "https://clojars.org/repo" :username :env/clojars_jenkins_username :password :env/clojars_jenkins_password :sign-releases false}]] ;; By declaring a classifier here and a corresponding profile below we'll get an additional jar ;; during `lein jar` that has all the code in the test/ directory. Downstream projects can then ;; depend on this test jar using a :classifier in their :dependencies to reuse the test utility ;; code that we have. :classifiers [["test" :testutils]] :profiles {:dev {:dependencies [[puppetlabs/kitchensink :classifier "test" :scope "test"] [puppetlabs/trapperkeeper :classifier "test" :scope "test"] [org.bouncycastle/bcpkix-jdk18on] [org.tcrawley/dynapath]] :jvm-opts ~(let [version (System/getProperty "java.specification.version") [major minor _] (clojure.string/split version #"\.")] (concat ["-Djruby.logger.class=com.puppetlabs.jruby_utils.jruby.Slf4jLogger" "-XX:+UseG1GC" "-Xms1G" "-Xmx2G"] (if (= 17 (java.lang.Integer/parseInt major)) ["--add-opens" "java.base/sun.nio.ch=ALL-UNNAMED" "--add-opens" "java.base/java.io=ALL-UNNAMED"] [])))} :testutils {:source-paths ^:replace ["test/unit" "test/integration"]}} :plugins [[lein-parent "0.3.7"] [puppetlabs/i18n "0.8.0" :hooks false]]) puppetlabs-jruby-utils-ca5d27b/src/000077500000000000000000000000001464330533300174445ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/000077500000000000000000000000001464330533300202145ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/000077500000000000000000000000001464330533300223735ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/000077500000000000000000000000001464330533300242165ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/000077500000000000000000000000001464330533300300745ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/impl/000077500000000000000000000000001464330533300310355ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/impl/instance_pool.clj000066400000000000000000000054071464330533300343720ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.impl.instance-pool (:require [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.services.protocols.jruby-pool :as pool-protocol] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [clojure.tools.logging :as log] [puppetlabs.i18n.core :as i18n]) (:import (puppetlabs.services.jruby_pool_manager.jruby_schemas InstancePool))) (extend-type InstancePool pool-protocol/JRubyPool (fill [pool-context] (let [modify-instance-agent (jruby-agents/get-modify-instance-agent pool-context)] (jruby-agents/send-agent modify-instance-agent #(jruby-agents/prime-pool! pool-context)))) (shutdown [pool-context] (jruby-agents/flush-pool-for-shutdown! pool-context)) (lock [pool-context] (let [pool (jruby-internal/get-pool pool-context)] (.lock pool))) (lock-with-timeout [pool-context timeout time-unit] (let [pool (jruby-internal/get-pool pool-context)] (.lockWithTimeout pool timeout time-unit))) (unlock [pool-context] (let [pool (jruby-internal/get-pool pool-context)] (.unlock pool))) (worker-id [pool-context instance] (:id instance)) (borrow [pool-context] (let [instance (jruby-internal/borrow-from-pool pool-context)] [instance (pool-protocol/worker-id pool-context instance)])) (borrow-with-timeout [pool-context timeout] (let [instance (jruby-internal/borrow-from-pool-with-timeout pool-context timeout)] [instance (pool-protocol/worker-id pool-context instance)])) (return [pool-context instance] (when (jruby-schemas/jruby-instance? instance) (let [new-state (swap! (jruby-internal/get-instance-state-container instance) #(update-in % [:borrow-count] inc)) {:keys [initial-borrows max-borrows pool]} (:internal instance) borrow-limit (or initial-borrows max-borrows) worker-id (pool-protocol/worker-id pool-context instance)] (if (and (pos? borrow-limit) (>= (:borrow-count new-state) borrow-limit)) (do (log/info (i18n/trs "Flushing JRubyInstance {0} because it has exceeded its borrow limit of {1}" worker-id borrow-limit)) (jruby-agents/send-flush-instance! pool-context instance)) (.releaseItem pool instance)) ;; Return the worker-id, to be used in metrics and event logging worker-id))) (flush-pool [pool-context] (jruby-agents/flush-and-repopulate-pool! pool-context))) puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_agents.clj000066400000000000000000000316331464330533300342310ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.impl.jruby-agents (:require [schema.core :as schema] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [clojure.tools.logging :as log] [puppetlabs.kitchensink.core :as ks] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [puppetlabs.i18n.core :as i18n]) (:import (clojure.lang IFn IDeref) (puppetlabs.services.jruby_pool_manager.jruby_schemas PoisonPill JRubyInstance) (java.util.concurrent TimeUnit TimeoutException ExecutionException Future ExecutorService))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Private (schema/defn execute-tasks! [tasks :- [IFn] task-executor :- ExecutorService] (let [results (.invokeAll task-executor tasks)] (try (doseq [result results] (.get ^Future result)) (catch ExecutionException ex (throw (.getCause ex)))))) (schema/defn ^:always-validate next-instance-id :- schema/Int [id :- schema/Int pool-context :- jruby-schemas/PoolContext] (let [pool-size (jruby-internal/get-pool-size pool-context) next-id (+ id pool-size)] (if (> next-id Integer/MAX_VALUE) (mod next-id pool-size) next-id))) (schema/defn get-shutdown-on-error-fn :- IFn [pool-context :- jruby-schemas/PoolContext] (get-in pool-context [:config :lifecycle :shutdown-on-error])) (schema/defn get-modify-instance-agent :- jruby-schemas/JRubyPoolAgent [pool-context :- jruby-schemas/PoolContext] (get-in pool-context [:internal :modify-instance-agent])) (schema/defn ^:always-validate send-agent :- jruby-schemas/JRubyPoolAgent "Utility function; given a JRubyPoolAgent, send the specified function. Ensures that the function call is wrapped in a `shutdown-on-error`." [jruby-agent :- jruby-schemas/JRubyPoolAgent f :- IFn] (letfn [(agent-fn [agent-ctxt] (let [shutdown-on-error (:shutdown-on-error agent-ctxt)] (shutdown-on-error f)) agent-ctxt)] (send jruby-agent agent-fn))) (declare send-flush-instance!) (schema/defn add-instance [{:keys [config] :as pool-context} :- jruby-schemas/PoolContext id :- schema/Int] (let [pool (jruby-internal/get-pool pool-context)] (try (jruby-internal/create-pool-instance! pool id config (:splay-instance-flush config)) (catch Exception e (.clear pool) (jruby-internal/insert-poison-pill pool e) (throw (IllegalStateException. (i18n/tru "There was a problem adding a JRubyInstance to the pool.") e)))))) (schema/defn ^:always-validate prime-pool! "Fill the pool with new JRubyInstances. Instantiates the first JRuby (Puppet will sometimes alter the filesystem on first instantiation) and the remaining instances in parallel. NOTE: this function should never be called except by the modify-instance-agent to create a pool's initial jruby instances." [{:keys [config] :as pool-context} :- jruby-schemas/PoolContext] (log/debug (format "%s\n%s" (i18n/trs "Initializing JRubyInstances with the following settings:") (ks/pprint-to-string config))) (let [pool (jruby-internal/get-pool pool-context) creation-service (jruby-internal/get-creation-service pool-context) total (.remainingCapacity pool) [first-id & ids] (->> total range (map inc)) add-instance* (fn [id] (log/debug (i18n/trs "Priming JRubyInstance {0} of {1}" id count)) (add-instance pool-context id) (log/info (i18n/trs "Finished creating JRubyInstance {0} of {1}" id count))) initial-task (fn [] (add-instance* first-id)) tasks (for [id ids] (fn [] (add-instance* id)))] (execute-tasks! [initial-task] creation-service) (when (seq ids) (execute-tasks! tasks creation-service)))) (schema/defn ^:always-validate flush-instance! "Flush a single JRubyInstance. Create a new replacement instance and insert it into the specified pool. Should only be called from the modify-instance-agent" [pool-context :- jruby-schemas/PoolContext instance :- JRubyInstance new-id :- schema/Int config :- jruby-schemas/JRubyConfig] (let [cleanup-fn (get-in pool-context [:config :lifecycle :cleanup]) pool (jruby-internal/get-pool pool-context)] (jruby-internal/cleanup-pool-instance! instance cleanup-fn) (jruby-internal/create-pool-instance! pool new-id config))) (schema/defn borrow-all-jrubies* "The core logic for borrow-all-jrubies. Should only be called from borrow-all-jrubies" [pool-context :- jruby-schemas/PoolContext borrow-exception :- IDeref] (let [pool-size (jruby-internal/get-pool-size pool-context) pool (jruby-internal/get-pool pool-context) borrow-fn (partial jruby-internal/borrow-from-pool pool-context)] (try (into [] (repeatedly pool-size borrow-fn)) ; We catch the exception here, place it in the borrow-exception atom ; for use in the calling fn, and then throw it again so that ; shutdown-on-error will also catch it and shutdown the app (catch Exception e (.clear pool) (jruby-internal/insert-poison-pill pool e) (let [exception (IllegalStateException. (i18n/tru "There was a problem borrowing a JRubyInstance from the pool.") e)] (reset! borrow-exception exception) (throw exception))) (finally (.unlock pool))))) (schema/defn borrow-all-jrubies :- [JRubyInstance] "Locks the pool and borrows all the instances" [pool-context :- jruby-schemas/PoolContext] (let [pool (jruby-internal/get-pool pool-context) flush-timeout (jruby-internal/get-flush-timeout pool-context) shutdown-on-error (get-shutdown-on-error-fn pool-context) borrow-exception (atom nil)] ; If lock fails, abort and throw an exception (try (.lockWithTimeout pool flush-timeout TimeUnit/MILLISECONDS) (catch TimeoutException e (jruby-internal/throw-jruby-lock-timeout e))) ; Bit of a hack to work around shutdown-on-error behavior: ; shutdown-on-error will either return the jrubies as expected, ; or if there was an error, it will shutdown the server and return a promise. ; We want to rethrow whatever exception it encountered so that it can bubble ; up to whatever code requested this flush, so borrow-all-jrubies* will put ; that exception into the borrow-exception atom (let [jrubies (shutdown-on-error #(borrow-all-jrubies* pool-context borrow-exception))] (if-let [exception @borrow-exception] (throw exception) jrubies)))) (schema/defn cleanup-and-refill-pool "Cleans up the given instances and optionally refills the pool with new instances. Should only be called from the modify-instance-agent" [pool-context :- jruby-schemas/PoolContext old-instances :- [JRubyInstance] refill? :- schema/Bool] (let [pool (jruby-internal/get-pool pool-context) pool-size (jruby-internal/get-pool-size pool-context) creation-service (jruby-internal/get-creation-service pool-context) new-instance-ids (map inc (range pool-size)) config (:config pool-context) cleanup-fn (get-in config [:lifecycle :cleanup]) cleanup-and-refill-instance (fn [old-instance new-id] (try (jruby-internal/cleanup-pool-instance! old-instance cleanup-fn) (when refill? (jruby-internal/create-pool-instance! pool new-id config (:splay-instance-flush config)) (log/info (i18n/trs "Finished creating JRubyInstance {0} of {1}" new-id pool-size))) (catch Exception e (.clear pool) (jruby-internal/insert-poison-pill pool e) (throw (IllegalStateException. (i18n/trs "There was a problem creating a JRubyInstance for the pool.") e))))) [[first-old-inst first-new-id] & remaining] (zipmap old-instances new-instance-ids) first-task [(fn [] (cleanup-and-refill-instance first-old-inst first-new-id))] remaining-tasks (for [[old-instance new-id] remaining] (fn [] (cleanup-and-refill-instance old-instance new-id)))] (execute-tasks! first-task creation-service) (when remaining-tasks (execute-tasks! remaining-tasks creation-service))) (if refill? (log/info (i18n/trs "Finished draining and refilling pool.")) (log/info (i18n/trs "Finished draining pool.")))) (schema/defn ^:always-validate drain-and-refill-pool! "Borrow and destroy all the jruby instances, optionally refilling the pool with fresh jrubies. Locks the pool in order to drain it, but releases the lock before destroying the instances and refilling the pool If an on-complete promise is given, it can be used by the caller to make this function syncronous. Otherwise it only blocks until the pool instances have been borrowed and the cleanup-and-refill-pool fn is sent to the agent" ([pool-context :- jruby-schemas/PoolContext refill? :- schema/Bool] (drain-and-refill-pool! pool-context refill? (promise))) ([pool-context :- jruby-schemas/PoolContext refill? :- schema/Bool on-complete :- IDeref] (if refill? (log/info (i18n/trs "Draining and refilling JRuby pool.")) (log/info (i18n/trs "Draining JRuby pool."))) (let [old-instances (borrow-all-jrubies pool-context) modify-instance-agent (get-modify-instance-agent pool-context) ; Make sure the promise is delivered even if cleanup fails try-cleanup-and-refill #(try (cleanup-and-refill-pool pool-context old-instances refill?) (finally (deliver on-complete true)))] (log/info (i18n/trs "Borrowed all JRuby instances, proceeding with cleanup.")) (send-agent modify-instance-agent try-cleanup-and-refill)))) (schema/defn ^:always-validate flush-pool-for-shutdown! "Flush of the current JRuby pool when shutting down during a stop." ;; Since the drain-pool! function takes the pool lock, we know that if we ;; receive multiple flush requests before the first one finishes, they will ;; be queued up waiting for the lock, which will never be granted because this ;; function does not refill the pool, but instead inserts a shutdown poison pill [pool-context :- jruby-schemas/PoolContext] (log/debug (i18n/trs "Beginning flush of JRuby pools for shutdown")) (let [pool-state (jruby-internal/get-pool-state pool-context) pool (:pool pool-state) on-complete (promise)] (drain-and-refill-pool! pool-context false on-complete) (jruby-internal/insert-shutdown-poison-pill pool) ; Wait for flush to complete @on-complete (log/debug (i18n/trs "Finished flush of JRuby pools for shutdown")))) (schema/defn ^:always-validate flush-and-repopulate-pool! "Flush of the current JRuby pool. Blocks until all the instances have been borrowed from the pool, but does not wait for the instances to be flushed or recreated" [pool-context :- jruby-schemas/PoolContext] ;; Since the drain-and-refill-pool! function takes the pool lock, we know that if we ;; receive multiple flush requests before the first one finishes, they will ;; be queued up waiting for the lock, which can't be granted until all the instances ;; are returned to the pool, which won't be done until sometimes after ;; this function exits (log/info (i18n/trs "Flush request received; flushing old JRuby instances.")) (drain-and-refill-pool! pool-context true)) (schema/defn ^:always-validate send-flush-instance! :- jruby-schemas/JRubyPoolAgent "Sends requests to the flush-instance agent to flush the instance and create a new one." [pool-context :- jruby-schemas/PoolContext instance :- JRubyInstance] ;; We use an agent to syncronize jruby creation and destruction to mitigate ;; any possible race conditions in the underlying jruby scripting container (let [{:keys [config]} pool-context modify-instance-agent (get-modify-instance-agent pool-context) id (next-instance-id (:id instance) pool-context)] (send-agent modify-instance-agent #(flush-instance! pool-context instance id config)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Public (schema/defn ^:always-validate pool-agent :- jruby-schemas/JRubyPoolAgent "Given a shutdown-on-error function, create an agent suitable for use in managing JRuby pools." [shutdown-on-error-fn :- (schema/pred ifn?)] (agent {:shutdown-on-error shutdown-on-error-fn})) puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/impl/jruby_events.clj000066400000000000000000000077061464330533300342600ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.impl.jruby-events (:require [schema.core :as schema] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas]) (:import (clojure.lang IFn))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Private (schema/defn create-requested-event :- jruby-schemas/JRubyRequestedEvent [reason :- jruby-schemas/JRubyEventReason] {:type :instance-requested :reason reason}) (schema/defn create-borrowed-event :- jruby-schemas/JRubyBorrowedEvent [requested-event :- jruby-schemas/JRubyRequestedEvent instance :- jruby-schemas/JRubyBorrowResult worker-id :- jruby-schemas/JRubyWorkerId] {:type :instance-borrowed :reason (:reason requested-event) :requested-event requested-event :instance instance :worker-id worker-id}) (schema/defn create-returned-event :- jruby-schemas/JRubyReturnedEvent [instance :- jruby-schemas/JRubyInstanceOrPill reason :- jruby-schemas/JRubyEventReason worker-id :- jruby-schemas/JRubyWorkerId] {:type :instance-returned :reason reason :instance instance :worker-id worker-id}) (schema/defn create-lock-requested-event :- jruby-schemas/JRubyLockRequestedEvent [reason :- jruby-schemas/JRubyEventReason] {:type :lock-requested :reason reason}) (schema/defn create-lock-acquired-event :- jruby-schemas/JRubyLockAcquiredEvent [reason :- jruby-schemas/JRubyEventReason] {:type :lock-acquired :reason reason}) (schema/defn create-lock-released-event :- jruby-schemas/JRubyLockReleasedEvent [reason :- jruby-schemas/JRubyEventReason] {:type :lock-released :reason reason}) (schema/defn notify-event-listeners :- jruby-schemas/JRubyEvent [event-callbacks :- [IFn] event :- jruby-schemas/JRubyEvent] (doseq [f event-callbacks] (f event)) event) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Public (schema/defn instance-requested :- jruby-schemas/JRubyRequestedEvent [event-callbacks :- [IFn] reason :- jruby-schemas/JRubyEventReason] (notify-event-listeners event-callbacks (create-requested-event reason))) (schema/defn instance-borrowed :- jruby-schemas/JRubyBorrowedEvent [event-callbacks :- [IFn] requested-event :- jruby-schemas/JRubyRequestedEvent instance :- jruby-schemas/JRubyBorrowResult worker-id :- jruby-schemas/JRubyWorkerId] (notify-event-listeners event-callbacks (create-borrowed-event requested-event instance worker-id))) (schema/defn instance-returned :- jruby-schemas/JRubyReturnedEvent [event-callbacks :- [IFn] instance :- jruby-schemas/JRubyInstanceOrPill reason :- jruby-schemas/JRubyEventReason worker-id :- jruby-schemas/JRubyWorkerId] (notify-event-listeners event-callbacks (create-returned-event instance reason worker-id))) (schema/defn lock-requested :- jruby-schemas/JRubyLockRequestedEvent [event-callbacks :- [IFn] reason :- jruby-schemas/JRubyEventReason] (notify-event-listeners event-callbacks (create-lock-requested-event reason))) (schema/defn lock-acquired :- jruby-schemas/JRubyLockAcquiredEvent [event-callbacks :- [IFn] reason :- jruby-schemas/JRubyEventReason] (notify-event-listeners event-callbacks (create-lock-acquired-event reason))) (schema/defn lock-released :- jruby-schemas/JRubyLockReleasedEvent [event-callbacks :- [IFn] reason :- jruby-schemas/JRubyEventReason] (notify-event-listeners event-callbacks (create-lock-released-event reason))) jruby_internal.clj000066400000000000000000000413031464330533300345000ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/impl(ns puppetlabs.services.jruby-pool-manager.impl.jruby-internal (:require [clj-time.core :as time-core] [clj-time.format :as time-format] [clojure.java.io :as io] [clojure.java.jmx :as jmx] [clojure.string :refer [upper-case]] [clojure.tools.logging :as log] [me.raynes.fs :as fs] [puppetlabs.i18n.core :as i18n] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [slingshot.slingshot :as sling] [schema.core :as schema]) (:import (clojure.lang IFn) (com.puppetlabs.jruby_utils.pool JRubyPool ReferencePool) (com.puppetlabs.jruby_utils.jruby InternalScriptingContainer ScriptingContainer) (java.io File) (java.util.concurrent TimeUnit Executors ExecutorService) (org.jruby CompatVersion Main Ruby RubyInstanceConfig RubyInstanceConfig$CompileMode RubyInstanceConfig$ProfilingMode) (org.jruby.embed LocalContextScope) (org.jruby.runtime.profile.builtin ProfileOutput) (org.jruby.util KCode) (puppetlabs.services.jruby_pool_manager.jruby_schemas JRubyInstance PoisonPill ShutdownPoisonPill))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Private (schema/defn ^:always-validate initialize-gem-path :- {schema/Keyword schema/Any} [{:keys [gem-path gem-home] :as jruby-config} :- {schema/Keyword schema/Any}] (if gem-path jruby-config (assoc jruby-config :gem-path nil))) (defn instantiate-instance-pool "Instantiate a new queue object to use as the pool of free JRuby's." [size] {:post [(instance? jruby-schemas/pool-queue-type %)]} (JRubyPool. size)) (defn instantiate-reference-pool "Instantiate a new queue object to use as the pool of free JRuby's." [max-concurrent-borrows] {:post [(instance? jruby-schemas/pool-queue-type %)]} (ReferencePool. max-concurrent-borrows)) (schema/defn ^:always-validate get-compile-mode :- RubyInstanceConfig$CompileMode [config-compile-mode :- jruby-schemas/SupportedJRubyCompileModes] (case config-compile-mode :jit RubyInstanceConfig$CompileMode/JIT :force RubyInstanceConfig$CompileMode/FORCE :off RubyInstanceConfig$CompileMode/OFF)) (schema/defn ^:always-validate get-profiling-mode :- RubyInstanceConfig$ProfilingMode [config-profiling-mode :- jruby-schemas/SupportedJRubyProfilingModes] (RubyInstanceConfig$ProfilingMode/valueOf (upper-case (name config-profiling-mode)))) (schema/defn ^:always-validate setup-profiling "Takes a jruby and sets profiling mode and profiler output, appending the current time to the filename for uniqueness and notifying the user via log message of the profile file name." [jruby :- jruby-schemas/ConfigurableJRuby profiler-output-file :- schema/Str profiling-mode :- schema/Keyword] (when (and profiler-output-file (not= :off profiling-mode)) (let [current-time-string (time-format/unparse (time-format/formatters :basic-date-time-no-ms) (time-core/now)) real-profiler-output-file (io/as-file (str profiler-output-file "-" (.hashCode jruby) "-" current-time-string))] (doto jruby (.setProfileOutput (ProfileOutput. ^File real-profiler-output-file)) (.setProfilingMode (get-profiling-mode profiling-mode))) (log/info (i18n/trs "Writing jruby profiling output to ''{0}''" real-profiler-output-file))))) (schema/defn ^:always-validate set-config-encoding :- RubyInstanceConfig "Sets the K code, source encoding, and external encoding of the JRuby config to the supplied encoding." [kcode :- KCode jruby :- RubyInstanceConfig] (let [encoding-string (str (.getEncoding kcode))] (doto jruby (.setKCode kcode) (.setSourceEncoding encoding-string) (.setExternalEncoding encoding-string)))) (schema/defn ^:always-validate set-ruby-encoding :- jruby-schemas/ConfigurableJRuby [kcode :- KCode jruby :- jruby-schemas/ConfigurableJRuby] (if (instance? RubyInstanceConfig jruby) (set-config-encoding kcode jruby) (set-config-encoding kcode (.getRubyInstanceConfig (.getProvider jruby))))) (schema/defn ^:always-validate init-jruby :- jruby-schemas/ConfigurableJRuby "Applies configuration to a JRuby... thing. See comments in `ConfigurableJRuby` schema for more details." [jruby :- jruby-schemas/ConfigurableJRuby config :- jruby-schemas/JRubyConfig] (let [{:keys [ruby-load-path compile-mode lifecycle profiling-mode profiler-output-file]} config initialize-scripting-container-fn (:initialize-scripting-container lifecycle)] (doto jruby (.setLoadPaths ruby-load-path) (.setCompileMode (get-compile-mode compile-mode))) (set-ruby-encoding KCode/UTF8 jruby) (setup-profiling jruby profiler-output-file profiling-mode) (System/setProperty "jruby.invokedynamic.yield" "false") (initialize-scripting-container-fn jruby config))) (schema/defn ^:always-validate empty-scripting-container :- ScriptingContainer "Creates a clean instance of a JRuby `ScriptingContainer` with no code loaded." [config :- jruby-schemas/JRubyConfig] (-> (InternalScriptingContainer. LocalContextScope/SINGLETHREAD) (init-jruby config))) (schema/defn ^:always-validate create-scripting-container :- ScriptingContainer "Creates an instance of `org.jruby.embed.ScriptingContainer`." [config :- jruby-schemas/JRubyConfig] ;; for information on other legal values for `LocalContextScope`, there ;; is some documentation available in the JRuby source code; e.g.: ;; https://github.com/jruby/jruby/blob/1.7.11/core/src/main/java/org/jruby/embed/LocalContextScope.java#L58 ;; I'm convinced that this is the safest and most reasonable value ;; to use here, but we could potentially explore optimizations in the future. (doto (empty-scripting-container config) ;; As of JRuby 1.7.20 (and the associated 'jruby-openssl' it pulls in), ;; we need to explicitly require 'jar-dependencies' so that it is used ;; to manage jar loading. We do this so that we can instruct ;; 'jar-dependencies' to not actually load any jars. See the environment ;; variable configuration in 'init-jruby-config' for more ;; information. (.runScriptlet "require 'jar-dependencies'"))) (schema/defn borrow-with-timeout-fn :- jruby-schemas/JRubyInternalBorrowResult [timeout :- schema/Int pool :- jruby-schemas/pool-queue-type] (.borrowItemWithTimeout pool timeout TimeUnit/MILLISECONDS)) (schema/defn insert-shutdown-poison-pill [pool :- jruby-schemas/pool-queue-type] (.insertPill pool (ShutdownPoisonPill. pool))) (schema/defn insert-poison-pill [pool :- jruby-schemas/pool-queue-type error :- Throwable] (.insertPill pool (PoisonPill. error))) (schema/defn ^:always-validate get-jruby-runtime :- Ruby "Get the org.jruby.Ruby instance associated with member of the pool." [{:keys [scripting-container]} :- JRubyInstance] (-> scripting-container .getProvider .getRuntime)) (schema/defn ^:always-validate management-enabled? :- schema/Bool [instance :- JRubyInstance] (-> (get-jruby-runtime instance) .getInstanceConfig .isManagementEnabled)) (def JRubyMBeanName "Enumeration of available JMX MBeans for a JRubyInstance." (schema/enum "Caches" "Config" "JITCompiler" "ParserStats" "Runtime")) (schema/defn ^:always-validate jmx-bean-name :- schema/Str "Get the fully-qualified name of a JMX MBean attached to a JRubyInstance." [instance :- JRubyInstance service-name :- JRubyMBeanName] (-> (get-jruby-runtime instance) .getBeanManager .base (str "service=" service-name))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Public (schema/defn ^:always-validate create-pool-from-config :- jruby-schemas/PoolState "Create a new PoolState based on the config input." [config :- jruby-schemas/JRubyConfig] (let [multithreaded (:multithreaded config) size (:max-active-instances config) creation-concurrency (:instance-creation-concurrency config) creation-service (Executors/newFixedThreadPool creation-concurrency)] (if multithreaded {:pool (instantiate-reference-pool size) :size 1 :creation-service creation-service} {:pool (instantiate-instance-pool size) :size size :creation-service creation-service}))) (schema/defn ^:always-validate cleanup-pool-instance! "Cleans up and cleanly terminates a JRubyInstance and removes it from the pool." [{:keys [scripting-container] :as instance} :- JRubyInstance cleanup-fn :- IFn] (let [pool (get-in instance [:internal :pool])] (.unregister pool instance) (cleanup-fn instance) (.terminate scripting-container) (log/info (i18n/trs "Cleaned up old JRubyInstance with id {0}." (:id instance))))) (schema/defn ^:always-validate initial-borrows-value :- (schema/maybe schema/Int) "Determines how many borrows before instance of given id should be flushed in order to best splay it. Returns nil if not applicable (either not `initial-jruby?` or there are more instances than max-borrows)." [id :- schema/Int total-instances :- schema/Int total-borrows :- schema/Int initial-jruby? :- schema/Bool] (when initial-jruby? (let [which-step (inc (mod id total-instances)) step-size (quot total-borrows total-instances)] ;; If total-instances is larger than total-borrows then step-size will ;; be 0. For example, if a user has configured their pool with 4 JRuby ;; instances but a max-requests-per-instance of 1. Users shouldn't need ;; splaying if they are never re-using JRuby instances, or if they are ;; re-using them fewer times than the number of JRubies in the pool. (when-not (= 0 step-size) (* step-size which-step))))) (schema/defn ^:always-validate create-pool-instance! :- JRubyInstance "Creates a new JRubyInstance and adds it to the pool." ([pool :- jruby-schemas/pool-queue-type id :- schema/Int config :- jruby-schemas/JRubyConfig] (create-pool-instance! pool id config false)) ([pool :- jruby-schemas/pool-queue-type id :- schema/Int config :- jruby-schemas/JRubyConfig initial-jruby? :- schema/Bool] (let [{:keys [ruby-load-path lifecycle max-active-instances max-borrows-per-instance]} config initialize-pool-instance-fn (:initialize-pool-instance lifecycle) initial-borrows (initial-borrows-value id max-active-instances max-borrows-per-instance initial-jruby?)] (when-not ruby-load-path (throw (Exception. (i18n/trs "JRuby service missing config value 'ruby-load-path'")))) (log/info (i18n/trs "Creating JRubyInstance with id {0}." id)) (let [scripting-container (create-scripting-container config)] (let [instance (jruby-schemas/map->JRubyInstance {:scripting-container scripting-container :id id :internal {:pool pool :max-borrows max-borrows-per-instance :initial-borrows initial-borrows :state (atom {:borrow-count 0})}}) modified-instance (initialize-pool-instance-fn instance)] (.register pool modified-instance) modified-instance))))) (schema/defn ^:always-validate get-pool-state-container :- jruby-schemas/PoolStateContainer "Gets the PoolStateContainer from the pool context." [context :- jruby-schemas/PoolContext] (get-in context [:internal :pool-state])) (schema/defn ^:always-validate get-pool-state :- jruby-schemas/PoolState "Gets the PoolState from the pool context." [context :- jruby-schemas/PoolContext] @(get-pool-state-container context)) (schema/defn ^:always-validate get-pool :- jruby-schemas/pool-queue-type "Gets the JRuby pool object from the pool context." [context :- jruby-schemas/PoolContext] (:pool (get-pool-state context))) (schema/defn ^:always-validate get-pool-size :- schema/Int "Gets the size of the JRuby pool from the pool context." [context :- jruby-schemas/PoolContext] (:size (get-pool-state context))) (schema/defn get-creation-service :- ExecutorService "Gets the ExecutorService that will execute instance creation and termination." [context :- jruby-schemas/PoolContext] (:creation-service (get-pool-state context))) (schema/defn ^:always-validate get-flush-timeout :- schema/Int "Gets the size of the JRuby pool from the pool context." [context :- jruby-schemas/PoolContext] (get-in context [:config :flush-timeout])) (schema/defn ^:always-validate get-instance-state-container :- jruby-schemas/JRubyInstanceStateContainer "Gets the InstanceStateContainer (atom) from the instance." [instance :- JRubyInstance] (get-in instance [:internal :state])) (schema/defn borrow-without-timeout-fn :- jruby-schemas/JRubyInternalBorrowResult [pool :- jruby-schemas/pool-queue-type] (.borrowItem pool)) (schema/defn borrow-from-pool!* :- jruby-schemas/JRubyBorrowResult "Given a borrow function and a pool, attempts to borrow a JRubyInstance from a pool. If successful, updates the state information and returns the JRubyInstance. Returns nil if the borrow function returns nil; throws an exception if the borrow function's return value indicates an error condition." [borrow-fn :- (schema/pred ifn?) pool :- jruby-schemas/pool-queue-type] (let [instance (borrow-fn pool)] (cond (instance? PoisonPill instance) (do (.releaseItem pool instance) (throw (IllegalStateException. (i18n/tru "Unable to borrow JRubyInstance from pool") (:err instance)))) (jruby-schemas/jruby-instance? instance) instance (jruby-schemas/shutdown-poison-pill? instance) instance (nil? instance) instance :else (throw (IllegalStateException. (i18n/tru "Borrowed unrecognized object from pool!: {0}" instance)))))) (schema/defn ^:always-validate borrow-from-pool :- jruby-schemas/JRubyInstanceOrPill "Borrows a JRuby interpreter from the pool. If there are no instances left in the pool then this function will block until there is one available." [pool-context :- jruby-schemas/PoolContext] (borrow-from-pool!* borrow-without-timeout-fn (get-pool pool-context))) (schema/defn ^:always-validate borrow-from-pool-with-timeout :- jruby-schemas/JRubyBorrowResult "Borrows a JRuby interpreter from the pool, like borrow-from-pool but a blocking timeout is provided. If an instance is available then it will be immediately returned to the caller, if not then this function will block waiting for an instance to be free for the number of milliseconds given in timeout. If the timeout runs out then nil will be returned, indicating that there were no instances available." [pool-context :- jruby-schemas/PoolContext timeout :- schema/Int] {:pre [(>= timeout 0)]} (borrow-from-pool!* (partial borrow-with-timeout-fn timeout) (get-pool pool-context))) (defn throw-jruby-lock-timeout [exception] (sling/throw+ {:kind ::jruby-lock-timeout :msg (i18n/trs "An attempt to lock the JRubyPool failed with a timeout")} exception)) (schema/defn ^:always-validate get-instance-thread-dump [instance :- JRubyInstance] (if (management-enabled? instance) (try {:thread-dump (jmx/invoke (jmx-bean-name instance "Runtime") :threadDump)} (catch Exception e (let [system_error (i18n/trs "Exception raised while generating thread dump") user_error (i18n/tru "Exception raised while generating thread dump")] (log/error e system_error) {:error (str user_error ": " (.toString e))}))) {:error (i18n/tru "JRuby management interface not enabled. Add ''-Djruby.management.enabled=true'' to JAVA_ARGS to enable thread dumps.")})) (schema/defn ^:always-validate new-main :- jruby-schemas/JRubyMain "Return a new JRubyMain instance which should only be used for CLI purposes, e.g. for the ruby, gem, and irb subcommands. Internal core services should use `create-scripting-container` instead of `new-main`." [config :- jruby-schemas/JRubyConfig] (let [jruby-config (init-jruby (RubyInstanceConfig.) config)] (Main. jruby-config))) jruby_pool_manager_core.clj000066400000000000000000000031261464330533300363400ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/impl(ns puppetlabs.services.jruby-pool-manager.impl.jruby-pool-manager-core (:require [schema.core :as schema] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.services.protocols.jruby-pool :as pool-protocol] [puppetlabs.services.jruby-pool-manager.impl.reference-pool] [puppetlabs.services.jruby-pool-manager.impl.instance-pool] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal]) (:import (puppetlabs.services.jruby_pool_manager.jruby_schemas ReferencePool InstancePool))) (schema/defn ^:always-validate create-pool-context :- jruby-schemas/PoolContext "Creates a new JRuby pool context with an empty pool. Once the JRuby pool object has been created, it will need to be filled using `prime-pool!`." [config :- jruby-schemas/JRubyConfig] (let [shutdown-on-error-fn (get-in config [:lifecycle :shutdown-on-error]) internal {:modify-instance-agent (jruby-agents/pool-agent shutdown-on-error-fn) :pool-state (atom (jruby-internal/create-pool-from-config config)) :event-callbacks (atom [])}] (if (:multithreaded config) (ReferencePool. config internal (atom 0)) (InstancePool. config internal)))) (schema/defn ^:always-validate create-pool :- jruby-schemas/PoolContext [config :- jruby-schemas/JRubyConfig] (let [pool-context (create-pool-context config)] (pool-protocol/fill pool-context) pool-context)) reference_pool.clj000066400000000000000000000146561464330533300344530ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/impl(ns puppetlabs.services.jruby-pool-manager.impl.reference-pool (:require [puppetlabs.services.protocols.jruby-pool :as pool-protocol] [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [clojure.tools.logging :as log] [puppetlabs.i18n.core :as i18n] [schema.core :as schema]) (:import (puppetlabs.services.jruby_pool_manager.jruby_schemas ReferencePool JRubyInstance) (java.util.concurrent TimeUnit TimeoutException))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Private (schema/defn flush-pool* "Flushes the pool, assuming it has already been locked by the calling function. Do not call this without first locking the pool, or the flush may never complete, since it requires that all references be returned to proceed." [pool-context :- jruby-schemas/PoolContext] (let [pool (jruby-internal/get-pool pool-context) borrow-count (:borrow-count pool-context) cleanup-fn (get-in pool-context [:config :lifecycle :cleanup]) old-instance (.borrowItem pool) id (inc (:id old-instance)) _ (.releaseItem pool old-instance)] ;; This will block waiting for all borrows to be returned (jruby-internal/cleanup-pool-instance! old-instance cleanup-fn) (jruby-agents/add-instance pool-context id) (log/info (i18n/trs "Finished creating JRuby instance with id {0}" id)) (reset! borrow-count 0))) (schema/defn max-borrows-exceeded :- schema/Bool "Returns true if max-borrows is set and the current borrow count has exceeded the allowed maximum." [current-borrows :- schema/Int max-borrows :- schema/Int] (and (pos? max-borrows) (>= current-borrows max-borrows))) (schema/defn flush-if-at-max-borrows [pool-context :- jruby-schemas/PoolContext instance :- JRubyInstance] (let [borrow-count (:borrow-count pool-context) max-borrows (get-in instance [:internal :max-borrows]) flush-timeout (jruby-internal/get-flush-timeout pool-context)] (try ;; Lock will block until all references have been returned to the pool or ;; until flush-timeout is reached (pool-protocol/lock-with-timeout pool-context flush-timeout TimeUnit/MILLISECONDS) (try ;; Now that we've successfully acquired the lock, check the borrows again ;; to make sure the pool wasn't flushed while we were waiting. (when (max-borrows-exceeded @borrow-count max-borrows) (flush-pool* pool-context)) (finally (pool-protocol/unlock pool-context))) (catch TimeoutException e (log/warn (i18n/trs "Max borrows reached, but JRubyPool could not be flushed because lock could not be acquired. Will try again later.")))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; ReferencePool definition (extend-type ReferencePool pool-protocol/JRubyPool (fill [pool-context] (let [modify-instance-agent (jruby-agents/get-modify-instance-agent pool-context)] (jruby-agents/send-agent modify-instance-agent #(jruby-agents/add-instance pool-context 1)))) (shutdown [pool-context] (let [pool (jruby-internal/get-pool pool-context) cleanup-fn (get-in pool-context [:config :lifecycle :cleanup]) flush-timeout (jruby-internal/get-flush-timeout pool-context)] ;; Lock the pool so no borrows or flushes can occur while we're shutting down (try (pool-protocol/lock-with-timeout pool-context flush-timeout TimeUnit/MILLISECONDS) (catch TimeoutException e (jruby-internal/throw-jruby-lock-timeout e))) (try (let [instance (.borrowItem pool) _ (.releaseItem pool instance)] ;; This will block until all borrows have been returned (jruby-internal/cleanup-pool-instance! instance cleanup-fn)) ;; Insert a shutdown pill to ensure that all pending borrows and locks ;; are rejected with the appropriate logging (jruby-internal/insert-shutdown-poison-pill pool) (finally (pool-protocol/unlock pool-context))))) (lock [pool-context] (let [pool (jruby-internal/get-pool pool-context)] (.lock pool))) (lock-with-timeout [pool-context timeout time-unit] (let [pool (jruby-internal/get-pool pool-context)] (.lockWithTimeout pool timeout time-unit))) (unlock [pool-context] (let [pool (jruby-internal/get-pool pool-context)] (.unlock pool))) (worker-id [pool-context instance] (.getId (Thread/currentThread))) (borrow [pool-context] (let [instance (jruby-internal/borrow-from-pool pool-context)] [instance (pool-protocol/worker-id pool-context instance)])) (borrow-with-timeout [pool-context timeout] (let [instance (jruby-internal/borrow-from-pool-with-timeout pool-context timeout)] [instance (pool-protocol/worker-id pool-context instance)])) (return [pool-context instance] (when (jruby-schemas/jruby-instance? instance) (let [pool (jruby-internal/get-pool pool-context) borrow-count (:borrow-count pool-context) max-borrows (get-in instance [:internal :max-borrows]) modify-instance-agent (jruby-agents/get-modify-instance-agent pool-context)] (.releaseItem pool instance) (swap! borrow-count inc) (when (max-borrows-exceeded @borrow-count max-borrows) (jruby-agents/send-agent modify-instance-agent #(flush-if-at-max-borrows pool-context instance))) ;; Return the worker-id, to be used in metrics and event logging (pool-protocol/worker-id pool-context instance)))) (flush-pool [pool-context] (let [flush-timeout (jruby-internal/get-flush-timeout pool-context)] ;; Lock will block until all references have been returned to the pool or ;; until flush-timeout is reached (try (pool-protocol/lock-with-timeout pool-context flush-timeout TimeUnit/MILLISECONDS) (catch TimeoutException e (jruby-internal/throw-jruby-lock-timeout e))) (try (flush-pool* pool-context) (finally (pool-protocol/unlock pool-context)))))) puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/jruby_core.clj000066400000000000000000000370051464330533300327360ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.jruby-core (:require [clojure.tools.logging :as log] [schema.core :as schema] [puppetlabs.kitchensink.core :as ks] [puppetlabs.ring-middleware.utils :as ringutils] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.services.jruby-pool-manager.impl.jruby-events :as jruby-events] [clojure.java.io :as io] [clojure.tools.logging :as log] [slingshot.slingshot :as sling] [puppetlabs.i18n.core :as i18n] [me.raynes.fs :as fs] [puppetlabs.services.protocols.jruby-pool :as pool-protocol]) (:import (puppetlabs.services.jruby_pool_manager.jruby_schemas JRubyInstance) (clojure.lang IFn) (java.util.concurrent TimeUnit) (org.jruby CompatVersion) (org.jruby.util.cli OutputStrings))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Constants (def default-jruby-compile-mode "Default for JRuby's CompileMode setting. Defaults to JIT for Jruby 9k." :jit) (def default-borrow-timeout "Default timeout when borrowing instances from the JRuby pool in milliseconds. Current value is 1200000ms, or 20 minutes." 1200000) (def default-flush-timeout "Default timeout when flushing the JRuby pool in milliseconds. Current value is 1200000ms, or 20 minutes." 1200000) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Functions (defn default-pool-size "Calculate the default size of the JRuby pool, based on the number of cpus." [num-cpus] (->> (- num-cpus 1) (max 1) (min 4))) (schema/defn ^:always-validate get-pool-state :- jruby-schemas/PoolState "Gets the PoolState from the pool context." [context :- jruby-schemas/PoolContext] (jruby-internal/get-pool-state context)) (schema/defn ^:always-validate get-pool :- jruby-schemas/pool-queue-type "Gets the JRuby pool object from the pool context." [context :- jruby-schemas/PoolContext] (jruby-internal/get-pool context)) (schema/defn ^:always-validate registered-instances :- [JRubyInstance] [context :- jruby-schemas/PoolContext] (-> (get-pool context) .getRegisteredElements .iterator iterator-seq vec)) (schema/defn ^:always-validate free-instance-count "Returns the number of JRubyInstances available in the pool." [pool :- jruby-schemas/pool-queue-type] {:post [(>= % 0)]} (.currentSize pool)) (schema/defn ^:always-validate get-instance-state :- jruby-schemas/JRubyInstanceState "Get the state metadata for a JRubyInstance." [jruby-instance :- JRubyInstance] @(jruby-internal/get-instance-state-container jruby-instance)) (schema/defn get-event-callbacks :- [IFn] "Gets the vector of event callbacks from the pool context." [pool-context :- jruby-schemas/PoolContext] @(get-in pool-context [:internal :event-callbacks])) (schema/defn get-system-env :- jruby-schemas/EnvPersistentMap "Same as System/getenv, but returns a clojure persistent map instead of a Java unmodifiable map." [] (into {} (System/getenv))) (schema/defn ^:always-validate add-gem-path [env :- {schema/Str schema/Str} config :- jruby-schemas/JRubyConfig] (if-let [gem-path (:gem-path config)] (assoc env "GEM_PATH" gem-path) env)) (def proxy-vars-allowed-list "A list of proxy-related variables that are allowed to be passed the environment" ["HTTP_PROXY" "http_proxy" "HTTPS_PROXY" "https_proxy" "NO_PROXY" "no_proxy"]) (def env-vars-allowed-list "A list of environment variables that are allowed to be passed through from the environment." (concat proxy-vars-allowed-list ["HOME" "PATH"])) (schema/defn ^:always-validate managed-environment :- jruby-schemas/EnvMap "The environment variables that should be passed to the JRuby interpreters. We don't want them to read any ruby environment variables, like $RUBY_LIB or anything like that, so pass it an empty environment map - except - most things needs HOME and PATH to work, so leave those, along with GEM_HOME and GEM_PATH, which are necessary for extensions that depend on gems. We need to set the JARS..REQUIRE variables in order to instruct JRuby's 'jar-dependencies' to not try to load any dependent jars. This is being done specifically to avoid JRuby trying to load its own version of Bouncy Castle, which may not the same as the one that 'puppetlabs/ssl-utils' uses. JARS_NO_REQUIRE was the legacy way to turn off jar loading but is being phased out in favor of JARS_REQUIRE. As of JRuby 1.7.20, only JARS_NO_REQUIRE is honored. Setting both of those here for forward compatibility. We also merge an environment-vars map with the config to allow for configured environment variables to be visible to the Ruby code. This map is by default set to {} if the user does not specify it in the configuration file." [env :- jruby-schemas/EnvMap config :- jruby-schemas/JRubyConfig] (let [clean-env (select-keys env env-vars-allowed-list)] (merge (-> (assoc clean-env "GEM_HOME" (:gem-home config) "JARS_NO_REQUIRE" "true" "JARS_REQUIRE" "false") (add-gem-path config)) (clojure.walk/stringify-keys (:environment-vars config))))) (schema/defn ^:always-validate default-initialize-scripting-container :- jruby-schemas/ConfigurableJRuby "Default lifecycle fn for initializing the settings on the scripting container. Currently it just sets the environment variables." [scripting-container :- jruby-schemas/ConfigurableJRuby config :- jruby-schemas/JRubyConfig] (.setEnvironment scripting-container (managed-environment (get-system-env) config)) scripting-container) (schema/defn ^:always-validate initialize-lifecycle-fns :- jruby-schemas/LifecycleFns [config :- (schema/maybe {(schema/optional-key :initialize-pool-instance) IFn (schema/optional-key :cleanup) IFn (schema/optional-key :shutdown-on-error) IFn (schema/optional-key :initialize-scripting-container) IFn})] (-> config (update-in [:initialize-pool-instance] #(or % identity)) (update-in [:cleanup] #(or % identity)) (update-in [:shutdown-on-error] #(or % (fn [f] (f)))) (update-in [:initialize-scripting-container] #(or % default-initialize-scripting-container)))) (schema/defn ^:always-validate initialize-config :- jruby-schemas/JRubyConfig "Initialize keys with default settings if they are not given a value. The config is validated after these defaults are set." [config :- {schema/Keyword schema/Any}] (-> config (update-in [:compile-mode] #(keyword (or % default-jruby-compile-mode))) (update-in [:profiling-mode] #(keyword (or % :off))) (update-in [:profiler-output-file] #(or % (str (fs/absolute (fs/temp-name "jruby-profiler"))))) (update-in [:borrow-timeout] #(or % default-borrow-timeout)) (update-in [:flush-timeout] #(or % default-flush-timeout)) (update-in [:max-active-instances] #(or % (default-pool-size (ks/num-cpus)))) (update-in [:max-borrows-per-instance] #(or % 0)) (update-in [:splay-instance-flush] #(if (nil? %) true %)) (update-in [:environment-vars] #(or % {})) (update-in [:lifecycle] initialize-lifecycle-fns) (update-in [:multithreaded] #(if (nil? %) false %)) (update-in [:instance-creation-concurrency] #(if (nil? %) 3 %)) jruby-internal/initialize-gem-path)) (schema/defn register-event-handler "Register the callback function by adding it to the event callbacks atom on the pool context." [pool-context :- jruby-schemas/PoolContext callback-fn :- IFn] (swap! (get-in pool-context [:internal :event-callbacks]) conj callback-fn)) (schema/defn ^:always-validate get-jruby-thread-dump "Get thread dumps from JRuby instances in the pool." [pool-context :- jruby-schemas/PoolContext] (reduce (fn [result instance] (assoc result (:id instance) (jruby-internal/get-instance-thread-dump instance))) {} (registered-instances pool-context))) (schema/defn ^:always-validate borrow-from-pool :- jruby-schemas/JRubyInstanceOrPill "Borrows a JRuby interpreter from the pool. If there are no instances left in the pool then this function will block until there is one available." [pool-context :- jruby-schemas/PoolContext reason :- schema/Any event-callbacks :- [IFn]] (let [requested-event (jruby-events/instance-requested event-callbacks reason) [instance worker-id] (pool-protocol/borrow pool-context)] (jruby-events/instance-borrowed event-callbacks requested-event instance worker-id) instance)) ;; TODO: consider adding a second arity that allows for passing in a ;; borrow-timeout, rather than relying on what is in the config. (schema/defn ^:always-validate borrow-from-pool-with-timeout :- jruby-schemas/JRubyBorrowResult "Borrows a JRuby interpreter from the pool, like borrow-from-pool but a blocking timeout is taken from the config in the context. If an instance is available then it will be immediately returned to the caller, if not then this function will block waiting for an instance to be free for the number of milliseconds given in timeout. If the timeout runs out then nil will be returned, indicating that there were no instances available." [pool-context :- jruby-schemas/PoolContext reason :- schema/Any event-callbacks :- [IFn]] (let [timeout (get-in pool-context [:config :borrow-timeout]) requested-event (jruby-events/instance-requested event-callbacks reason) [instance worker-id] (pool-protocol/borrow-with-timeout pool-context timeout)] (jruby-events/instance-borrowed event-callbacks requested-event instance worker-id) instance)) (schema/defn ^:always-validate return-to-pool "Return a borrowed pool instance to its free pool." [pool-context :- jruby-schemas/PoolContext instance :- jruby-schemas/JRubyInstanceOrPill reason :- schema/Any event-callbacks :- [IFn]] (let [worker-id (pool-protocol/worker-id pool-context instance)] (jruby-events/instance-returned event-callbacks instance reason worker-id) (pool-protocol/return pool-context instance))) (schema/defn ^:always-validate flush-pool! "Flush all the current JRubyInstances and repopulate the pool." [pool-context] (pool-protocol/flush-pool pool-context)) (schema/defn ^:always-validate flush-pool-for-shutdown! "Flush all the current JRubyInstances so that the pool can be shutdown without any instances being active." [pool-context] (pool-protocol/shutdown pool-context)) (schema/defn ^:always-validate lock-pool "Locks the JRuby pool for exclusive access." [pool-context :- jruby-schemas/PoolContext reason :- schema/Any event-callbacks :- [IFn]] (log/info (i18n/trs "Acquiring lock on JRubyPool...")) (jruby-events/lock-requested event-callbacks reason) (pool-protocol/lock pool-context) (jruby-events/lock-acquired event-callbacks reason) (log/info (i18n/trs "Lock acquired"))) (schema/defn ^:always-validate lock-pool-with-timeout "Locks the JRuby pool for exclusive access using a timeout in milliseconds. If the timeout is exceeded, a TimeoutException will be thrown and the pool will remain unlocked" [pool-context :- jruby-schemas/PoolContext timeout-ms :- schema/Int reason :- schema/Any event-callbacks :- [IFn]] (log/info (i18n/trs "Acquiring lock on JRubyPool...")) (jruby-events/lock-requested event-callbacks reason) (pool-protocol/lock-with-timeout pool-context timeout-ms TimeUnit/MILLISECONDS) (jruby-events/lock-acquired event-callbacks reason) (log/info (i18n/trs "Lock acquired"))) (schema/defn ^:always-validate unlock-pool "Unlocks the JRuby pool, restoring concurernt access." [pool-context :- jruby-schemas/PoolContext reason :- schema/Any event-callbacks :- [IFn]] (pool-protocol/unlock pool-context) (jruby-events/lock-released event-callbacks reason) (log/info (i18n/trs "Lock on JRubyPool released"))) (schema/defn ^:always-validate cli-ruby! :- jruby-schemas/JRubyMainStatus "Run JRuby as though native `ruby` were invoked with args on the CLI" [config :- jruby-schemas/JRubyConfig args :- [schema/Str]] (let [main (jruby-internal/new-main config) argv (into-array String (concat ["-rjar-dependencies"] args))] (.run main argv))) (schema/defn ^:always-validate cli-run! :- (schema/maybe jruby-schemas/JRubyMainStatus) "Run a JRuby CLI command, e.g. gem, irb, etc..." [config :- jruby-schemas/JRubyConfig command :- schema/Str args :- [schema/Str]] (let [bin-dir "META-INF/jruby.home/bin" load-path (format "%s/%s" bin-dir command) url (io/resource load-path (.getClassLoader org.jruby.Main))] (if url (cli-ruby! config (concat ["-e" (format "load '%s'" url) "--"] args)) (log/error (i18n/trs "command {0} could not be found in {1}" command bin-dir))))) (defmacro with-jruby-instance "Encapsulates the behavior of borrowing and returning a JRubyInstance. Example usage: (let [pool-manager-service (tk-app/get-service app :PoolManagerService) pool-context (pool-manager/create-pool pool-manager-service config)] (with-jruby-instance jruby-instance pool-context reason (do-something-with-a-jruby-instance jruby-instance))) Will throw an IllegalStateException if borrowing a JRubyInstance times out." [jruby-instance pool-context reason & body] `(let [event-callbacks# (get-event-callbacks ~pool-context)] (loop [pool-instance# (borrow-from-pool-with-timeout ~pool-context ~reason event-callbacks#)] (if (nil? pool-instance#) (sling/throw+ {:kind ::jruby-timeout :msg (i18n/tru "Attempt to borrow a JRubyInstance from the pool timed out.")})) (when (jruby-schemas/shutdown-poison-pill? pool-instance#) (return-to-pool ~pool-context pool-instance# ~reason event-callbacks#) (ringutils/throw-service-unavailable! (format "%s %s" (i18n/tru "Attempted to borrow a JRubyInstance from the pool during a shutdown.") (i18n/tru "Please try again.")))) (let [~jruby-instance pool-instance#] (try ~@body (finally (return-to-pool ~pool-context pool-instance# ~reason event-callbacks#))))))) (defmacro with-lock "Acquires a lock on the pool, executes the body, and releases the lock." [pool-context reason & body] `(let [event-callbacks# (get-event-callbacks ~pool-context)] (lock-pool ~pool-context ~reason event-callbacks#) (try ~@body (finally (unlock-pool ~pool-context ~reason event-callbacks#))))) (defmacro with-lock-with-timeout "Acquires a lock on the pool with a timeout in milliseconds, executes the body, and releases the lock. If the timeout is exceeded, a TimeoutException will be thrown" [pool-context timeout-ms reason & body] `(let [event-callbacks# (get-event-callbacks ~pool-context)] (lock-pool-with-timeout ~pool-context ~timeout-ms ~reason event-callbacks#) (try ~@body (finally (unlock-pool ~pool-context ~reason event-callbacks#))))) (def jruby-version-info "Default version info string for jruby" (OutputStrings/getVersionString)) jruby_pool_manager_service.clj000066400000000000000000000013011464330533300361000ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service (:require [puppetlabs.trapperkeeper.core :as trapperkeeper] [puppetlabs.services.protocols.pool-manager :as pool-manager-protocol] [puppetlabs.services.jruby-pool-manager.impl.jruby-pool-manager-core :as jruby-pool-manager-core] [clojure.tools.logging :as log] [puppetlabs.i18n.core :as i18n])) (trapperkeeper/defservice jruby-pool-manager-service pool-manager-protocol/PoolManagerService [] (create-pool [this config] (log/info (i18n/trs "Initializing the JRuby service")) (jruby-pool-manager-core/create-pool config))) puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/jruby_pool_manager/jruby_schemas.clj000066400000000000000000000231521464330533300334270ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.jruby-schemas (:require [schema.core :as schema]) (:import (clojure.lang Atom Agent IFn PersistentArrayMap PersistentHashMap) (com.puppetlabs.jruby_utils.jruby ScriptingContainer) (com.puppetlabs.jruby_utils.pool LockablePool) (java.util.concurrent ExecutorService) (org.jruby Main Main$Status RubyInstanceConfig))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Schemas (def pool-queue-type "The Java datastructure type used to store JRubyInstances which are free to be borrowed." LockablePool) (defrecord PoisonPill ;; A sentinel object to put into a pool in case an error occurs while we're trying ;; to populate it. This can be used by the `borrow` functions to detect error ;; state in a thread-safe manner. [err]) (defrecord ShutdownPoisonPill ;; A sentinel object to put into a pool when we are shutting down. ;; This can be used to build `borrow` functionality that will detect the ;; case where we're trying to borrow during a shutdown, so we can return ;; a sane error code. [pool]) (def supported-jruby-compile-modes #{:jit :force :off}) (def SupportedJRubyCompileModes "Schema defining the supported values for the JRuby CompileMode setting." (apply schema/enum supported-jruby-compile-modes)) (def SupportedJRubyProfilingModes "Schema defining the supported values for the JRuby ProfilingMode setting." (schema/enum :api :flat :graph :html :json :off :service)) (def LifecycleFns {:initialize-pool-instance IFn :cleanup IFn :shutdown-on-error IFn :initialize-scripting-container IFn}) (def JRubyConfig "Schema defining the config map for the JRuby pooling functions. The keys should have the following values: * :ruby-load-path - a vector of file paths, containing the locations of ruby source code. * :gem-home - The location that JRuby gems will be installed * :gem-path - The full path where JRuby should look for gems * :compile-mode - The value to use for JRuby's CompileMode setting. Legal values are `:jit`, `:force`, and `:off`. Defaults to `:off`. * :max-active-instances - The maximum number of JRubyInstances that will be pooled. * :splay-instance-flush - Whether or not to splay flushing of instances * :environment-vars - A map of environment variables and their values to be passed through to the JRuby scripting container and visible to any Ruby code. * :profiling-mode - The value to use for JRuby's ProfilerMode setting. Legal values are `:api`, `:flat`, `:graph`, `:html`, `:json`, `:off`, and `:service`. Defaults to `:off`. * :profiler-output-file - A target file to direct profiler output to. If not set, defaults to a random file relative to the working directory of the service. * :multithreaded - Instead of managing the number of JRuby Instances create a single JRuby instance and manage the number of threads that may access it. * :instance-creation-concurrency - How many instances to create at once. This will improve start up and potentially reload times, but if too high may create unaceptable load on the system during startup or reload." {:ruby-load-path [schema/Str] :gem-home schema/Str :gem-path (schema/maybe schema/Str) :compile-mode SupportedJRubyCompileModes :borrow-timeout schema/Int :flush-timeout schema/Int :max-active-instances schema/Int :max-borrows-per-instance schema/Int :splay-instance-flush schema/Bool :lifecycle LifecycleFns :environment-vars {schema/Keyword schema/Str} :profiling-mode SupportedJRubyProfilingModes :profiler-output-file schema/Str :multithreaded schema/Bool :instance-creation-concurrency schema/Int}) (def JRubyPoolAgent "An agent configured for use in managing JRuby pools" (schema/both Agent (schema/pred (fn [a] (let [state @a] (and (map? state) (ifn? (:shutdown-on-error state)))))))) (def PoolState "A map that describes all attributes of a particular JRuby pool." {:pool pool-queue-type :size schema/Int :creation-service ExecutorService}) (def PoolStateContainer "An atom containing the current state of all of the JRuby pool." (schema/pred #(and (instance? Atom %) (nil? (schema/check PoolState @%))) 'PoolStateContainer)) (def PoolContextInternal "The data structure that stores all JRuby pools" {:modify-instance-agent JRubyPoolAgent :pool-state PoolStateContainer :event-callbacks Atom}) (schema/defrecord ReferencePool [config :- JRubyConfig internal :- PoolContextInternal borrow-count :- Atom]) (schema/defrecord InstancePool [config :- JRubyConfig internal :- PoolContextInternal]) (def PoolContext (schema/pred #(or (instance? ReferencePool %) (instance? InstancePool %)))) (def JRubyInstanceState "State metadata for an individual JRubyInstance" {:borrow-count schema/Int}) (def JRubyInstanceStateContainer "An atom containing the current state of a given JRubyInstance." (schema/pred #(and (instance? Atom %) (nil? (schema/check JRubyInstanceState @%))) 'JRubyInstanceState)) (def JRubyPuppetInstanceInternal {:pool pool-queue-type :initial-borrows (schema/maybe schema/Int) :max-borrows schema/Int :state JRubyInstanceStateContainer}) (schema/defrecord JRubyInstance [internal :- JRubyPuppetInstanceInternal id :- schema/Int scripting-container :- ScriptingContainer] {schema/Keyword schema/Any}) (defn jruby-instance? [x] (instance? JRubyInstance x)) (defn jruby-main-instance? [x] (instance? Main x)) (defn jruby-main-status-instance? [x] (instance? Main$Status x)) (defn jruby-scripting-container? [x] (instance? ScriptingContainer x)) (defn jruby-instance-config? [x] (instance? RubyInstanceConfig x)) (defn poison-pill? [x] (instance? PoisonPill x)) (defn shutdown-poison-pill? [x] (instance? ShutdownPoisonPill x)) (def JRubyInstanceOrPill (schema/conditional jruby-instance? (schema/pred jruby-instance?) shutdown-poison-pill? (schema/pred shutdown-poison-pill?))) (def JRubyInternalBorrowResult ;; Result of calling `.borrowItem` on the pool (schema/pred (some-fn nil? poison-pill? shutdown-poison-pill? jruby-instance?))) (def JRubyBorrowResult ;; Result of doing some error handling after calling `.borrowItem` on the ;; pool. Specifically, if the item borrow was a poison pill, an error is ;; thrown, so `poison-pill?` is not part of this schema. (schema/pred (some-fn nil? shutdown-poison-pill? jruby-instance?))) (def JRubyWorkerId (schema/pred (some-fn nil? (partial instance? Long)))) (def JRubyMain (schema/pred jruby-main-instance?)) (def JRubyMainStatus (schema/pred jruby-main-status-instance?)) (def ConfigurableJRuby ;; This schema is a bit weird. We have some common configuration that we need ;; to apply to two different kinds of JRuby objects: `ScriptingContainer` and ;; `JRubyInstanceConfig`. These classes both have the same signatures for ;; all of the setter methods that we need to call on them (see ;; `jruby-internal/init-jruby-config`), but unfortunately the JRuby API doesn't ;; define an interface for those methods. So, rather than duplicating the logic ;; in multiple places in the code, we use this (gross) schema to enforce that ;; an object must be an instance of one of those two types. (schema/conditional jruby-scripting-container? (schema/pred jruby-scripting-container?) jruby-instance-config? (schema/pred jruby-instance-config?))) (def EnvMap "System Environment variables have strings for the keys and values of a map" {schema/Str schema/Str}) (def EnvPersistentMap "Schema for a clojure persistent map for the system environment" (schema/both EnvMap (schema/either PersistentArrayMap PersistentHashMap))) (defn event-type-requested? [e] (= :instance-requested (:type e))) (defn event-type-borrowed? [e] (= :instance-borrowed (:type e))) (defn event-type-returned? [e] (= :instance-returned (:type e))) (defn event-type-lock-requested? [e] (= :lock-requested (:type e))) (defn event-type-lock-acquired? [e] (= :lock-acquired (:type e))) (defn event-type-lock-released? [e] (= :lock-released (:type e))) (def JRubyEventReason schema/Any) (def JRubyRequestedEvent {:type (schema/eq :instance-requested) :reason JRubyEventReason}) (def JRubyBorrowedEvent {:type (schema/eq :instance-borrowed) :reason JRubyEventReason :requested-event JRubyRequestedEvent :instance JRubyBorrowResult :worker-id JRubyWorkerId}) (def JRubyReturnedEvent {:type (schema/eq :instance-returned) :reason JRubyEventReason :instance JRubyInstanceOrPill :worker-id JRubyWorkerId}) (def JRubyLockRequestedEvent {:type (schema/eq :lock-requested) :reason JRubyEventReason}) (def JRubyLockAcquiredEvent {:type (schema/eq :lock-acquired) :reason JRubyEventReason}) (def JRubyLockReleasedEvent {:type (schema/eq :lock-released) :reason JRubyEventReason}) (def JRubyEvent (schema/conditional event-type-requested? JRubyRequestedEvent event-type-borrowed? JRubyBorrowedEvent event-type-returned? JRubyReturnedEvent event-type-lock-requested? JRubyLockRequestedEvent event-type-lock-acquired? JRubyLockAcquiredEvent event-type-lock-released? JRubyLockReleasedEvent)) puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/protocols/000077500000000000000000000000001464330533300262425ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/protocols/jruby_pool.clj000066400000000000000000000041461464330533300311250ustar00rootroot00000000000000(ns puppetlabs.services.protocols.jruby-pool) (defprotocol JRubyPool (fill [pool-context] "Creates all the necessary JRuby instances and adds them to the pool.") (shutdown [pool-context] "Shuts down the JRuby pool, inserting a poison pill to prevent further borrows and terminating all JRuby instances.") (lock [pool-context] "Blocks waiting for all currently held JRubies to be returned to the pool, preventing further borrows until the pool is unlocked.") (lock-with-timeout [pool-context timeout time-unit] "Attempts to lock the JRuby pool, timing out if the supplied interval has elapsed.") (unlock [pool-context] "Unlocks the JRuby pool, allowing borrows to proceed.") (worker-id [pool-context instance] "Returns the worker id for given instance (instance id or thread id).") (borrow [pool-context] "Returns a reference to a JRuby instance and a worker id (instance id or thread id). Will block if the pool is locked or no instances are available.") (borrow-with-timeout [pool-context timeout] "Returns a reference to a JRuby instance and a worker id (instance id or thread id). Will block if the pool is locked or no instances are available, timing out when the supplied number of milliseconds has elapsed.") (return [pool-context instance] "Releases a held reference to a JRuby instance back to the pool and returns the worker id (instance id or thread id) for the thing being returned. If `max-requests-per-instance` is configured and has been reached for this instance, this function will trigger a flush of the instance. Note that when using the ReferencePool, this will also cause the pool to be locked. If something besides a JRuby instance is passed to return (e.g. a Pill), this function is a no-op.") (flush-pool [pool-context] "Removes and terminates all the JRuby instances from the pool, then creates new ones and adds them to the pool. Note that when using the ReferencePool, this will cause the pool to be locked, with a timeout equal to the configured `flush-timeout`.")) puppetlabs-jruby-utils-ca5d27b/src/clj/puppetlabs/services/protocols/pool_manager.clj000066400000000000000000000003341464330533300313770ustar00rootroot00000000000000(ns puppetlabs.services.protocols.pool-manager) (defprotocol PoolManagerService (create-pool [this config] "Create a pool and fill it with the number of JRubyInstances specified. Return the pool context.")) puppetlabs-jruby-utils-ca5d27b/src/java/000077500000000000000000000000001464330533300203655ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/java/com/000077500000000000000000000000001464330533300211435ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/000077500000000000000000000000001464330533300233225ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/000077500000000000000000000000001464330533300256755ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/jruby/000077500000000000000000000000001464330533300270305ustar00rootroot00000000000000InternalScriptingContainer.java000066400000000000000000000033241464330533300351200ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/jrubypackage com.puppetlabs.jruby_utils.jruby; import org.jruby.embed.LocalContextScope; import org.jruby.embed.LocalVariableBehavior; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An extension of the JRuby ScriptingContainer class which is * slightly easier to use from Clojure. */ public class InternalScriptingContainer extends org.jruby.embed.ScriptingContainer implements ScriptingContainer { private static final Logger LOGGER = LoggerFactory.getLogger( InternalScriptingContainer.class); public InternalScriptingContainer(LocalContextScope scope) { super(scope); } public InternalScriptingContainer(LocalContextScope scope, LocalVariableBehavior behavior) { super(scope, behavior); } /** * This method delegates to a specific signature of #callMethod from the * parent class. There are many overloaded signatures in the parent class, * many of which have overlapping arities. This sometimes causes problems * with Clojure attempting to determine the correct signature to call. * * @param receiver - the Ruby object to call a method on * @param methodName - the name of the method to call * @param args - an array of args to call the method with * @param returnType - the expected type of the return value from the method call * @return - the result of calling the method on the Ruby receiver object */ public Object callMethodWithArgArray(Object receiver, String methodName, Object[] args, Class returnType) { return callMethod(receiver, methodName, args, returnType); } } puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/jruby/ScriptingContainer.java000066400000000000000000000010221464330533300334730ustar00rootroot00000000000000package com.puppetlabs.jruby_utils.jruby; import org.jruby.embed.EmbedEvalUnit; import org.jruby.embed.EmbedRubyInstanceConfigAdapter; import org.jruby.runtime.Block; /** */ public interface ScriptingContainer extends EmbedRubyInstanceConfigAdapter { Object callMethodWithArgArray(Object receiver, String methodName, Object[] args, Class returnType); Object runScriptlet(String script); void terminate(); } puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/jruby/Slf4jLogger.java000066400000000000000000000036461464330533300320260ustar00rootroot00000000000000package com.puppetlabs.jruby_utils.jruby; import org.jruby.util.log.Logger; import org.slf4j.LoggerFactory; public class Slf4jLogger implements Logger { private final org.slf4j.Logger logger; public Slf4jLogger(String loggerName) { logger = LoggerFactory.getLogger("jruby." + loggerName); } @Override public String getName() { return logger.getName(); } @Override public void warn(String message, Object... args) { logger.warn(message, args); } @Override public void warn(Throwable throwable) { logger.warn("", throwable); } @Override public void warn(String message, Throwable throwable) { logger.warn(message, throwable); } @Override public void error(String message, Object... args) { logger.error(message, args); } @Override public void error(Throwable throwable) { logger.error("", throwable); } @Override public void error(String message, Throwable throwable) { logger.error(message, throwable); } @Override public void info(String message, Object... args) { logger.info(message, args); } @Override public void info(Throwable throwable) { logger.info("", throwable); } @Override public void info(String message, Throwable throwable) { logger.info(message, throwable); } @Override public void debug(String message, Object... args) { logger.debug(message, args); } @Override public void debug(Throwable throwable) { logger.debug("", throwable); } @Override public void debug(String message, Throwable throwable) { logger.debug(message, throwable); } @Override public boolean isDebugEnabled() { return true; } @Override public void setDebugEnable(boolean b) { warn("setDebugEnable not implemented", null, null); } } puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/pool/000077500000000000000000000000001464330533300266465ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/pool/JRubyPool.java000066400000000000000000000472631464330533300314120ustar00rootroot00000000000000package com.puppetlabs.jruby_utils.pool; import java.util.Deque; import java.util.LinkedList; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; /** * An implementation of LockablePool for managing a pool of JRubyInstances. * * @param the type of element that can be added to the pool. */ public final class JRubyPool implements LockablePool { // The `LockingPool` contract requires some synchronization behaviors that // are not natively present in any of the JDK deque implementations - // specifically to allow one calling thread to call lock() to supercede // and hold off any pending pool borrowers until unlock() is called. // This class implementation fulfills the contract by managing the // synchronization constructs directly rather than deferring to an // underlying JDK data structure to manage concurrent access. // // This implementation is modeled somewhat off of what the // `LinkedBlockingDeque` class in the OpenJDK does to manage // concurrency. It uses a single `ReentrantLock` to provide mutual // exclusion around offer and take requests, with condition variables // used to park and later reawaken requests as needed, e.g., when pool // items are unavailable for borrowing or when the pool lock is // unavailable. // // See http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/LinkedBlockingDeque.java#l157 // // Because access to the underlying deque is synchronized within // this class, the pool is backed by a non-synchronized JDK `LinkedList`. // Underlying queue which holds the elements that clients can borrow. private final Deque liveQueue; // Lock which guards all accesses to the underlying queue and registered // element set. Constructed as "nonfair" for performance, like the // lock that a `LinkedBlockingDeque` does. Not clear that we need this // to be a "fair" lock. private final ReentrantLock queueLock = new ReentrantLock(false); // Condition signaled when all elements that have been registered have been // returned to the queue or if a pill has been inserted. Awaited when a // lock has been requested but one or more registered elements has been // borrowed from the pool. private final Condition lockAvailable = queueLock.newCondition(); // Condition signaled when an element has been added into the queue. // Awaited when a request has been made to borrow an item but no elements // currently exist in the queue. private final Condition queueNotEmpty = queueLock.newCondition(); // Condition signaled when the pool has been unlocked. Awaited when a // request has been made to borrow an item or lock the pool but the pool // is currently locked. private final Condition poolNotLocked = queueLock.newCondition(); // Holds a reference to all of the elements that have been registered. // Newly registered elements are also added into the `liveQueue`. // Elements only exist in the `liveQueue` when not currently // borrowed whereas elements that have been registered (but not // yet unregistered) will be accessible via `registeredElements` // even while they are borrowed. private final Set registeredElements = new CopyOnWriteArraySet<>(); // Maximum size that the underlying queue can grow to. private int maxSize; // Thread which currently holds the pool lock. null indicates that // there is no current pool lock holder. Using the current Thread // object for tracking the pool lock owner is comparable to what the JDK's // `ReentrantLock` class does via the `AbstractOwnableSynchronizer` class: // // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/locks/ReentrantLock.java#l164 // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/locks/AbstractOwnableSynchronizer.java#l64 // // Unlike the `AbstractOwnableSynchronizer` class implementation, we marked // this variable as `volatile` because we couldn't convince ourselves // that it would be safe to update this variable from different threads and // not be susceptible to per-thread / per-CPU caching causing the wrong // value to be seen by a thread. `volatile` seems safer and doesn't appear // to impose any noticeable performance degradation. private volatile Thread poolLockThread = null; // Holds a poison pill object for errors and shutdowns // If not null, takes priority over any pool instance when a call to // borrowItem is made. Returns made using releaseItem are ignored if the // released item is the poison pill already stored here private volatile E pill; /** * Create a JRubyPool * * @param size maximum capacity for the pool. */ public JRubyPool(int size) { liveQueue = new LinkedList<>(); maxSize = size; } @Override public void register(E e) { final ReentrantLock lock = this.queueLock; lock.lock(); try { if (registeredElements.size() == maxSize) throw new IllegalStateException( "Unable to register additional instance, pool full"); registeredElements.add(e); liveQueue.addLast(e); signalPoolNotEmpty(); } finally { lock.unlock(); } } @Override public void unregister(E e) { final ReentrantLock lock = this.queueLock; lock.lock(); try { registeredElements.remove(e); signalIfLockCanProceed(); } finally { lock.unlock(); } } @Override public E borrowItem() throws InterruptedException { E item = null; final ReentrantLock lock = this.queueLock; lock.lock(); try { final Thread currentThread = Thread.currentThread(); do { if (this.pill != null) { // Return the pill immediately if there is one item = pill; } else if (isPoolLockHeldByAnotherThread(currentThread)) { poolNotLocked.await(); } else if (liveQueue.size() < 1) { queueNotEmpty.await(); } else { item = liveQueue.removeFirst(); } } while (item == null); } finally { lock.unlock(); } return item; } @Override public E borrowItemWithTimeout(long timeout, TimeUnit unit) throws InterruptedException { E item = null; final ReentrantLock lock = this.queueLock; long remainingMaxTimeToWait = unit.toNanos(timeout); // `queueLock.lockInterruptibly()` is called here as opposed to just // `queueLock.queueLock` to follow the pattern that the JDK's // `LinkedBlockingDeque` does for a timed poll from a deque. See: // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/LinkedBlockingDeque.java#l516 lock.lockInterruptibly(); try { final Thread currentThread = Thread.currentThread(); // This pattern of using timed `awaitNanos` on a condition // variable to track the total time spent waiting for an item to // be available to be borrowed follows the logic that the JDK's // `LinkedBlockingDeque` in `pollFirst` uses. See: // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/LinkedBlockingDeque.java#l522 do { if (this.pill != null) { // Return the pill immediately if there is one item = pill; } else if (isPoolLockHeldByAnotherThread(currentThread)) { if (remainingMaxTimeToWait <= 0) { break; } remainingMaxTimeToWait = poolNotLocked.awaitNanos(remainingMaxTimeToWait); } else if (liveQueue.size() < 1) { if (remainingMaxTimeToWait <= 0) { break; } remainingMaxTimeToWait = queueNotEmpty.awaitNanos(remainingMaxTimeToWait); } else { item = liveQueue.removeFirst(); } } while (item == null); } finally { lock.unlock(); } return item; } /** * Release an item and return it to the pool. Does nothing if the item * being released is the pill. * Throws an `IllegalArgumentException` if the item is not currently * registered by the pool and the item is not the pill, if one has been * inserted */ @Override public void releaseItem(E e) { final ReentrantLock lock = this.queueLock; lock.lock(); try { if (e != this.pill) { if (!isRegistered(e)) { String errorMsg = "The item being released is not registered with the pool"; throw new IllegalArgumentException(errorMsg); } addFirst(e); } } finally { lock.unlock(); } } private boolean isRegistered(E e) { return this.registeredElements.contains(e); } /** * Insert a poison pill into the pool. It should only ever be used to * insert a `PoisonPill` or `ShutdownPoisonPill` to the pool. Only the * first call will insert a pill. Subsequent insertions will be ignored */ @Override public void insertPill(E e) { final ReentrantLock lock = this.queueLock; lock.lock(); try { if (this.pill == null) { this.pill = e; signalPoolNotEmpty(); } } finally { lock.unlock(); } } @Override public void clear() { final ReentrantLock lock = this.queueLock; lock.lock(); try { // It would be simpler to just call .clear() on both the liveQueue // and registeredElements here. It is possible, however, that this // method might be called while one or more elements are being // borrowed from the liveQueue. If the associated element from // registeredElements were removed, it would then be possible for // the borrowed elements to be returned to the pool, making them // appear in liveQueue but not in registeredElements. This would // be bad because any subsequent actions that need to be done to // all members of the pool - for example, marking environments in // the pool instance as expired - might inadvertently skip over // any of the elements that are no longer in registeredElements // but can appear in liveQueue. // // To avoid this problem, the implementation only removes elements // from registeredElements which have a corresponding entry which // is being removed from the liveQueue. int queueSize = liveQueue.size(); for (int i=0; i getRegisteredElements() { return registeredElements; } private void addFirst(E e) { liveQueue.addFirst(e); signalPoolNotEmpty(); } private void freePoolLock() { poolLockThread = null; // Need to use 'signalAll' here because there might be multiple // waiters (e.g., multiple borrowers) queued up, waiting for the // pool to be unlocked. poolNotLocked.signalAll(); // Borrowers that are woken up when an instance is returned to the // pool and the pool queueLock is held would then start waiting on a // 'poolNotLocked' signal instead. Re-signalling 'queueNotEmpty' here // allows any borrowers still waiting on the 'queueNotEmpty' signal to be // reawoken when the pool lock is released, compensating for any // 'queueNotEmpty' signals that might have been essentially ignored from // when the pool lock was held. if (liveQueue.size() > 0) { queueNotEmpty.signalAll(); } } /** * Should be called if the pool is no longer empty (or a pill is inserted), * so that threads waiting for pool instances can be woken up */ private void signalPoolNotEmpty() { // Could use 'signalAll' here instead of 'signal' but 'signal' is // less expensive in that only one waiter will be woken up. Can use // signal here because the thread being awoken will be able to borrow // a pool instance and any further waiters will be woken up by // subsequent posts of this signal when instances are added/returned to // the queue. queueNotEmpty.signal(); signalIfLockCanProceed(); } /** * Checks if threads waiting on the pool lock should be woken up. * This will wake them up if either the number of available instances is * equal to the maximum size of the pool, or if a pill has been inserted */ private void signalIfLockCanProceed() { // Could use 'signalAll' here instead of 'signal'. Doesn't really // matter though in that there will only be one waiter at most which // is active at a time - a caller of lock() that has just acquired // the pool lock but is waiting for the live queue to be completely // filled if (this.maxSize == liveQueue.size() || pill != null) { lockAvailable.signal(); } } private boolean isPoolLockHeld() { return poolLockThread != null; } private boolean isPoolLockHeldByCurrentThread(Thread currentThread) { return poolLockThread == currentThread; } private boolean isPoolLockHeldByAnotherThread(Thread currentThread) { return (poolLockThread != null) && (poolLockThread != currentThread); } } puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/pool/LockablePool.java000066400000000000000000000171001464330533300320560ustar00rootroot00000000000000package com.puppetlabs.jruby_utils.pool; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; public interface LockablePool { /** * Introduce a new element to the pool. * * @param e the element to register * @throws IllegalStateException if an attempt is made to register an * element but the number of registered * elements is already equal to the maximum * capacity for the pool */ void register(E e) throws IllegalStateException; /** * Unregister an element from the pool. It is assumed that the * caller of this method has previously called {@link #borrowItem()} to * retrieve the item back from the pool. This method does not * implicitly remove the element from the list of elements available for * {@link #borrowItem()} calls. This method does remove the element * from the set returned from subsequent calls to * {@link #getRegisteredElements()}. * * @param e the element to remove from the list of registered elements * @throws InterruptedException if the calling thread is interrupted while * it waits for all instances to be returned * to the pool */ void unregister(E e) throws InterruptedException; /** * Borrow an element from the pool. This method will block until * whichever of the following happens first: * * - the element can be returned * - the caller's thread is interrupted * * On a successful return, subsequent calls to {@link #borrowItem()} and * {@link #borrowItemWithTimeout(long, TimeUnit)} will not return the same * element returned for the call to this method until/unless a subsequent * call to {@link #releaseItem(Object)} is made to return the element back * to the pool. * * @return the borrowed element * @throws InterruptedException if the calling thread is interrupted * while waiting for the pool to be * unlocked or for an element to * be available in the queue for borrowing * @see #borrowItemWithTimeout(long, TimeUnit) */ E borrowItem() throws InterruptedException; /** * Borrow an element from the pool. This method will block until * whichever of the following happens first: * * - the element can be returned * - the maximum time specified by the timeout parameter has * elapsed * - the caller's thread is interrupted * * On a successful return, subsequent calls to {@link #borrowItem()} and * {@link #borrowItemWithTimeout(long, TimeUnit)} will not return the same * element returned for the call to this method until/unless a subsequent * call to {@link #releaseItem(Object)} is made to return the element * back to the pool. * * @param timeout how long to wait before giving up, in units of unit * @param unit a TimeUnit determining how to interpret the * timeout parameter * @return The borrowed element or null if the specified waiting * time elapses before an element is available * @throws InterruptedException if the calling thread is interrupted * while waiting for the pool to be * unlocked or for an element to * be available in the queue for borrowing * @see #borrowItem() */ E borrowItemWithTimeout(long timeout, TimeUnit unit) throws InterruptedException; /** * Release an item back into the pool. * * @param e the element to return to the pool */ void releaseItem(E e); /** * Insert a poison pill into the pool. * * @param e the pill element to add to the queue */ void insertPill(E e); /** * Unregister all elements which are currently in the pool. Elements * in the pool at the time this method is called would no longer be * returned in subsequent {@link #borrowItem()}, * {@link #borrowItemWithTimeout(long, TimeUnit)}, or * {@link #getRegisteredElements()} calls. Note that any elements that * have been borrowed but not yet returned to the pool at the time this * method is called will remain registered. */ void clear() throws InterruptedException; /** * Returns the number of elements that can be added into the pool. Equal * to the capacity of the pool minus the number of elements that have * been registered with but not borrowed from the pool. */ int remainingCapacity(); /** * Returns the number of elements that have been registered with but not * borrowed from the pool. */ int currentSize(); /** * Lock the pool. This method should make the following guarantees: * * a) blocks until all registered elements are returned to the pool * b) once this method is called (even before it returns), any new threads * that attempt a borrow should block until {@link #unlock()} * is called * c) if there are other threads that were already blocking in a * borrow before this method was called, they should continue * to block until {@link #unlock()} is called * d) elements may be returned by other threads while this method is * being executed * e) when the method returns, the caller holds an exclusive lock on the * pool; the lock should be re-entrant in the sense that this thread * should still be able to perform borrows while it's holding the lock * * @throws InterruptedException if the calling thread is interrupted while * waiting for the pool to be unlocked */ void lock() throws InterruptedException; /** * Lock the pool. Behaves the same as {@link #lock()} but only waits for * the amount of time specified in the timeout parameter. Throws * a TimeoutException if the timeout is exceeded. * * @param timeout how long to wait before giving up, in units of unit * @param unit a TimeUnit determining how to interpret the * timeout parameter */ void lockWithTimeout(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException; /** * Returns whether or not the pool is currently locked. Note that the * value returned may no longer be accurate by the time it is consumed by * the caller since activity on the pool, e.g., locks taken or released, * is not implicitly held off until the caller has a chance to consume * the value returned from a call to this method. */ boolean isLocked(); /** * Release the exclusive pool lock so that other threads may begin to * perform borrow operations again. Note that this method must be called * from the same thread from which {@link #lock} was called in order to * obtain the exclusive pool lock. * * @throws IllegalStateException if the calling thread does not currently * hold the exclusive lock */ void unlock(); /** * Returns a set of all of the elements that are currently registered with * this pool. The set includes both elements that are available to be * borrowed and elements that have already been borrowed. */ Set getRegisteredElements(); } puppetlabs-jruby-utils-ca5d27b/src/java/com/puppetlabs/jruby_utils/pool/ReferencePool.java000066400000000000000000000501751464330533300322510ustar00rootroot00000000000000package com.puppetlabs.jruby_utils.pool; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An implementation of LockablePool for managing a pool of JRubyInstances. * * @param the type of element that can be added to the pool. */ public final class ReferencePool implements LockablePool { private static final Logger LOGGER = LoggerFactory.getLogger( ReferencePool.class); // The `LockingPool` contract requires some synchronization behaviors that // are not natively present in any of the JDK deque implementations - // specifically to allow one calling thread to call lock() to supercede // and hold off any pending pool borrowers until unlock() is called. // This class implementation fulfills the contract by managing the // synchronization constructs directly rather than deferring to an // underlying JDK data structure to manage concurrent access. // // This implementation is modeled somewhat off of what the // `LinkedBlockingDeque` class in the OpenJDK does to manage // concurrency. It uses a single `ReentrantLock` to provide mutual // exclusion around offer and take requests, with condition variables // used to park and later reawaken requests as needed, e.g., when pool // items are unavailable for borrowing or when the pool lock is // unavailable. // // See http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/LinkedBlockingDeque.java#l157 // Lock which guards all accesses to the underlying instance. // Constructed as "nonfair" for performance, like the lock that a // `LinkedBlockingDeque` does. Not clear that we need this // to be a "fair" lock. private final ReentrantLock borrowLock = new ReentrantLock(false); // Condition signaled when all borrowed references have been // handed back or if a pill has been inserted. Awaited when a // lock has been requested but one or more references have been // borrowed from the pool. private final Condition lockAvailable = borrowLock.newCondition(); // Condition signaled when an element has been added into the queue. // Awaited when a request has been made to borrow an item but no elements // currently exist in the queue. private final Condition borrowsAvailable = borrowLock.newCondition(); // Condition signaled when the pool has been unlocked. Awaited when a // request has been made to borrow a reference or lock the pool but the pool // is currently locked. private final Condition poolNotLocked = borrowLock.newCondition(); // Condition signaled when all borrowed references have been returned // to the pool. Awaited when we are preparing to delete the instance // and need to make sure it is not in use. private final Condition instanceNotBorrowed = borrowLock.newCondition(); // The JRuby instance that this pool hands out references to private volatile E instance; // How many times the JRuby instance can be borrowed at once private int maxBorrowCount; // Current number of references to the pool held out in the world. // Updates to this need to be visible to all threads. private volatile AtomicInteger currentBorrowCount; // Thread which currently holds the pool lock. null indicates that // there is no current pool lock holder. Using the current Thread // object for tracking the pool lock owner is comparable to what the JDK's // `ReentrantLock` class does via the `AbstractOwnableSynchronizer` class: // // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/locks/ReentrantLock.java#l164 // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/locks/AbstractOwnableSynchronizer.java#l64 // // Unlike the `AbstractOwnableSynchronizer` class implementation, we marked // this variable as `volatile` because we couldn't convince ourselves // that it would be safe to update this variable from different threads and // not be susceptible to per-thread / per-CPU caching causing the wrong // value to be seen by a thread. `volatile` seems safer and doesn't appear // to impose any noticeable performance degradation. private volatile Thread poolLockThread = null; // Holds a poison pill object for errors and shutdowns // If not null, takes priority over any pool instance when a call to // borrowItem is made. Returns made using releaseItem are ignored if the // released item is the poison pill already stored here private volatile E pill; /** * Create a "pool" of handles to a Jruby instance. * * @param maxBorrows the max number of instance refs that can be handed out */ public ReferencePool(int maxBorrows) { this.instance = null; this.maxBorrowCount = maxBorrows; this.currentBorrowCount = new AtomicInteger(0); } @Override public void register(E e) { final ReentrantLock lock = this.borrowLock; lock.lock(); try { if (this.instance != null) { throw new IllegalStateException( "Unable to register additional instance, pool full"); } this.instance = e; currentBorrowCount.set(0); signalPoolNotEmpty(); } finally { lock.unlock(); } } /** * Unregisters the JRuby instance. Blocks waiting for all borrows of the * instance to be returned before clearing it out. In this pool implementation, * `clear` and `unregister` are aliases of one another, since clearing the pool * is the same as clearing the one active instance. * * @param e the instance to clean up * @throws InterruptedException */ @Override public void unregister(E e) throws InterruptedException { final ReentrantLock lock = this.borrowLock; lock.lock(); try { if (currentBorrowCount.get() != 0) { instanceNotBorrowed.await(); } instance = null; signalIfLockCanProceed(); } finally { lock.unlock(); } } @Override public E borrowItem() throws InterruptedException { E item = null; final ReentrantLock lock = this.borrowLock; lock.lock(); try { final Thread currentThread = Thread.currentThread(); do { if (this.pill != null) { // Return the pill immediately if there is one item = pill; } else if (isPoolLockHeldByAnotherThread(currentThread)) { poolNotLocked.await(); } else if (instance == null) { // No instance initialized yet borrowsAvailable.await(); } else if (this.currentBorrowCount.get() >= this.maxBorrowCount) { // Max borrow count reached, wait for one to be returned borrowsAvailable.await(); } else { item = this.instance; this.currentBorrowCount.getAndIncrement(); } } while (item == null); } finally { lock.unlock(); } return item; } @Override public E borrowItemWithTimeout(long timeout, TimeUnit unit) throws InterruptedException { E item = null; final ReentrantLock lock = this.borrowLock; long remainingMaxTimeToWait = unit.toNanos(timeout); // `lockInterruptibly()` is called here as opposed to just // `lock()` to follow the pattern that the JDK's // `LinkedBlockingDeque` does for a timed poll from a deque. See: // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/LinkedBlockingDeque.java#l516 lock.lockInterruptibly(); try { final Thread currentThread = Thread.currentThread(); // This pattern of using timed `awaitNanos` on a condition // variable to track the total time spent waiting for an item to // be available to be borrowed follows the logic that the JDK's // `LinkedBlockingDeque` in `pollFirst` uses. See: // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/LinkedBlockingDeque.java#l522 do { if (this.pill != null) { // Return the pill immediately if there is one item = pill; } else if (isPoolLockHeldByAnotherThread(currentThread)) { if (remainingMaxTimeToWait <= 0) { break; } remainingMaxTimeToWait = poolNotLocked.awaitNanos(remainingMaxTimeToWait); } else if (this.instance == null) { // No instance initialized yet if (remainingMaxTimeToWait <= 0) { break; } remainingMaxTimeToWait = borrowsAvailable.awaitNanos(remainingMaxTimeToWait); } else if (this.currentBorrowCount.get() >= this.maxBorrowCount) { // Max borrow count reached, wait for one to be returned if (remainingMaxTimeToWait <= 0) { break; } remainingMaxTimeToWait = borrowsAvailable.awaitNanos(remainingMaxTimeToWait); } else if (instance != null) { item = instance; this.currentBorrowCount.getAndIncrement(); } } while (item == null); } finally { lock.unlock(); } return item; } /** * Release an item and return it to the pool. Does nothing if the item * being released is the pill. * Throws an `IllegalArgumentException` if the item is not currently * registered by the pool and the item is not the pill, if one has been * inserted */ @Override public void releaseItem(E e) { final ReentrantLock lock = this.borrowLock; lock.lock(); try { if (e != this.pill) { if (!isRegistered(e)) { String errorMsg = "The item being released is not registered with the pool"; throw new IllegalArgumentException(errorMsg); } this.currentBorrowCount.getAndDecrement(); signalPoolNotEmpty(); if (currentBorrowCount.get() == 0) { instanceNotBorrowed.signal(); } } } finally { lock.unlock(); } } private boolean isRegistered(E e) { return instance.equals(e); } /** * Insert a poison pill into the pool. It should only ever be used to * insert a `PoisonPill` or `ShutdownPoisonPill` to the pool. Only the * first call will insert a pill. Subsequent insertions will be ignored */ @Override public void insertPill(E e) { final ReentrantLock lock = this.borrowLock; lock.lock(); try { if (this.pill == null) { this.pill = e; signalPoolNotEmpty(); } } finally { lock.unlock(); } } /** * Reduce max borrow count down to the number of currently borrowed instances. * Used when shutting down to help prevent additional borrows of the instance, * in preparation for unregistering it. */ @Override public void clear() throws InterruptedException { unregister(this.instance); } @Override public int remainingCapacity() { return instance == null ? 1 : 0; } @Override public int currentSize() { int size; final ReentrantLock lock = this.borrowLock; lock.lock(); try { if (instance == null) { size = 0; } else { size = this.maxBorrowCount - this.currentBorrowCount.get(); } } finally { lock.unlock(); } return size; } /** * Lock the pool. Blocks until the lock is granted and the pool has been filled * back up to its full capacity * @throws InterruptedException */ @Override public void lock() throws InterruptedException { final ReentrantLock lock = this.borrowLock; lock.lock(); try { String pillErrorMsg = "Lock can't be granted because a pill has been inserted"; final Thread currentThread = Thread.currentThread(); while (!isPoolLockHeldByCurrentThread(currentThread)) { if (this.pill != null) { throw new InterruptedException(pillErrorMsg); } if (!isPoolLockHeld()) { poolLockThread = currentThread; } else { poolNotLocked.await(); } } try { // Wait until all references have been returned to the pool while (this.currentBorrowCount.get() > 0) { lockAvailable.await(); if (this.pill != null) { throw new InterruptedException(pillErrorMsg); } } } catch (Exception e) { freePoolLock(); throw e; } } finally { lock.unlock(); } } @Override public void lockWithTimeout(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { final ReentrantLock lock = this.borrowLock; long remainingMaxTimeToWait = unit.toNanos(timeout); // `borrowLock.lockInterruptibly()` is called here as opposed to just // `borrowLock.borrowLock` to follow the pattern that the JDK's // `LinkedBlockingDeque` does for a timed poll from a deque. See: // http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/java/util/concurrent/LinkedBlockingDeque.java#l516 lock.lockInterruptibly(); try { String pillErrorMsg = "Lock can't be granted because a pill has been inserted"; String timeoutErrorMsg = "Timeout limit reached before lock could be granted"; final Thread currentThread = Thread.currentThread(); while (!isPoolLockHeldByCurrentThread(currentThread)) { if (this.pill != null) { throw new InterruptedException(pillErrorMsg); } if (!isPoolLockHeld()) { poolLockThread = currentThread; } else { if (remainingMaxTimeToWait <= 0) { throw new TimeoutException(timeoutErrorMsg); } remainingMaxTimeToWait = poolNotLocked.awaitNanos(remainingMaxTimeToWait); } } try { // Wait until all references have been returned to the pool while (this.currentBorrowCount.get() > 0) { if (remainingMaxTimeToWait <= 0) { throw new TimeoutException(timeoutErrorMsg); } remainingMaxTimeToWait = lockAvailable.awaitNanos(remainingMaxTimeToWait); if (this.pill != null) { throw new InterruptedException(pillErrorMsg); } } } catch (Exception e) { freePoolLock(); throw e; } } finally { lock.unlock(); } } @Override public boolean isLocked() { boolean locked; final ReentrantLock lock = this.borrowLock; lock.lock(); try { locked = isPoolLockHeld(); } finally { lock.unlock(); } return locked; } @Override public void unlock() { final ReentrantLock lock = this.borrowLock; lock.lock(); try { final Thread currentThread = Thread.currentThread(); if (!isPoolLockHeldByCurrentThread(currentThread)) { String lockErrorMessage; if (isPoolLockHeldByAnotherThread(currentThread)) { lockErrorMessage = "held by " + poolLockThread; } else { lockErrorMessage = "not held by any thread"; } throw new IllegalStateException( "Unlock requested from thread not holding the lock. " + "Requested from " + currentThread + " but lock " + lockErrorMessage + "."); } freePoolLock(); } finally { lock.unlock(); } } public Set getRegisteredElements() { Set registered = new CopyOnWriteArraySet(); if (instance != null) { registered.add(instance); } return registered; } private void freePoolLock() { poolLockThread = null; // Need to use 'signalAll' here because there might be multiple // waiters (e.g., multiple borrowers) queued up, waiting for the // pool to be unlocked. poolNotLocked.signalAll(); // Borrowers that are woken up when an instance is returned to the // pool and the pool borrowLock is held would then start waiting on a // 'poolNotLocked' signal instead. Re-signalling 'borrowsAvailable' here // allows any borrowers still waiting on the 'borrowsAvailable' signal to be // reawoken when the pool lock is released, compensating for any // 'borrowsAvailable' signals that might have been essentially ignored from // when the pool lock was held. if (this.currentBorrowCount.get() < this.maxBorrowCount) { borrowsAvailable.signalAll(); } } /** * Should be called if the pool is no longer empty (or a pill is inserted), * so that threads waiting for pool instances can be woken up */ private void signalPoolNotEmpty() { // Could use 'signalAll' here instead of 'signal' but 'signal' is // less expensive in that only one waiter will be woken up. Can use // signal here because the thread being awoken will be able to borrow // a pool instance and any further waiters will be woken up by // subsequent posts of this signal when instances are added/returned to // the queue. borrowsAvailable.signal(); signalIfLockCanProceed(); } /** * Checks if threads waiting on the pool lock should be woken up. * This will wake them up if either all borrowed references have * been returned to the pool, or if a pill has been inserted */ private void signalIfLockCanProceed() { // Could use 'signalAll' here instead of 'signal'. Doesn't really // matter though in that there will only be one waiter at most which // is active at a time - a caller of lock() that has just acquired // the pool lock but is waiting for the live queue to be completely // filled if ((instance != null && currentBorrowCount.get() == 0) || pill != null) { lockAvailable.signal(); } } private boolean isPoolLockHeld() { return poolLockThread != null; } private boolean isPoolLockHeldByCurrentThread(Thread currentThread) { return poolLockThread == currentThread; } private boolean isPoolLockHeldByAnotherThread(Thread currentThread) { return isPoolLockHeld() && !isPoolLockHeldByCurrentThread(currentThread); } } puppetlabs-jruby-utils-ca5d27b/test/000077500000000000000000000000001464330533300176345ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/integration/000077500000000000000000000000001464330533300221575ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/integration/puppetlabs/000077500000000000000000000000001464330533300243365ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/integration/puppetlabs/services/000077500000000000000000000000001464330533300261615ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/integration/puppetlabs/services/jruby_pool_manager/000077500000000000000000000000001464330533300320375ustar00rootroot00000000000000jruby_internal_test.clj000066400000000000000000000140621464330533300365430ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/integration/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-internal-test (:require [clojure.test :refer :all] [clojure.java.jmx :as jmx] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [puppetlabs.trapperkeeper.testutils.logging :as logutils] [puppetlabs.kitchensink.core :as ks] [me.raynes.fs :as fs]) (:import (java.io StringReader) (com.puppetlabs.jruby_utils.pool JRubyPool) (org.jruby RubyInstanceConfig$CompileMode CompatVersion RubyInstanceConfig$ProfilingMode) (org.jruby.util.cli Options) (clojure.lang ExceptionInfo))) ;; Clear changes to JRuby management settings after each test. (use-fixtures :each (fn [f] (f) (.unforce Options/MANAGEMENT_ENABLED))) (deftest get-compile-mode-test (testing "returns correct compile modes for SupportedJRubyCompileModes enum" (is (= RubyInstanceConfig$CompileMode/JIT (jruby-internal/get-compile-mode :jit))) (is (= RubyInstanceConfig$CompileMode/FORCE (jruby-internal/get-compile-mode :force))) (is (= RubyInstanceConfig$CompileMode/OFF (jruby-internal/get-compile-mode :off)))) (testing "returns a valid CompileMode for all values of enum" (doseq [mode jruby-schemas/supported-jruby-compile-modes] (is (instance? RubyInstanceConfig$CompileMode (jruby-internal/get-compile-mode mode))))) (testing "throws an exception if mode is nil" (is (thrown? ExceptionInfo (jruby-internal/get-compile-mode nil)))) (testing "throws an exception for values not in enum" (is (thrown? ExceptionInfo (jruby-internal/get-compile-mode :foo))))) (deftest settings-plumbed-into-jruby-container (testing "settings plumbed into jruby container" (let [pool (JRubyPool. 2) profiler-file (str (ks/temp-file-name "foo")) config (logutils/with-test-logging (jruby-testutils/jruby-config {:compile-mode :jit :profiler-output-file profiler-file :profiling-mode :flat})) instance (jruby-internal/create-pool-instance! pool 0 config) instance-two (jruby-internal/create-pool-instance! pool 1 config) container (:scripting-container instance) container-two (:scripting-container instance-two)] (try (is (= RubyInstanceConfig$CompileMode/JIT (.getCompileMode container))) (is (= RubyInstanceConfig$ProfilingMode/FLAT (.getProfilingMode container))) (finally (.terminate container) (.terminate container-two))) ;; Because we add the current datetime and scripting container ;; hashcode to the filename we need to glob for it here. (let [profiler-files (fs/glob (fs/parent profiler-file) (str (fs/base-name profiler-file) "*")) real-profiler-file (first profiler-files)] (is (= 2 (count profiler-files))) (is (not-empty (slurp real-profiler-file))))))) (deftest default-compile-mode (testing "default compile-mode changes based on jruby version" (let [pool (JRubyPool. 1) config (logutils/with-test-logging (jruby-testutils/jruby-config {})) instance (jruby-internal/create-pool-instance! pool 0 config) container (:scripting-container instance)] (try (is (= RubyInstanceConfig$CompileMode/JIT (.getCompileMode container))) (finally (.terminate container)))))) (deftest jruby-thread-dump (testing "returns an error when jruby.management.enabled is set to false" (.force Options/MANAGEMENT_ENABLED "false") (let [pool (JRubyPool. 1) config (logutils/with-test-logging (jruby-testutils/jruby-config {})) instance (jruby-internal/create-pool-instance! pool 0 config) result (jruby-internal/get-instance-thread-dump instance)] (is (some? (:error result))) (is (re-find #"JRuby management interface not enabled" (:error result))))) (testing "returns a thread dump when jruby.management.enabled is set to true" (.force Options/MANAGEMENT_ENABLED "true") (let [pool (JRubyPool. 1) config (logutils/with-test-logging (jruby-testutils/jruby-config {})) instance (jruby-internal/create-pool-instance! pool 0 config) _ (-> (:scripting-container instance) (.runScriptlet (StringReader. "def naptime Kernel.sleep(1) end Thread.new {naptime}") "jruby-thread-dump.rb")) result (jruby-internal/get-instance-thread-dump instance)] (is (some? (:thread-dump result))) (is (re-find #"jruby-thread-dump\.rb" (:thread-dump result))))) (testing "returns an error if an exception is raised" (.force Options/MANAGEMENT_ENABLED "true") (let [pool (JRubyPool. 1) config (logutils/with-test-logging (jruby-testutils/jruby-config {})) instance (jruby-internal/create-pool-instance! pool 0 config) mbean-name (jruby-internal/jmx-bean-name instance "Runtime") _ (jmx/unregister-mbean mbean-name) failing-mbean (proxy [org.jruby.management.Runtime] [(jruby-internal/get-jruby-runtime instance)] (threadDump [] (throw (Exception. "thread dump exception")))) _ (jmx/register-mbean failing-mbean mbean-name) result (logutils/with-test-logging (jruby-internal/get-instance-thread-dump instance))] (is (some? (:error result))) (is (re-find #"Exception raised while generating thread dump" (:error result)))))) jruby_locking_test.clj000066400000000000000000000154711464330533300363620ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/integration/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-locking-test (:require [clojure.test :refer :all] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [schema.test :as schema-test] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core]) (:import (java.util.concurrent TimeoutException))) (use-fixtures :once schema-test/validate-schemas) (defn jruby-test-config [pool-size] (jruby-testutils/jruby-config {:max-active-instances pool-size :borrow-timeout 1})) (defn can-borrow-from-different-thread? [pool-context] @(future (if-let [instance (jruby-core/borrow-from-pool-with-timeout pool-context :test [])] (do (jruby-core/return-to-pool pool-context instance :test []) true)))) (deftest ^:integration with-lock-test (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 1) (jruby-testutils/wait-for-jrubies-from-pool-context pool-context) (testing "initial state of write lock is unlocked" (is (can-borrow-from-different-thread? pool-context)) (testing "with-lock macro holds write lock while executing body" (jruby-core/with-lock pool-context :with-lock-holds-lock-test (is (not (can-borrow-from-different-thread? pool-context))))) (testing "with-lock macro releases write lock after exectuing body" (is (can-borrow-from-different-thread? pool-context)))))) (deftest ^:integration with-lock-exception-test (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 1) (jruby-testutils/wait-for-jrubies-from-pool-context pool-context) (testing "initial state of write lock is unlocked" (is (can-borrow-from-different-thread? pool-context))) (testing "with-lock macro releases lock even if body throws exception" (is (thrown? IllegalStateException (jruby-core/with-lock pool-context :with-lock-exception-test (is (not (can-borrow-from-different-thread? pool-context))) (throw (IllegalStateException. "exception"))))) (is (can-borrow-from-different-thread? pool-context))))) (deftest ^:integration with-lock-event-notification-test (testing "locking sends event notifications" (let [events (atom []) callback (fn [{:keys [type]}] (swap! events conj type))] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 1) (jruby-testutils/wait-for-jrubies-from-pool-context pool-context) (jruby-core/register-event-handler pool-context callback) (testing "locking events trigger event notifications" (jruby-core/with-jruby-instance jruby-instance pool-context :with-lock-events-test (testing "borrowing a jruby triggers 'requested'/'borrow' events" (is (= [:instance-requested :instance-borrowed] @events)))) (testing "returning a jruby triggers 'returned' event" (is (= [:instance-requested :instance-borrowed :instance-returned] @events))) (jruby-core/with-lock pool-context :with-lock-events-test (testing "acquiring a lock triggers 'lock-requested'/'lock-acquired' events" (is (= [:instance-requested :instance-borrowed :instance-returned :lock-requested :lock-acquired] @events))))) (testing "releasing the lock triggers 'lock-released' event" (is (= [:instance-requested :instance-borrowed :instance-returned :lock-requested :lock-acquired :lock-released] @events))))))) (deftest ^:integration with-lock-and-borrow-contention-test (testing "contention for instances with borrows and locking handled properly" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2) (jruby-testutils/wait-for-jrubies-from-pool-context pool-context) (let [instance (jruby-core/borrow-from-pool-with-timeout pool-context :with-lock-and-borrow-contention-test []) lock-acquired? (promise) unlock-thread? (promise) lock-thread (future (jruby-core/with-lock pool-context :with-lock-and-borrow-contention-test (deliver lock-acquired? true) @unlock-thread?))] (testing "lock not granted yet when instance still borrowed" (is (not (realized? lock-acquired?)))) (jruby-core/return-to-pool pool-context instance :with-lock-and-borrow-contention-test []) @lock-acquired? (testing "cannot borrow from non-locking thread when locked" (is (not (can-borrow-from-different-thread? pool-context)))) (deliver unlock-thread? true) @lock-thread (testing "can borrow from non-locking thread after lock released" (is (can-borrow-from-different-thread? pool-context))))))) (deftest ^:integration with-lock-with-timeout-test (testing "can obtain lock when timeout is not exceeded" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 1) (let [pool (jruby-core/get-pool pool-context)] (jruby-core/with-lock-with-timeout pool-context 10000000 :lock-with-timeout-test (is (.isLocked pool))) (is (not (.isLocked pool)))))) (testing "TimeoutException thrown when lock timeout is exceeded" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 1) (let [pool (jruby-core/get-pool pool-context) borrowed-instance (jruby-core/borrow-from-pool pool-context :lock-timeout-exceeded-test [])] (try ; Since an instance has been borrowed the lock won't be granted and should ; trigger the timeout immediately (is (thrown-with-msg? TimeoutException #"Timeout limit reached before lock could be granted" (jruby-core/with-lock-with-timeout pool-context 1 :lock-timeout-exceeded-test ; should not reach here (is false)))) (is (not (.isLocked pool))) (finally (jruby-core/return-to-pool pool-context borrowed-instance :lock-timeout-exceeded-test []))))))) jruby_pool_int_test.clj000066400000000000000000000354071464330533300365600ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/integration/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-pool-int-test (:require [clojure.test :refer :all] [puppetlabs.trapperkeeper.testutils.bootstrap :as tk-bootstrap] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [puppetlabs.trapperkeeper.app :as tk-app] [puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service :as pool-manager] [puppetlabs.services.protocols.pool-manager :as pool-manager-protocol] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Utilities (def test-borrow-timeout 180000) (defn get-stack-trace-for-thread-as-str [stack-trace-elements] (reduce (fn [acc stack-trace-element] (str acc " " (.getClassName stack-trace-element) "." (.getMethodName stack-trace-element) "(" (.getFileName stack-trace-element) ":" (.getLineNumber stack-trace-element) ")" "\n")) "" stack-trace-elements)) (defn get-all-stack-traces-as-str [] (reduce (fn [acc thread-stack-element] (let [thread (key thread-stack-element)] (str acc "\"" (.getName thread) "\" id=" (.getId thread) " state=" (.getState thread) "\n" (get-stack-trace-for-thread-as-str (val thread-stack-element))))) "" (Thread/getAllStackTraces))) (def script-to-check-if-constant-is-defined "! $instance_id.nil?") (defn set-constants-and-verify [pool-context num-instances] ;; here we set a variable called 'instance_id' in each instance (jruby-testutils/reduce-over-jrubies! pool-context num-instances #(format "$instance_id = %s" %)) ;; and validate that we can read that value back from each instance (= (set (range num-instances)) (-> (jruby-testutils/reduce-over-jrubies! pool-context num-instances (constantly "$instance_id")) set))) (defn check-all-jrubies-for-constants [pool-context num-instances] (jruby-testutils/reduce-over-jrubies! pool-context num-instances (constantly script-to-check-if-constant-is-defined))) (defn check-jrubies-for-constant-counts [pool-context expected-num-true expected-num-false] (let [constants (check-all-jrubies-for-constants pool-context (+ expected-num-false expected-num-true))] (and (= (+ expected-num-false expected-num-true) (count constants)) (= expected-num-true (count (filter true? constants))) (= expected-num-false (count (filter false? constants)))))) (defn verify-no-constants [pool-context num-instances] ;; verify that the constants are cleared out from the instances by looping ;; over them and expecting a 'NameError' when we reference the constant by name. (every? false? (check-all-jrubies-for-constants pool-context num-instances))) (defn borrow-until-desired-borrow-count [pool-context desired-borrow-count] (let [max-borrow-wait-count 100000] (loop [instance (jruby-core/borrow-from-pool-with-timeout pool-context :borrow-until-desired-borrow-count []) loop-count 0] (let [borrow-count (:borrow-count (jruby-core/get-instance-state instance))] (jruby-core/return-to-pool pool-context instance :borrow-until-desired-borrow-count []) (cond (= (inc borrow-count) desired-borrow-count) true (= loop-count max-borrow-wait-count) false :else (recur (jruby-core/borrow-from-pool-with-timeout pool-context :borrow-until-desired-borrow-count []) (inc loop-count))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Tests (deftest ^:integration flush-pool-test (testing "Flushing the pool results in all new JRubyInstances" (jruby-testutils/with-pool-context pool-context [pool-manager/jruby-pool-manager-service] (jruby-testutils/jruby-config {:max-active-instances 4 :borrow-timeout test-borrow-timeout}) ;; set a ruby constant in each instance so that we can recognize them (is (true? (set-constants-and-verify pool-context 4))) (jruby-core/flush-pool! pool-context) (is (jruby-testutils/timed-await (jruby-agents/get-modify-instance-agent pool-context)) (str "timed out waiting for the flush to complete, stack:\n" (get-all-stack-traces-as-str))) ;; now the pool is flushed, so the constants should be cleared (is (true? (verify-no-constants pool-context 4)))))) (deftest ^:integration flush-pool-for-shutdown-test (testing "Flushing the pool for shutdown results in no JRubyInstances left" (tk-bootstrap/with-app-with-config app [pool-manager/jruby-pool-manager-service] {} (let [config (jruby-testutils/jruby-config {:max-active-instances 4 :borrow-timeout test-borrow-timeout}) pool-manager-service (tk-app/get-service app :PoolManagerService) pool-context (pool-manager-protocol/create-pool pool-manager-service config) pool-state (jruby-internal/get-pool-state pool-context) pool (:pool pool-state)] ;; wait for all jrubies to be added to the pool (jruby-testutils/wait-for-jrubies-from-pool-context pool-context) (is (= 4 (.currentSize (jruby-core/get-pool pool-context)))) (jruby-core/flush-pool-for-shutdown! pool-context) ;; flushing the pool should remove all JRubyInstances (is (= 0 (.currentSize pool))) ;; any borrows should now return shutdown poison pill (is (jruby-schemas/shutdown-poison-pill? (.borrowItem pool))))))) (deftest ^:integration hold-file-handle-on-instance-while-another-is-flushed-test (testing "file handle opened from one pool instance is held after other jrubies are destoyed" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances 4 :max-borrows-per-instance 10 :borrow-timeout test-borrow-timeout}) ;; set a ruby constant in each instance so that we can recognize them (is (true? (set-constants-and-verify pool-context 4))) ;; borrow an instance and hold the reference to it. (let [instance (jruby-core/borrow-from-pool-with-timeout pool-context :hold-file-handle-while-another-instance-is-flushed []) sc (:scripting-container instance)] (.runScriptlet sc (str "require 'tempfile'\n\n" "$unique_file = " "Tempfile.new" "('hold-instance-test-', './target')")) (try ; After this, the next borrow and return will trigger a flush (borrow-until-desired-borrow-count pool-context 9) (let [instance-to-flush (jruby-core/borrow-from-pool pool-context :instance-to-flush [])] (is (= 9 (:borrow-count (jruby-core/get-instance-state instance-to-flush)))) (jruby-core/return-to-pool pool-context instance-to-flush :instance-to-flush [])) (is (nil? (.runScriptlet sc "$unique_file.close")) "Unexpected response on attempt to close unique file") (finally (.runScriptlet sc "$unique_file.unlink"))) ;; return the instance (jruby-core/return-to-pool pool-context instance :hold-file-handle-while-another-instance-is-flushed []) ;; Show that the instance-to-flush instance did actually get flushed (check-jrubies-for-constant-counts pool-context 3 1))))) (deftest ^:integration max-borrows-flush-while-pool-flush-in-progress-test (testing "hitting max-borrows while flush in progress doesn't interfere with flush" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances 4 :max-borrows-per-instance 10 :splay-instance-flush false :borrow-timeout test-borrow-timeout}) (let [pool (jruby-core/get-pool pool-context)] ;; set a ruby constant in each instance so that we can recognize them. ;; this counts as one request for each instance. (is (true? (set-constants-and-verify pool-context 4))) ;; borrow one instance and hold the reference to it, to prevent ;; the flush operation from completing (let [instance1 (jruby-core/borrow-from-pool-with-timeout pool-context :max-borrows-flush-while-pool-flush-in-progress-test [])] ;; we are going to borrow and return a second instance until we get its ;; request count up to max-borrows - 1, so that we can use it to test ;; flushing behavior the next time we return it. (is (true? (borrow-until-desired-borrow-count pool-context 9))) ;; now we grab a reference to that instance and hold onto it for later. (let [instance2 (jruby-core/borrow-from-pool-with-timeout pool-context :max-borrows-flush-while-pool-flush-in-progress-test [])] (is (= 9 (:borrow-count (jruby-core/get-instance-state instance2)))) (is (= 2 (jruby-core/free-instance-count pool))) ; Just to show that the pool is not locked yet (is (not (.isLocked pool))) ;; trigger a flush asynchronously (let [flush-future (future (jruby-core/flush-pool! pool-context))] ;; Once the lock is held this means that the flush is waiting ;; for all the instances to be returned before continuing (is (jruby-testutils/wait-for-pool-to-be-locked pool)) ;; now we're going to return instance2 to the pool. This should cause it ;; to get flushed. The main pool flush operation is still blocked. (jruby-core/return-to-pool pool-context instance2 :max-borrows-flush-while-pool-flush-in-progress-test []) ;; Wait until instance2 is returned (is (jruby-testutils/wait-for-instances pool 3) "Timed out waiting for instance2 to return to pool") ;; and finally, we return the last instance we borrowed to the pool (jruby-core/return-to-pool pool-context instance1 :max-borrows-flush-while-pool-flush-in-progress-test []) ;; wait until the flush is complete (is (deref flush-future 10000 false)) (is (not (.isLocked pool))) (is (jruby-testutils/wait-for-instances pool 4) "Timed out waiting for the flush to finish")))) ;; we should have 4 fresh instances without the constant. (is (true? (verify-no-constants pool-context 4))) ;; The jruby return instance calls done within the previous ;; check jrubies call may cause an instance to be in the process of ;; being flushed when the server is shut down. This ensures that ;; the flushing is all done before the server is shut down - since ;; that could otherwise cause an annoying error message about the ;; pool not being full at shut down to be displayed. (jruby-testutils/timed-await (jruby-agents/get-modify-instance-agent pool-context)))))) (deftest initialization-and-cleanup-hooks-test (testing "custom initialization and cleanup callbacks get called appropriately" (let [foo-atom (atom "FOO") lifecycle-fns {:initialize-pool-instance (fn [instance] (assoc instance :foo foo-atom)) :cleanup (fn [instance] (reset! (:foo instance) "Terminating FOO"))} config (jruby-testutils/jruby-config {:max-active-instances 1 :max-borrows-per-instance 10 :borrow-timeout test-borrow-timeout :lifecycle lifecycle-fns})] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services config (let [instance (jruby-core/borrow-from-pool-with-timeout pool-context :initialization-and-cleanup-hooks-test [])] (is (= "FOO" (deref (:foo instance)))) (jruby-core/return-to-pool pool-context instance :initialization-and-cleanup-hooks-test []) (jruby-core/flush-pool! pool-context) ; wait until the flush is complete (is (jruby-testutils/timed-await (jruby-agents/get-modify-instance-agent pool-context))) (is (= "Terminating FOO" (deref foo-atom)))))))) (deftest initialize-scripting-container-hook-test (testing "can set custom environment variables via :initialize-scripting-container hook" (let [lifecycle-fns {:initialize-scripting-container (fn [scripting-container {}] (.setEnvironment scripting-container {"CUSTOMENV" "foobar"}) scripting-container)} config (jruby-testutils/jruby-config {:max-active-instances 1 :max-borrows-per-instance 10 :borrow-timeout test-borrow-timeout :lifecycle lifecycle-fns})] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services config (let [instance (jruby-core/borrow-from-pool-with-timeout pool-context :initialize-environment-variables-test []) scripting-container (:scripting-container instance) jruby-env (.runScriptlet scripting-container "ENV")] (.remove jruby-env "RUBY") (is (= {"CUSTOMENV" "foobar"} jruby-env)) (jruby-core/return-to-pool pool-context instance :initialize-environment-variables-test [])))))) puppetlabs-jruby-utils-ca5d27b/test/unit/000077500000000000000000000000001464330533300206135ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/000077500000000000000000000000001464330533300227725ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/jruby_utils/000077500000000000000000000000001464330533300253455ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/jruby_utils/lockable_pool_test.clj000066400000000000000000000720371464330533300317140ustar00rootroot00000000000000(ns puppetlabs.jruby_utils.lockable-pool-test (:require [clojure.test :refer :all] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils]) (:import (com.puppetlabs.jruby_utils.pool JRubyPool) (java.util.concurrent TimeUnit ExecutionException TimeoutException))) (defn timed-deref [ref] (deref ref 10000 :timed-out)) (defn create-empty-pool [size] (JRubyPool. size)) (defn create-populated-pool [size] (let [pool (create-empty-pool size)] (dotimes [i size] (.register pool (str "foo" i))) pool)) (defn borrow-n-instances [pool n] (doall (for [_ (range n)] (.borrowItem pool)))) (defn return-instances [pool instances] (doseq [instance instances] (.releaseItem pool instance))) (deftest pool-register-above-maximum-throws-exception-test (testing "attempt to register new instance with pool at max capacity fails" (let [pool (create-empty-pool 1)] (.register pool "foo ok") (is (thrown? IllegalStateException (.register pool "foo bar")))))) (deftest pool-unregister-from-pool-test (testing "registered elements properly removed for" (let [pool (create-populated-pool 3) instances (borrow-n-instances pool 2)] (testing "first unregister call" (let [first-instance (first instances) _ (.unregister pool first-instance) registered-elements (.getRegisteredElements pool)] (is (= 2 (.size registered-elements))) (is (false? (contains? registered-elements first-instance))))) (testing "second unregister call" (let [second-instance (second instances) _ (.unregister pool second-instance) registered-elements (.getRegisteredElements pool)] (is (= 1 (.size registered-elements))) (is (false? (contains? registered-elements second-instance)))))))) (deftest pool-lock-is-blocking-until-borrows-returned-test (let [pool (create-populated-pool 3) instances (borrow-n-instances pool 3) future-started? (promise) lock-acquired? (promise) unlock? (promise)] (is (= 3 (count instances))) (is (not (.isLocked pool))) (let [lock-thread (future (deliver future-started? true) (.lock pool) (deliver lock-acquired? true) @unlock? (.unlock pool) true)] @future-started? (testing "pool.lock() blocks until all instances are returned to the pool" (is (not (realized? lock-acquired?))) (testing (str "other threads may successfully return instances while " "pool.lock() is being executed") (.releaseItem pool (first instances)) (is (not (realized? lock-acquired?))) (.releaseItem pool (second instances)) (is (not (realized? lock-acquired?))) (.releaseItem pool (nth instances 2))) (is (true? (timed-deref lock-acquired?)) "timed out waiting for the lock to be acquired") (is (not (realized? lock-thread))) (is (.isLocked pool)) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (not (.isLocked pool)))) (testing "borrows may be resumed after unlock()" (let [instance (.borrowItem pool)] (.releaseItem pool instance)) ;; make sure we got here (is (true? true)))))) (deftest pool-lock-blocks-borrows-test (testing "no other threads may borrow once pool.lock() has been invoked (before or after it returns)" (let [pool (create-populated-pool 2) instances (borrow-n-instances pool 2) lock-thread-started? (promise) lock-acquired? (promise) unlock? (promise)] (is (= 2 (count instances))) (is (not (.isLocked pool))) (let [lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-acquired? true) @unlock? (.unlock pool) true)] @lock-thread-started? (is (not (realized? lock-acquired?))) (let [borrow-after-lock-requested-thread-started? (promise) borrow-after-lock-requested-instance-acquired? (promise) borrow-after-lock-requested-thread (future (deliver borrow-after-lock-requested-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-after-lock-requested-instance-acquired? true) (.releaseItem pool instance)) true)] @borrow-after-lock-requested-thread-started? (is (not (realized? borrow-after-lock-requested-instance-acquired?))) (return-instances pool instances) (is (true? (timed-deref lock-acquired?)) "timed out waiting for the lock acquired thread to start") (is (.isLocked pool)) (is (not (realized? borrow-after-lock-requested-instance-acquired?))) (is (not (realized? lock-thread))) (let [borrow-after-lock-acquired-thread-started? (promise) borrow-after-lock-acquired-instance-acquired? (promise) borrow-after-lock-acquired-thread (future (deliver borrow-after-lock-acquired-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-after-lock-acquired-instance-acquired? true) (.releaseItem pool instance)) true)] @borrow-after-lock-acquired-thread-started? (is (not (realized? borrow-after-lock-acquired-instance-acquired?))) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (true? (timed-deref borrow-after-lock-requested-instance-acquired?)) (str "timed out waiting for the borrow after lock requested " "to be completed")) (is (true? (timed-deref borrow-after-lock-requested-thread)) (str "timed out waiting for the borrow after lock requested " "thread to finish")) (is (true? (timed-deref borrow-after-lock-acquired-instance-acquired?)) "timed out waiting for the borrow after lock acquired") (is (true? (timed-deref borrow-after-lock-acquired-thread)) (str "timed out waiting for the borrow after lock acquired " "thread to finish"))))) (is (not (.isLocked pool)))))) (deftest pool-lock-supersedes-existing-borrows-test (testing "if there are pending borrows when pool.lock() is called, they aren't fulfilled until after unlock()" (let [pool (create-populated-pool 2) instances (borrow-n-instances pool 2) blocked-borrow-thread-started? (promise) blocked-borrow-thread-borrowed? (promise) blocked-borrow-thread (future (deliver blocked-borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver blocked-borrow-thread-borrowed? true) (.releaseItem pool instance)) true) lock-thread-started? (promise) lock-acquired? (promise) unlock? (promise) lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-acquired? true) @unlock? (.unlock pool) true)] @blocked-borrow-thread-started? @lock-thread-started? (is (not (realized? blocked-borrow-thread-borrowed?))) (let [start (System/currentTimeMillis)] (while (and (not (.isLocked pool)) (< (- (System/currentTimeMillis) start) 10000)) (Thread/yield))) (is (.isLocked pool)) (is (not (realized? lock-acquired?))) (return-instances pool instances) (is (not (realized? blocked-borrow-thread-borrowed?))) (is (not (realized? lock-thread))) (is (true? (timed-deref lock-acquired?)) "timed out waiting for the lock to be acquired") (is (.isLocked pool)) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (true? (timed-deref blocked-borrow-thread-borrowed?)) "timed out waiting for the borrow to complete") (is (true? (timed-deref blocked-borrow-thread)) "timed out waiting for the borrow thread to complete") (is (not (.isLocked pool)))))) (deftest pool-lock-reentrant-for-borrow-from-locking-thread (testing "the thread that holds the pool lock may borrow instances while holding the lock" (let [pool (create-populated-pool 2)] (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (let [instance (.borrowItem pool)] (is (true? true)) (.releaseItem pool instance)) (is (true? true)) (.unlock pool) (is (not (.isLocked pool)))))) (deftest pool-lock-reentrant-with-many-borrows-test (testing "the thread that holds the pool lock may borrow instances while holding the lock, even with other borrows queued" (let [pool (create-populated-pool 2)] (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (let [borrow-thread-started-1? (promise) borrow-thread-started-2? (promise) borrow-thread-borrowed-1? (promise) borrow-thread-borrowed-2? (promise) borrow-thread-1 (future (deliver borrow-thread-started-1? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed-1? true) (.releaseItem pool instance)) true) borrow-thread-2 (future (deliver borrow-thread-started-2? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed-2? true) (.releaseItem pool instance)) true)] @borrow-thread-started-1? @borrow-thread-started-2? (is (not (realized? borrow-thread-borrowed-1?))) (is (not (realized? borrow-thread-borrowed-2?))) (let [instance (.borrowItem pool)] (is (true? true)) (.releaseItem pool instance)) (is (true? true)) (.unlock pool) (is (true? (timed-deref borrow-thread-1)) "timed out waiting for first borrow thread to finish") (is (true? (timed-deref borrow-thread-2)) "timed out waiting for second borrow thread to finish")) (is (not (.isLocked pool)))))) (deftest pool-lock-reentrant-for-many-locks-test (testing "multiple threads cannot lock the pool while it is already locked" (let [pool (create-populated-pool 1)] (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (let [lock-thread-started-1? (promise) lock-thread-started-2? (promise) lock-thread-locked-1? (promise) lock-thread-locked-2? (promise) lock-thread-1 (future (deliver lock-thread-started-1? true) (.lock pool) (deliver lock-thread-locked-1? true) (.unlock pool) true) lock-thread-2 (future (deliver lock-thread-started-2? true) (.lock pool) (deliver lock-thread-locked-2? true) (.unlock pool) true)] @lock-thread-started-1? @lock-thread-started-2? (is (not (realized? lock-thread-locked-1?))) (is (not (realized? lock-thread-locked-2?))) (.unlock pool) (is (true? (timed-deref lock-thread-1)) "timed out waiting for first lock thread to finish") (is (true? (timed-deref lock-thread-2)) "timed out waiting for second lock thread to finish")) (is (not (.isLocked pool)))))) (deftest pool-lock-not-held-after-thread-interrupt (let [pool (create-populated-pool 1) item (.borrowItem pool) lock-thread-obj (promise) lock-thread-locked? (promise) lock-thread (future (deliver lock-thread-obj (Thread/currentThread)) (.lock pool) (deliver lock-thread-locked? true))] (testing (str "write locker's thread can be interrupted while waiting for " "instances to be returned") (.interrupt @lock-thread-obj) (is (thrown? ExecutionException (timed-deref lock-thread))) (is (not (realized? lock-thread-locked?)))) (is (not (.isLocked pool))) (.releaseItem pool item) (testing "new write lock can be taken after prior write lock interrupted" (.lock pool) (is (.isLocked pool))))) (deftest pool-unlock-from-thread-not-holding-lock-fails (testing "call to unlock pool when no lock held throws exception" (let [pool (create-populated-pool 1)] (is (thrown? IllegalStateException (.unlock pool))))) (testing "call to unlock pool from thread not holding lock throws exception" (let [pool (create-populated-pool 1) lock-started? (promise) unlock? (promise) lock-thread (future (.lock pool) (deliver lock-started? true) @unlock? (.unlock pool) true)] @lock-started? (is (.isLocked pool)) (is (thrown? IllegalStateException (.unlock pool))) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for lock thread to finish") (is (not (.isLocked pool)))))) (deftest pool-lock-with-timeout-test (testing "lock is granted if timeout is not exceeded" (let [pool (create-populated-pool 1)] (.lockWithTimeout pool 1 TimeUnit/NANOSECONDS) (is (.isLocked pool)))) (testing "lock throws TimeoutException if timeout is exceeded" (let [pool (create-populated-pool 1) borrowed-instance (.borrowItem pool)] (is (thrown-with-msg? TimeoutException #"Timeout limit reached before lock could be granted" (.lockWithTimeout pool 1 TimeUnit/NANOSECONDS) (is (not (.isLocked pool)))))))) (deftest pool-release-item-test (testing "releaseItem returns item to pool and allows pool to still be lockable" (let [pool (create-populated-pool 2) instance (.borrowItem pool)] (is (= 1 (.currentSize pool))) (.releaseItem pool instance) (is (= 2 (.currentSize pool))) (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (is (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS)))) "timed out waiting for borrow with timeout in lock to finish") (.unlock pool) (is (not (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS))))) "timed out waiting for borrow with timeout after unlock to finish") (is (not (.isLocked pool)))))) (deftest pool-borrow-blocks-borrow-when-pool-empty (testing "borrow from pool blocks while the pool is empty" (let [pool (create-populated-pool 1) item (.borrowItem pool) borrow-thread-started? (promise) borrow-thread-borrowed? (promise) borrow-thread (future (deliver borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed? true) (.releaseItem pool instance)) true)] @borrow-thread-started? (is (not (realized? borrow-thread-borrowed?))) (.releaseItem pool item) (is (true? (timed-deref borrow-thread)) "timed out waiting for borrow thread to finish")))) (deftest pool-timed-borrows-test (testing "pool borrows with timeout" (let [pool (create-populated-pool 1) item (.borrowItem pool)] (testing "borrow times out and returns nil when pool is empty" (is (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS)))))) (.releaseItem pool item) (testing "borrow succeeds when pool is non-empty" (is (identical? item (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS))))))))) (deftest pool-can-do-blocking-borrow-after-borrow-timed-out (testing "can do blocking borrow from pool after previous borrow timed out" (let [pool (create-populated-pool 1) item (.borrowItem pool)] (is (nil? (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS))) (let [borrow-thread-started? (promise) borrow-thread-borrowed? (promise) borrow-thread (future (deliver borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed? true) (.releaseItem pool instance)) true)] @borrow-thread-started? (is (not (realized? borrow-thread-borrowed?))) (.releaseItem pool item) (is (true? (timed-deref borrow-thread)) "timed out waiting for borrow thread to finish"))))) (deftest pool-can-do-blocking-borrow-after-borrow-timed-out-during-lock (testing (str "can do a blocking borrow from pool after previous borrow " "timed out while a write lock was held") (let [pool (create-populated-pool 1)] (.lock pool) (is (.isLocked pool)) (is (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS)))) "timed out waiting for borrow with timeout to finish") (let [borrow-thread-started? (promise) borrow-thread-borrowed? (promise) borrow-thread (future (deliver borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed? true) (.releaseItem pool instance)) true)] @borrow-thread-started? (is (not (realized? borrow-thread-borrowed?))) (.unlock pool) (is (true? (timed-deref borrow-thread)) "timed out waiting for borrow thread to finish") (is (not (.isLocked pool))))))) (deftest pool-can-borrow-after-borrow-interrupted-during-lock (testing (str "can do a borrow after another borrow was interrupted " "while a write lock was held") (let [pool (create-populated-pool 1) borrow-1 (.borrowItem pool) borrow-thread-started-2 (promise) borrow-thread-started-3? (promise) lock-thread-locked? (promise) borrow-thread-2 (future (deliver borrow-thread-started-2 (Thread/currentThread)) (timed-deref lock-thread-locked?) (.borrowItem pool)) borrow-thread-3 (future (deliver borrow-thread-started-3? true) (timed-deref lock-thread-locked?) (.borrowItem pool)) borrow-thread-obj-2 @borrow-thread-started-2 _ @borrow-thread-started-3? lock-thread-started? (promise) unlock? (promise) lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-thread-locked? true) @unlock? (.unlock pool) true)] @lock-thread-started? (.releaseItem pool borrow-1) (is (true? (timed-deref lock-thread-locked?)) "timed out waiting for the lock to be acquired") (is (true? (.isLocked pool))) ;; Interrupt the second borrow thread so that it will stop waiting for the ;; pool to be not empty and not locked. (.interrupt borrow-thread-obj-2) (is (thrown? ExecutionException (timed-deref borrow-thread-2)) "second borrow could not be interrupted") (is (not (realized? borrow-thread-3))) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (not (.isLocked pool))) ;; If the third borrow doesn't block indefinitely, we've confirmed that ;; the interruption of the second borrow while the write lock was ;; held did not adversely affect the ability of the third borrow call ;; to return. (is (identical? borrow-1 (timed-deref borrow-thread-3)) (str "did not get back the same instance from the third borrow " "attempt as was returned from the first borrow attempt"))))) (deftest pool-can-borrow-after-borrow-timed-out-during-lock (testing (str "can do a timed borrow from pool after previous borrow " "timed out while a write lock was held") (let [pool (create-populated-pool 1) borrow-1 (.borrowItem pool) borrow-thread-started-2? (promise) borrow-thread-started-3? (promise) lock-thread-locked? (promise) borrow-thread-2 (future (deliver borrow-thread-started-2? true) (timed-deref lock-thread-locked?) (.borrowItemWithTimeout pool 50 TimeUnit/MILLISECONDS)) borrow-thread-3 (future (deliver borrow-thread-started-3? true) (timed-deref lock-thread-locked?) (.borrowItemWithTimeout pool 1 TimeUnit/SECONDS)) _ @borrow-thread-started-2? _ @borrow-thread-started-3? unlock? (promise) lock-thread-started? (promise) lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-thread-locked? true) @unlock? (.unlock pool) true)] @lock-thread-started? (.releaseItem pool borrow-1) (is (true? (timed-deref lock-thread-locked?)) "timed out waiting for the lock to be acquired") (is (.isLocked pool)) (is (nil? (timed-deref borrow-thread-2)) "second borrow attempt did not time out") (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (not (.isLocked pool))) (is (identical? borrow-1 (timed-deref borrow-thread-3)) (str "did not get back the same instance from the third borrow" "attempt as was returned from the first borrow attempt"))))) (deftest pool-insert-pill-test (testing "inserted pill is next item borrowed" (let [pool (create-populated-pool 1) pill (str "i'm a pill")] (.insertPill pool pill) (is (identical? (.borrowItem pool) pill)) (testing "subsequent borrows return the same pill" (is (identical? (.borrowItem pool) pill))))) (testing "when borrow is blocked, inserting a pill unblocks it" (let [pool (create-populated-pool 1) pill (str "I'm just a pill, yes I'm only a pill") instance (.borrowItem pool) blocked-borrow (future (.borrowItem pool))] (is (= 0 (.currentSize pool))) ; Give future a chance to run and block (Thread/sleep 500) ; Pool is empty and the borrow future is blocked (is (not (realized? blocked-borrow))) ; inserting the pill should unblock the promise (.insertPill pool pill) ; borrow finishes and gives us back the pill (let [future-result (timed-deref blocked-borrow)] (is (identical? future-result pill))))) (testing "second insert doesn't change the pill" (let [pool (create-populated-pool 1) first-pill (str "pill clinton") second-pill (str "pillary clinton")] (.insertPill pool first-pill) (is (identical? first-pill (.borrowItem pool))) (.insertPill pool second-pill) ; Should still be equal to the first pill (is (identical? first-pill (.borrowItem pool))) (is (not (identical? second-pill (.borrowItem pool))))))) (deftest release-item-exceptions-test (testing "releasing a different pill than the one that was inserted errors" (let [pool (create-populated-pool 1) first-pill (str "a city upon a pill") second-pill (str "capitol pill")] (.insertPill pool first-pill) ; Only to show that it does not error (is (nil? (.releaseItem pool first-pill))) (is (thrown-with-msg? IllegalArgumentException #"The item being released is not registered with the pool" (.releaseItem pool second-pill))))) (testing "releasing a jruby not registered with the pool errors" (let [pool (create-populated-pool 1) not-in-pool-instance "I was never registered"] (is (thrown-with-msg? IllegalArgumentException #"The item being released is not registered with the pool" (.releaseItem pool not-in-pool-instance)))))) (deftest lock-interrupted-by-pill-insertion-test (testing "a call to .lock will throw if there is a pill" (let [pool (create-populated-pool 1) pill "pilliam shakespeare"] (.insertPill pool pill) (is (thrown-with-msg? InterruptedException #"Lock can't be granted because a pill has been inserted" (.lock pool))))) (testing "a blocked .lock call throws an InterruptedException once a pill is inserted" (let [pool (create-populated-pool 1) pill "pillful ignorance"] ; Make it so the pool is not full (.borrowItem pool) ; Exceptions thrown from the future will be returned as InterruptedException, ; so we can't use thrown-with-msg?. We'll catch it, return it instead of ; throwing it, and inspect it manually below (let [blocked-lock-future (future (try (.lock pool) (catch InterruptedException e e)))] ; The future's thread will take the lock, and then block waiting for ; either the pool to fill up, or a pill to be inserted (jruby-testutils/wait-for-pool-to-be-locked pool) (.insertPill pool pill) (let [exception @blocked-lock-future] (is (= InterruptedException (type exception))) (is (= "Lock can't be granted because a pill has been inserted" (.getMessage exception)))))))) (deftest pool-clear-test (testing (str "pool clear removes all elements from queue and only matching" "registered elements") (let [pool (create-populated-pool 3) instance (.borrowItem pool)] (is (= 2 (.currentSize pool))) (is (= 3 (.. pool getRegisteredElements size))) (.clear pool) (is (= 0 (.currentSize pool))) (let [registered-elements (.getRegisteredElements pool)] (is (= 1 (.size registered-elements))) (is (identical? instance (-> registered-elements (.iterator) iterator-seq first))))))) (deftest pool-remaining-capacity (testing "remaining capacity in pool correct per instances in the queue" (let [pool (create-populated-pool 5)] (is (= 0 (.remainingCapacity pool))) (let [instances (borrow-n-instances pool 2)] (is (= 2 (.remainingCapacity pool))) (return-instances pool instances) (is (= 0 (.remainingCapacity pool))))))) puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/jruby_utils/lockable_ref_pool_test.clj000066400000000000000000000674521464330533300325550ustar00rootroot00000000000000(ns puppetlabs.jruby_utils.lockable-ref-pool-test (:require [clojure.test :refer :all] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils]) (:import (com.puppetlabs.jruby_utils.pool ReferencePool) (java.util.concurrent TimeUnit ExecutionException TimeoutException))) (defn timed-deref [ref] (deref ref 10000 :timed-out)) (defn create-empty-pool ([] (create-empty-pool 10)) ([maxBorrows] (ReferencePool. maxBorrows))) (defn create-populated-pool ([] create-populated-pool 10) ([size] (let [pool (create-empty-pool size)] (.register pool (str "foo")) pool))) (defn borrow-n-instances [pool n] (doall (for [_ (range n)] (.borrowItem pool)))) (defn return-instances [pool instances] (doseq [instance instances] (.releaseItem pool instance))) (deftest pool-register-above-maximum-throws-exception-test (testing "attempt to register new instance with pool at max capacity fails" (let [pool (create-empty-pool)] (.register pool "foo ok") (is (thrown? IllegalStateException (.register pool "foo bar")))))) (deftest pool-lock-is-blocking-until-borrows-returned-test (let [pool (create-populated-pool 3) instances (borrow-n-instances pool 3) future-started? (promise) lock-acquired? (promise) unlock? (promise)] (is (= 3 (count instances))) (is (not (.isLocked pool))) (let [lock-thread (future (deliver future-started? true) (.lock pool) (deliver lock-acquired? true) @unlock? (.unlock pool) true)] @future-started? (testing "pool.lock() blocks until all instances are returned to the pool" (is (not (realized? lock-acquired?))) (testing (str "other threads may successfully return instances while " "pool.lock() is being executed") (.releaseItem pool (first instances)) (is (not (realized? lock-acquired?))) (.releaseItem pool (second instances)) (is (not (realized? lock-acquired?))) (.releaseItem pool (nth instances 2))) (is (true? (timed-deref lock-acquired?)) "timed out waiting for the lock to be acquired") (is (not (realized? lock-thread))) (is (.isLocked pool)) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (not (.isLocked pool)))) (testing "borrows may be resumed after unlock()" (let [instance (.borrowItem pool)] (.releaseItem pool instance)) ;; make sure we got here (is (true? true)))))) (deftest pool-lock-blocks-borrows-test (testing "no other threads may borrow once pool.lock() has been invoked (before or after it returns)" (let [pool (create-populated-pool 2) instances (borrow-n-instances pool 2) lock-thread-started? (promise) lock-acquired? (promise) unlock? (promise)] (is (= 2 (count instances))) (is (not (.isLocked pool))) (let [lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-acquired? true) @unlock? (.unlock pool) true)] @lock-thread-started? (is (not (realized? lock-acquired?))) (let [borrow-after-lock-requested-thread-started? (promise) borrow-after-lock-requested-instance-acquired? (promise) borrow-after-lock-requested-thread (future (deliver borrow-after-lock-requested-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-after-lock-requested-instance-acquired? true) (.releaseItem pool instance)) true)] @borrow-after-lock-requested-thread-started? (is (not (realized? borrow-after-lock-requested-instance-acquired?))) (return-instances pool instances) (is (true? (timed-deref lock-acquired?)) "timed out waiting for the lock acquired thread to start") (is (.isLocked pool)) (is (not (realized? borrow-after-lock-requested-instance-acquired?))) (is (not (realized? lock-thread))) (let [borrow-after-lock-acquired-thread-started? (promise) borrow-after-lock-acquired-instance-acquired? (promise) borrow-after-lock-acquired-thread (future (deliver borrow-after-lock-acquired-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-after-lock-acquired-instance-acquired? true) (.releaseItem pool instance)) true)] @borrow-after-lock-acquired-thread-started? (is (not (realized? borrow-after-lock-acquired-instance-acquired?))) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (true? (timed-deref borrow-after-lock-requested-instance-acquired?)) (str "timed out waiting for the borrow after lock requested " "to be completed")) (is (true? (timed-deref borrow-after-lock-requested-thread)) (str "timed out waiting for the borrow after lock requested " "thread to finish")) (is (true? (timed-deref borrow-after-lock-acquired-instance-acquired?)) "timed out waiting for the borrow after lock acquired") (is (true? (timed-deref borrow-after-lock-acquired-thread)) (str "timed out waiting for the borrow after lock acquired " "thread to finish"))))) (is (not (.isLocked pool)))))) (deftest pool-lock-supersedes-existing-borrows-test (testing "if there are pending borrows when pool.lock() is called, they aren't fulfilled until after unlock()" (let [pool (create-populated-pool 2) instances (borrow-n-instances pool 2) blocked-borrow-thread-started? (promise) blocked-borrow-thread-borrowed? (promise) blocked-borrow-thread (future (deliver blocked-borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver blocked-borrow-thread-borrowed? true) (.releaseItem pool instance)) true) lock-thread-started? (promise) lock-acquired? (promise) unlock? (promise) lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-acquired? true) @unlock? (.unlock pool) true)] @blocked-borrow-thread-started? @lock-thread-started? (is (not (realized? blocked-borrow-thread-borrowed?))) ;; Ensure that the pool is locked before returning any instances (let [start (System/currentTimeMillis)] (while (and (not (.isLocked pool)) (< (- (System/currentTimeMillis) start) 10000)) (Thread/yield))) (is (.isLocked pool)) ;; Borrows are blocked, but lock cannot yet proceed because ;; some instances are still borrowed (is (not (realized? lock-acquired?))) (return-instances pool instances) ;; Lock can proceed, but threads are still blocked (is (not (realized? blocked-borrow-thread-borrowed?))) (is (not (realized? lock-thread))) (is (true? (timed-deref lock-acquired?)) "timed out waiting for the lock to be acquired") (is (.isLocked pool)) (deliver unlock? true) ;; Lock thread completes, then borrows can proceed (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (true? (timed-deref blocked-borrow-thread-borrowed?)) "timed out waiting for the borrow to complete") (is (true? (timed-deref blocked-borrow-thread)) "timed out waiting for the borrow thread to complete") (is (not (.isLocked pool)))))) (deftest pool-lock-reentrant-for-borrow-from-locking-thread (testing "the thread that holds the pool lock may borrow instances while holding the lock" (let [pool (create-populated-pool 2)] (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (let [instance (.borrowItem pool)] (is (true? true)) (.releaseItem pool instance)) (is (true? true)) (.unlock pool) (is (not (.isLocked pool)))))) (deftest pool-lock-reentrant-with-many-borrows-test (testing "the thread that holds the pool lock may borrow instances while holding the lock, even with other borrows queued" (let [pool (create-populated-pool 2)] (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (let [borrow-thread-started-1? (promise) borrow-thread-started-2? (promise) borrow-thread-borrowed-1? (promise) borrow-thread-borrowed-2? (promise) borrow-thread-1 (future (deliver borrow-thread-started-1? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed-1? true) (.releaseItem pool instance)) true) borrow-thread-2 (future (deliver borrow-thread-started-2? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed-2? true) (.releaseItem pool instance)) true)] @borrow-thread-started-1? @borrow-thread-started-2? (is (not (realized? borrow-thread-borrowed-1?))) (is (not (realized? borrow-thread-borrowed-2?))) (let [instance (.borrowItem pool)] (is (true? true)) (.releaseItem pool instance)) (is (true? true)) (.unlock pool) (is (true? (timed-deref borrow-thread-1)) "timed out waiting for first borrow thread to finish") (is (true? (timed-deref borrow-thread-2)) "timed out waiting for second borrow thread to finish")) (is (not (.isLocked pool)))))) (deftest pool-lock-reentrant-for-many-locks-test (testing "multiple threads cannot lock the pool while it is already locked" (let [pool (create-populated-pool 1)] (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (let [lock-thread-started-1? (promise) lock-thread-started-2? (promise) lock-thread-locked-1? (promise) lock-thread-locked-2? (promise) lock-thread-1 (future (deliver lock-thread-started-1? true) (.lock pool) (deliver lock-thread-locked-1? true) (.unlock pool) true) lock-thread-2 (future (deliver lock-thread-started-2? true) (.lock pool) (deliver lock-thread-locked-2? true) (.unlock pool) true)] @lock-thread-started-1? @lock-thread-started-2? (is (not (realized? lock-thread-locked-1?))) (is (not (realized? lock-thread-locked-2?))) (.unlock pool) (is (true? (timed-deref lock-thread-1)) "timed out waiting for first lock thread to finish") (is (true? (timed-deref lock-thread-2)) "timed out waiting for second lock thread to finish")) (is (not (.isLocked pool)))))) (deftest pool-lock-not-held-after-thread-interrupt (let [pool (create-populated-pool 1) item (.borrowItem pool) lock-thread-obj (promise) lock-thread-locked? (promise) lock-thread (future (deliver lock-thread-obj (Thread/currentThread)) (.lock pool) (deliver lock-thread-locked? true))] (testing (str "write locker's thread can be interrupted while waiting for " "instances to be returned") (.interrupt @lock-thread-obj) (is (thrown? ExecutionException (timed-deref lock-thread))) (is (not (.isLocked pool))) (is (not (realized? lock-thread-locked?)))) (.releaseItem pool item) (testing "new write lock can be taken after prior write lock interrupted" (.lock pool) (is (.isLocked pool))))) (deftest pool-unlock-from-thread-not-holding-lock-fails (testing "call to unlock pool when no lock held throws exception" (let [pool (create-populated-pool 1)] (is (thrown? IllegalStateException (.unlock pool))))) (testing "call to unlock pool from thread not holding lock throws exception" (let [pool (create-populated-pool 1) lock-started? (promise) unlock? (promise) lock-thread (future (.lock pool) (deliver lock-started? true) @unlock? (.unlock pool) true)] @lock-started? (is (.isLocked pool)) (is (thrown? IllegalStateException (.unlock pool))) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for lock thread to finish") (is (not (.isLocked pool)))))) (deftest pool-lock-with-timeout-test (testing "lock is granted if timeout is not exceeded" (let [pool (create-populated-pool 1)] (.lockWithTimeout pool 1 TimeUnit/NANOSECONDS) (is (.isLocked pool)))) (testing "lock throws TimeoutException if timeout is exceeded" (let [pool (create-populated-pool 1) borrowed-instance (.borrowItem pool)] (is (thrown-with-msg? TimeoutException #"Timeout limit reached before lock could be granted" (.lockWithTimeout pool 1 TimeUnit/NANOSECONDS) (is (not (.isLocked pool)))))))) (deftest pool-release-item-test (testing "releaseItem returns item to pool and allows pool to still be lockable" (let [pool (create-populated-pool 2) instance (.borrowItem pool)] (is (= 1 (.currentSize pool))) (.releaseItem pool instance) (is (= 2 (.currentSize pool))) (is (not (.isLocked pool))) (.lock pool) (is (.isLocked pool)) (is (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS)))) "timed out waiting for borrow with timeout in lock to finish") (.unlock pool) (is (not (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS))))) "timed out waiting for borrow with timeout after unlock to finish") (is (not (.isLocked pool)))))) (deftest pool-borrow-blocks-borrow-when-pool-empty (testing "borrow from pool blocks while the pool is empty" (let [pool (create-populated-pool 1) item (.borrowItem pool) borrow-thread-started? (promise) borrow-thread-borrowed? (promise) borrow-thread (future (deliver borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed? true) (.releaseItem pool instance)) true)] @borrow-thread-started? (is (not (realized? borrow-thread-borrowed?))) (.releaseItem pool item) (is (true? (timed-deref borrow-thread)) "timed out waiting for borrow thread to finish")))) (deftest pool-timed-borrows-test (testing "pool borrows with timeout" (let [pool (create-populated-pool 1) item (.borrowItem pool)] (testing "borrow times out and returns nil when pool is empty" (is (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS)))))) (.releaseItem pool item) (testing "borrow succeeds when pool is non-empty" (is (identical? item (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS))))))))) (deftest pool-can-do-blocking-borrow-after-borrow-timed-out (testing "can do blocking borrow from pool after previous borrow timed out" (let [pool (create-populated-pool 1) item (.borrowItem pool)] (is (nil? (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS))) (let [borrow-thread-started? (promise) borrow-thread-borrowed? (promise) borrow-thread (future (deliver borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed? true) (.releaseItem pool instance)) true)] @borrow-thread-started? (is (not (realized? borrow-thread-borrowed?))) (.releaseItem pool item) (is (true? (timed-deref borrow-thread)) "timed out waiting for borrow thread to finish"))))) (deftest pool-can-do-blocking-borrow-after-borrow-timed-out-during-lock (testing (str "can do a blocking borrow from pool after previous borrow " "timed out while a write lock was held") (let [pool (create-populated-pool 1)] (.lock pool) (is (.isLocked pool)) (is (nil? (timed-deref (future (.borrowItemWithTimeout pool 1 TimeUnit/MICROSECONDS)))) "timed out waiting for borrow with timeout to finish") (let [borrow-thread-started? (promise) borrow-thread-borrowed? (promise) borrow-thread (future (deliver borrow-thread-started? true) (let [instance (.borrowItem pool)] (deliver borrow-thread-borrowed? true) (.releaseItem pool instance)) true)] @borrow-thread-started? (is (not (realized? borrow-thread-borrowed?))) (.unlock pool) (is (true? (timed-deref borrow-thread)) "timed out waiting for borrow thread to finish") (is (not (.isLocked pool))))))) (deftest pool-can-borrow-after-borrow-interrupted-during-lock (testing (str "can do a borrow after another borrow was interrupted " "while a write lock was held") (let [pool (create-populated-pool 1) borrow-1 (.borrowItem pool) borrow-thread-started-2 (promise) borrow-thread-started-3? (promise) lock-thread-locked? (promise) borrow-thread-2 (future (deliver borrow-thread-started-2 (Thread/currentThread)) (timed-deref lock-thread-locked?) (.borrowItem pool)) borrow-thread-3 (future (deliver borrow-thread-started-3? true) (timed-deref lock-thread-locked?) (.borrowItem pool)) borrow-thread-obj-2 @borrow-thread-started-2 _ @borrow-thread-started-3? lock-thread-started? (promise) unlock? (promise) lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-thread-locked? true) @unlock? (.unlock pool) true)] @lock-thread-started? (.releaseItem pool borrow-1) (is (true? (timed-deref lock-thread-locked?)) "timed out waiting for the lock to be acquired") (is (true? (.isLocked pool))) ;; Interrupt the second borrow thread so that it will stop waiting for the ;; pool to be not empty and not locked. (.interrupt borrow-thread-obj-2) (is (thrown? ExecutionException (timed-deref borrow-thread-2)) "second borrow could not be interrupted") (is (not (realized? borrow-thread-3))) (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (not (.isLocked pool))) ;; If the third borrow doesn't block indefinitely, we've confirmed that ;; the interruption of the second borrow while the write lock was ;; held did not adversely affect the ability of the third borrow call ;; to return. (is (identical? borrow-1 (timed-deref borrow-thread-3)) (str "did not get back the same instance from the third borrow " "attempt as was returned from the first borrow attempt"))))) (deftest pool-can-borrow-after-borrow-timed-out-during-lock (testing (str "can do a timed borrow from pool after previous borrow " "timed out while a write lock was held") (let [pool (create-populated-pool 1) borrow-1 (.borrowItem pool) borrow-thread-started-2? (promise) borrow-thread-started-3? (promise) lock-thread-locked? (promise) borrow-thread-2 (future (deliver borrow-thread-started-2? true) (timed-deref lock-thread-locked?) (.borrowItemWithTimeout pool 50 TimeUnit/MILLISECONDS)) borrow-thread-3 (future (deliver borrow-thread-started-3? true) (timed-deref lock-thread-locked?) (.borrowItemWithTimeout pool 1 TimeUnit/SECONDS)) _ @borrow-thread-started-2? _ @borrow-thread-started-3? unlock? (promise) lock-thread-started? (promise) lock-thread (future (deliver lock-thread-started? true) (.lock pool) (deliver lock-thread-locked? true) @unlock? (.unlock pool) true)] @lock-thread-started? (.releaseItem pool borrow-1) (is (true? (timed-deref lock-thread-locked?)) "timed out waiting for the lock to be acquired") (is (.isLocked pool)) (is (nil? (timed-deref borrow-thread-2)) "second borrow attempt did not time out") (deliver unlock? true) (is (true? (timed-deref lock-thread)) "timed out waiting for the lock thread to finish") (is (not (.isLocked pool))) (is (identical? borrow-1 (timed-deref borrow-thread-3)) (str "did not get back the same instance from the third borrow" "attempt as was returned from the first borrow attempt"))))) (deftest pool-insert-pill-test (testing "inserted pill is next item borrowed" (let [pool (create-populated-pool 1) pill (str "i'm a pill")] (.insertPill pool pill) (is (identical? (.borrowItem pool) pill)) (testing "subsequent borrows return the same pill" (is (identical? (.borrowItem pool) pill))))) (testing "when borrow is blocked, inserting a pill unblocks it" (let [pool (create-populated-pool 1) pill (str "I'm just a pill, yes I'm only a pill") _ (.borrowItem pool) blocked-borrow (future (.borrowItem pool))] (is (= 0 (.currentSize pool))) ; Give future a chance to run and block (Thread/sleep 500) ; Pool is empty and the borrow future is blocked (is (not (realized? blocked-borrow))) ; inserting the pill should unblock the promise (.insertPill pool pill) ; borrow finishes and gives us back the pill (let [future-result (timed-deref blocked-borrow)] (is (identical? future-result pill))))) (testing "second insert doesn't change the pill" (let [pool (create-populated-pool 1) first-pill "pill clinton" second-pill "pillary clinton"] (.insertPill pool first-pill) (is (identical? first-pill (.borrowItem pool))) (.insertPill pool second-pill) ; Should still be equal to the first pill (is (identical? first-pill (.borrowItem pool))) (is (not (identical? second-pill (.borrowItem pool))))))) (deftest release-item-exceptions-test (testing "releasing a different pill than the one that was inserted errors" (let [pool (create-populated-pool 1) first-pill "a city upon a pill" second-pill "capitol pill"] (.insertPill pool first-pill) ; Only to show that it does not error (is (nil? (.releaseItem pool first-pill))) (is (thrown-with-msg? IllegalArgumentException #"The item being released is not registered with the pool" (.releaseItem pool second-pill))))) (testing "releasing a jruby not registered with the pool errors" (let [pool (create-populated-pool 1) not-in-pool-instance "I was never registered"] (is (thrown-with-msg? IllegalArgumentException #"The item being released is not registered with the pool" (.releaseItem pool not-in-pool-instance)))))) (deftest lock-interrupted-by-pill-insertion-test (testing "a call to .lock will throw if there is a pill" (let [pool (create-populated-pool 1) pill "pilliam shakespeare"] (.insertPill pool pill) (is (thrown-with-msg? InterruptedException #"Lock can't be granted because a pill has been inserted" (.lock pool))))) (testing "a blocked .lock call throws an InterruptedException once a pill is inserted" (let [pool (create-populated-pool 1) pill "pillful ignorance"] ; Make it so the pool is not full (.borrowItem pool) ; Exceptions thrown from a future will be returned and then thrown when the ; future is dereferenced, so we can't use `throw-with-msg?`. Instead, we catch ; the exception and return it so it can be inspected below. (let [blocked-lock-future (future (try (.lock pool) (catch InterruptedException e e)))] ; The future's thread will take the lock, and then block while waiting for ; either the pool to fill up, or a pill to be inserted (jruby-testutils/wait-for-pool-to-be-locked pool) (.insertPill pool pill) (let [exception @blocked-lock-future] (is (= InterruptedException (type exception))) (is (= "Lock can't be granted because a pill has been inserted" (.getMessage exception)))))))) (deftest pool-remaining-capacity (testing "remaining capacity in pool correct per instances registered" (let [empty-pool (create-empty-pool) pool (create-populated-pool 5)] (is (= 1 (.remainingCapacity empty-pool))) (is (= 0 (.remainingCapacity pool)))))) puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/jruby_utils/slj4j_logger_test.clj000066400000000000000000000100561464330533300314650ustar00rootroot00000000000000(ns puppetlabs.jruby-utils.slj4j-logger-test (:require [clojure.test :refer :all] [puppetlabs.trapperkeeper.testutils.logging :as logutils]) (:import (org.jruby.util.log LoggerFactory))) (deftest slf4j-logger-test (let [actual-logger-name "my-test-logger" exception-message "exceptionally bad news" expected-logger-name (str "jruby." actual-logger-name) logger (LoggerFactory/getLogger actual-logger-name) actual-log-event (fn [event] (assoc event :exception (when-let [exception (:exception event)] (.getMessage exception)))) expected-log-event (fn [message level exception] {:message message :level level :exception exception :logger expected-logger-name})] (testing "name stored in logger" (is (= expected-logger-name (.getName logger)))) (testing "warn with a string and objects" (logutils/with-test-logging (.warn logger "a {} {} warning" (into-array Object ["strongly" "worded"])) (is (logged? "a strongly worded warning" :warn)))) (testing "warn with an exception" (logutils/with-test-logging (.warn logger (Exception. exception-message)) (is (logged? #(= (expected-log-event "" :warn exception-message) (actual-log-event %)))))) (testing "warn with a string and an exception" (logutils/with-test-logging (.warn logger "a warning" (Exception. exception-message)) (is (logged? #(= (expected-log-event "a warning" :warn exception-message) (actual-log-event %)))))) (testing "error with a string and objects" (logutils/with-test-logging (.error logger "a {} {} error" (into-array Object ["strongly" "worded"])) (is (logged? "a strongly worded error" :error)))) (testing "error with an exception" (logutils/with-test-logging (.error logger (Exception. exception-message)) (is (logged? #(= (expected-log-event "" :error exception-message) (actual-log-event %)))))) (testing "error with a string and an exception" (logutils/with-test-logging (.error logger "an error" (Exception. exception-message)) (is (logged? #(= (expected-log-event "an error" :error exception-message) (actual-log-event %)))))) (testing "info with a string and objects" (logutils/with-test-logging (.info logger "some {} {} info" (into-array Object ["strongly" "worded"])) (is (logged? "some strongly worded info" :info)))) (testing "info with an exception" (logutils/with-test-logging (.info logger (Exception. exception-message)) (is (logged? #(= (expected-log-event "" :info exception-message) (actual-log-event %)))))) (testing "info with a string and an exception" (logutils/with-test-logging (.info logger "some info" (Exception. exception-message)) (is (logged? #(= (expected-log-event "some info" :info exception-message) (actual-log-event %)))))) (testing "debug with a string and objects" (logutils/with-test-logging (.debug logger "some {} {} debug" (into-array Object ["strongly" "worded"])) (is (logged? "some strongly worded debug" :debug)))) (testing "info with an exception" (logutils/with-test-logging (.debug logger (Exception. exception-message)) (is (logged? #(= (expected-log-event "" :debug exception-message) (actual-log-event %)))))) (testing "debug with a string and an exception" (logutils/with-test-logging (.debug logger "some debug" (Exception. exception-message)) (is (logged? #(= (expected-log-event "some debug" :debug exception-message) (actual-log-event %)))))))) puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/000077500000000000000000000000001464330533300246155ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager/000077500000000000000000000000001464330533300304735ustar00rootroot00000000000000jruby_agents_test.clj000066400000000000000000000070411464330533300346430ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-agents-test (:require [clojure.test :refer :all] [schema.test :as schema-test] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [puppetlabs.services.jruby-pool-manager.impl.jruby-pool-manager-core :as jruby-pool-manager-core]) (:import (puppetlabs.services.jruby_pool_manager.jruby_schemas JRubyInstance))) (use-fixtures :once schema-test/validate-schemas) (deftest execute-tasks!-test (let [pool-context (jruby-pool-manager-core/create-pool-context (jruby-testutils/jruby-config {:instance-creation-concurrency 3})) creation-service (jruby-internal/get-creation-service pool-context)] (testing "creation-service is a FixedThreadPool of configured number of threads" (is (= 3 (.getMaximumPoolSize creation-service)))) ;; this isn't a requirement and should be able to change in the future without issue, ;; but none of the current callers require the result, so explictly test the assumption. (testing "does not return results of task execution" (let [tasks [(fn [] :foo) (fn [] :bar)] results (jruby-agents/execute-tasks! tasks creation-service)] (is (nil? results)))) (testing "throws original execptions" (let [tasks [(fn [] (throw (IllegalStateException. "BOOM")))]] (is (thrown? IllegalStateException (jruby-agents/execute-tasks! tasks creation-service))))))) (deftest next-instance-id-test (let [pool-context (jruby-pool-manager-core/create-pool-context (jruby-testutils/jruby-config {:max-active-instances 8}))] (testing "next instance id should be based on the pool size" (is (= 10 (jruby-agents/next-instance-id 2 pool-context))) (is (= 100 (jruby-agents/next-instance-id 92 pool-context)))) (testing "next instance id should wrap after max int" (let [id (- Integer/MAX_VALUE 1)] (is (= (mod id 8) (jruby-agents/next-instance-id id pool-context))))))) (deftest custom-termination-test (testing "Flushing the pool causes cleanup hook to be called" (let [cleanup-atom (atom nil) config (assoc-in (jruby-testutils/jruby-config {:max-active-instances 1}) [:lifecycle :cleanup] (fn [x] (reset! cleanup-atom "Hello from cleanup")))] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services config (jruby-core/flush-pool! pool-context) ; wait until the flush is complete (is (jruby-testutils/timed-await (jruby-agents/get-modify-instance-agent pool-context))) (is (= "Hello from cleanup" (deref cleanup-atom))))))) (deftest collect-all-jrubies-test (testing "returns list of all the jruby instances" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances 4}) (let [pool (jruby-core/get-pool pool-context) jruby-list (jruby-agents/borrow-all-jrubies pool-context)] (try (is (= 4 (count jruby-list))) (is (every? #(instance? JRubyInstance %) jruby-list)) (is (= 0 (.currentSize pool))) (finally (jruby-testutils/fill-drained-pool pool-context jruby-list))))))) puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager/jruby_core_test.clj000066400000000000000000000124071464330533300343730ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.jruby-core-test (:require [clojure.test :refer :all] [schema.test :as schema-test] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] [puppetlabs.trapperkeeper.testutils.logging :as logutils] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas]) (:import (java.io ByteArrayOutputStream PrintStream ByteArrayInputStream))) (use-fixtures :once schema-test/validate-schemas) (def min-config (jruby-core/initialize-config {:gem-home "./target/jruby-gem-home", :ruby-load-path ["./dev-resources/puppetlabs/services/jruby_pool_manager/jruby_core_test"]})) (defmacro with-stdin-str "Evaluates body in a context in which System/in is bound to a fresh input stream initialized with the string s. The return value of evaluating body is returned." [s & body] `(let [system-input# (System/in) string-input# (new ByteArrayInputStream (.getBytes ~s))] (try (System/setIn string-input#) ~@body (finally (System/setIn system-input#))))) (defmacro capture-out "capture System.out and return it as the value of :out in the return map. The return value of body is available as :return in the return map. This macro is intended to be used for JRuby interop. Please see with-out-str for an idiomatic clojure equivalent. This macro is not thread safe." [& body] `(let [return-map# (atom {}) system-output# (System/out) captured-output# (new ByteArrayOutputStream) capturing-print-stream# (new PrintStream captured-output#)] (try (System/setOut capturing-print-stream#) (swap! return-map# assoc :return (do ~@body)) (finally (.flush capturing-print-stream#) (swap! return-map# assoc :out (.toString captured-output#)) (System/setOut system-output#))) @return-map#)) (deftest default-num-cpus-test (testing "1 jruby instance for a 1 or 2-core box" (is (= 1 (jruby-core/default-pool-size 1))) (is (= 1 (jruby-core/default-pool-size 2)))) (testing "2 jruby instances for a 3-core box" (is (= 2 (jruby-core/default-pool-size 3)))) (testing "3 jruby instances for a 4-core box" (is (= 3 (jruby-core/default-pool-size 4)))) (testing "4 jruby instances for anything above 5 cores" (is (= 4 (jruby-core/default-pool-size 5))) (is (= 4 (jruby-core/default-pool-size 8))) (is (= 4 (jruby-core/default-pool-size 16))) (is (= 4 (jruby-core/default-pool-size 32))) (is (= 4 (jruby-core/default-pool-size 64))))) (deftest cli-run!-error-handling-test (testing "when command is not found as a resource" (logutils/with-test-logging (is (nil? (jruby-core/cli-run! min-config "DNE" []))) (is (logged? #"DNE could not be found" :error))))) (deftest ^:integration cli-run!-test (testing "jruby cli command output" (testing "gem env (SERVER-262)" (let [m (capture-out (jruby-core/cli-run! min-config "gem" ["env"])) {:keys [return out]} m exit-code (.getStatus return)] (is (= 0 exit-code)) ; The choice of SHELL PATH is arbitrary, just need something to scan for (is (re-find #"SHELL PATH:" out)))) (testing "gem list" (let [m (capture-out (jruby-core/cli-run! min-config "gem" ["list"])) {:keys [return out]} m exit-code (.getStatus return)] (is (= 0 exit-code)) ; The choice of json is arbitrary, just need something to scan for (is (re-find #"\bjson\b" out)))) ;; IRB testing does not work on *nix based systems until a version ;; of io/console >= 0.6.0 is included in JRuby ; (testing "irb" ; (let [m (capture-out ; (with-stdin-str "puts %{HELLO}\n" ; (jruby-core/cli-run! min-config "irb" ["-f"]))) ; {:keys [return out]} m ; exit-code (.getStatus return)] ; (is (= 0 exit-code)) ; (is (re-find #"\nHELLO\n" out))) ; (let [m (capture-out ; (with-stdin-str "Kernel.exit(42)\n" ; (jruby-core/cli-run! min-config "irb" ["-f"]))) ; {:keys [return _]} m ; exit-code (.getStatus return)] ; (is (= 42 exit-code)))) ; (testing "irb with -r foo" ; (let [m (capture-out ; (with-stdin-str "puts %{#{foo}}\n" ; (jruby-core/cli-run! min-config "irb" ["-r" "foo" "-f"]))) ; {:keys [return out]} m ; exit-code (.getStatus return)] ; (is (= 0 exit-code)) ; (is (re-find #"bar" out)))) (testing "non existing subcommand returns nil" (logutils/with-test-logging (is (nil? (jruby-core/cli-run! min-config "doesnotexist" []))))))) (deftest ^:integration cli-ruby!-test (testing "jruby cli command output" ;; TODO: consider bringing the CLI clj files back into the repo? (TK-378) (testing "ruby -r puppet" (let [m (capture-out (with-stdin-str "puts %{#{foo}}" (jruby-core/cli-ruby! min-config ["-r" "foo"]))) {:keys [return out]} m exit-code (.getStatus return)] (is (= 0 exit-code)) (is (re-find #"bar" out)))))) (deftest jruby-version-info-test (is (re-find #"jruby 9." jruby-core/jruby-version-info))) jruby_interpreter_test.clj000066400000000000000000000035431464330533300357300ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-interpreter-test (:require [clojure.test :refer :all] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core])) (deftest jruby-env-vars (testing "the environment used by the JRuby interpreters" (jruby-testutils/with-scripting-container jruby-interpreter (jruby-testutils/jruby-config) (let [jruby-env (.runScriptlet jruby-interpreter "ENV")] ;; $HOME and $PATH are left in by `jruby-env` ;; Note that other environment variables are allowed through (e.g. ;; `HTTP_PROXY` - see jruby-core/env-vars-allowed-list for the full list), ;; but are not expected to be set in most environments. However, in ;; order to make this more test robust, these variables are always ;; filtered out. (is (= #{"HOME" "PATH" "GEM_HOME" "JARS_NO_REQUIRE" "JARS_REQUIRE" "RUBY"} (set (remove (set jruby-core/proxy-vars-allowed-list) (keys jruby-env))))))))) (deftest jruby-configured-env-vars (testing "the environment used by the JRuby interpreters can be added to via the config" (jruby-testutils/with-scripting-container jruby-interpreter (jruby-testutils/jruby-config {:environment-vars {:FOO "for_jruby"}}) (let [jruby-env (.runScriptlet jruby-interpreter "ENV")] ;; Note that other environment variables are allowed through, ;; but are not expected to be set in most environments. However, in ;; order to make this test more robust, these variables are always ;; filtered out. (is (= #{"HOME" "PATH" "GEM_HOME" "JARS_NO_REQUIRE" "JARS_REQUIRE" "FOO" "RUBY"} (set (remove (set jruby-core/proxy-vars-allowed-list) (keys jruby-env))))) (is (= (.get jruby-env "FOO") "for_jruby")))))) puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager/jruby_pool_test.clj000066400000000000000000000456051464330533300344220ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.jruby-pool-test (:require [clojure.set :as set] [clojure.test :refer :all] [puppetlabs.kitchensink.core :as ks] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [puppetlabs.trapperkeeper.testutils.logging :as logutils] [puppetlabs.services.jruby-pool-manager.impl.jruby-pool-manager-core :as jruby-pool-manager-core] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [puppetlabs.services.protocols.jruby-pool :as pool-protocol]) (:import (clojure.lang ExceptionInfo) (puppetlabs.services.jruby_pool_manager.jruby_schemas ShutdownPoisonPill))) (defn- initialize-jruby-config-with-logging-suppressed [config] (logutils/with-test-logging (jruby-core/initialize-config config))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Tests (deftest configuration-validation (testing "malformed configuration fails" (let [malformed-config {:illegal-key [1 2 3]}] (is (thrown-with-msg? ExceptionInfo #"Input to create-pool-context does not match schema" (jruby-pool-manager-core/create-pool-context malformed-config))))) (let [minimal-config {:gem-home "/dev/null" :ruby-load-path ["/dev/null"]} config (initialize-jruby-config-with-logging-suppressed minimal-config)] (testing "max-active-instances is set to default if not specified" (is (= (jruby-core/default-pool-size (ks/num-cpus)) (:max-active-instances config)))) (testing "max-borrows-per-instance is set to 0 if not specified" (is (= 0 (:max-borrows-per-instance config)))) (testing "max-borrows-per-instance is honored if specified" (is (= 5 (-> minimal-config (assoc :max-borrows-per-instance 5) (initialize-jruby-config-with-logging-suppressed) :max-borrows-per-instance)))) (testing "compile-mode is set to default if not specified" (is (= jruby-core/default-jruby-compile-mode (:compile-mode config)))) (testing "compile-mode is honored if specified" (is (= :off (-> minimal-config (assoc :compile-mode "off") (initialize-jruby-config-with-logging-suppressed) :compile-mode))) (is (= :jit (-> minimal-config (assoc :compile-mode "jit") (initialize-jruby-config-with-logging-suppressed) :compile-mode)))) (testing "gem-path is set to nil if not specified" (is (nil? (-> minimal-config initialize-jruby-config-with-logging-suppressed :gem-path)))) (testing "gem-path is respected if specified" (is (= "/tmp/foo:/dev/null" (-> minimal-config (assoc :gem-path "/tmp/foo:/dev/null") initialize-jruby-config-with-logging-suppressed :gem-path)))))) (deftest test-jruby-core-funcs (let [pool-size 2 timeout 250 config (jruby-testutils/jruby-config {:max-active-instances pool-size :borrow-timeout timeout}) pool-context (jruby-pool-manager-core/create-pool-context config) pool (jruby-core/get-pool pool-context)] (testing "The pool should not yet be full as it is being primed in the background." (is (= (jruby-core/free-instance-count pool) 0)) (jruby-agents/prime-pool! pool-context) (try (testing "Borrowing all instances from a pool while it is being primed and returning them." (let [all-the-jrubys (jruby-testutils/drain-pool pool-context pool-size)] (is (= 0 (jruby-core/free-instance-count pool))) (doseq [instance all-the-jrubys] (is (not (nil? instance)) "One of JRubyInstances is nil")) (jruby-testutils/fill-drained-pool pool-context all-the-jrubys) (is (= pool-size (jruby-core/free-instance-count pool))))) (testing "Borrowing from an empty pool with a timeout returns nil within the proper amount of time." (let [all-the-jrubys (jruby-testutils/drain-pool pool-context pool-size) test-start-in-millis (System/currentTimeMillis)] (is (nil? (jruby-core/borrow-from-pool-with-timeout pool-context :test []))) (is (>= (- (System/currentTimeMillis) test-start-in-millis) timeout) "The timeout value was not honored.") (jruby-testutils/fill-drained-pool pool-context all-the-jrubys) (is (= (jruby-core/free-instance-count pool) pool-size) "All JRubyInstances were not returned to the pool."))) (testing "Removing an instance decrements the pool size by 1." (let [jruby-instance (jruby-core/borrow-from-pool pool-context :test [])] (is (= (jruby-core/free-instance-count pool) (dec pool-size))) (jruby-core/return-to-pool pool-context jruby-instance :test []))) (testing "Borrowing an instance increments its request count." (let [drain-via (fn [borrow-fn] (doall (repeatedly pool-size borrow-fn))) assoc-count (fn [acc jruby] (assoc acc (:id jruby) (:borrow-count (jruby-core/get-instance-state jruby)))) get-counts (fn [jrubies] (reduce assoc-count {} jrubies))] (doseq [drain-fn [#(jruby-core/borrow-from-pool pool-context :test []) #(jruby-core/borrow-from-pool-with-timeout pool-context :test [])]] (let [jrubies (drain-via drain-fn) counts (get-counts jrubies)] (jruby-testutils/fill-drained-pool pool-context jrubies) (let [jrubies (drain-via drain-fn) new-counts (get-counts jrubies)] (jruby-testutils/fill-drained-pool pool-context jrubies) (is (= (ks/keyset counts) (ks/keyset new-counts))) (doseq [k (keys counts)] (is (= (inc (counts k)) (new-counts k))))))))) (finally (jruby-core/flush-pool-for-shutdown! pool-context)))))) (deftest borrow-while-pool-is-being-initialized-test (testing "borrow will block until an instance is available while the pool is coming online" (let [pool-initialized? (promise) init-fn (fn [instance] @pool-initialized? instance) pool-size 1] ;; start a pool initialization, which will block on the `initialize-pool-instance` ;; function's deref of the promise (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances pool-size :lifecycle {:initialize-pool-instance init-fn}}) ;; start a borrow, which should block until an instance becomes available (let [borrow-instance (future (jruby-core/borrow-from-pool-with-timeout pool-context :borrow-during-pool-init-test []))] (try (is (not (realized? borrow-instance))) ;; deliver the promise, allowing the pool initialization to complete (deliver pool-initialized? true) ;; now the borrow can complete (is (jruby-schemas/jruby-instance? @borrow-instance)) (finally (jruby-core/return-to-pool pool-context @borrow-instance :borrow-during-pool-init-test [])))))))) (deftest borrow-while-no-instances-available-test (testing "when all instances are in use, borrow blocks until an instance becomes available" (let [pool-size 2] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances pool-size}) ;; borrow both instances from the pool (let [drained-instances (jruby-testutils/drain-pool pool-context pool-size)] (try (is (= 2 (count drained-instances))) ;; attempt a borrow, which will block because no instances are free (let [borrow-instance (future (jruby-core/borrow-from-pool-with-timeout pool-context :borrow-with-no-free-instances-test []))] (is (not (realized? borrow-instance))) ;; return an instance to the pool (jruby-core/return-to-pool pool-context (first drained-instances) :borrow-with-no-free-instances-test []) ;; now the borrow can complete (is (some? @borrow-instance))) (finally (jruby-testutils/fill-drained-pool pool-context drained-instances)))))))) (deftest prime-pools-failure (let [pool-size 2 config (jruby-testutils/jruby-config {:max-active-instances pool-size}) pool-context (jruby-pool-manager-core/create-pool-context config) err-msg (re-pattern "Unable to borrow JRubyInstance from pool")] (is (thrown? IllegalStateException (jruby-agents/prime-pool! (assoc-in pool-context [:config :lifecycle :initialize-pool-instance] (fn [_] (throw (IllegalStateException. "BORK!"))))))) (testing "borrow and borrow-with-timeout both throw an exception if the pool failed to initialize" (is (thrown-with-msg? IllegalStateException err-msg (jruby-core/borrow-from-pool pool-context :test []))) (is (thrown-with-msg? IllegalStateException err-msg (jruby-core/borrow-from-pool-with-timeout pool-context :test [])))) (testing "borrow and borrow-with-timeout both continue to throw exceptions on subsequent calls" (is (thrown-with-msg? IllegalStateException err-msg (jruby-core/borrow-from-pool pool-context :test []))) (is (thrown-with-msg? IllegalStateException err-msg (jruby-core/borrow-from-pool-with-timeout pool-context :test [])))))) (deftest test-default-pool-size (let [config (jruby-testutils/jruby-config) pool (jruby-pool-manager-core/create-pool-context config) pool-state (jruby-core/get-pool-state pool)] (is (= (jruby-core/default-pool-size (ks/num-cpus)) (:size pool-state))))) (defn jruby-test-config ([max-borrows] (jruby-test-config max-borrows 1)) ([max-borrows max-instances] (jruby-testutils/jruby-config {:max-active-instances max-instances :max-borrows-per-instance max-borrows}))) (deftest worker-id-persists (testing "worker id is the same at borrow time and return time" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2) (let [[instance borrowed-id] (pool-protocol/borrow pool-context) returned-id (pool-protocol/return pool-context instance)] (is (= borrowed-id returned-id)))))) (deftest splay-jruby-instance-flushing (testing "Disabled JRuby instance splaying -" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances 5 :max-borrows-per-instance 3 :splay-instance-flush false}) (let [first-ids (jruby-testutils/drain-and-refill pool-context) second-ids (jruby-testutils/drain-and-refill pool-context) ;; All jruby instances should be recycled after this refill ;; but the ids will be of old instances that were drained third-ids (jruby-testutils/drain-and-refill pool-context)] (testing "Does not flush any instances prior to max borrows" (is (= first-ids second-ids third-ids))) (let [fourth-ids (jruby-testutils/drain-and-refill pool-context)] (testing "All instances flushed after max borrows" (is (empty? (set/intersection fourth-ids third-ids)))))))) (testing "Splayed JRuby instance flushing -" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services ;; with three instances, each with three max borrows, we should recycle ;; an instance every cycle of draining. The first instance to be ;; recycled will do so after the first drain, its replacement should ;; then stay until the fourth drain/refill. (jruby-test-config 3 3) (let [first-ids (jruby-testutils/drain-and-refill pool-context) second-ids (jruby-testutils/drain-and-refill pool-context) third-ids (jruby-testutils/drain-and-refill pool-context) fourth-ids (jruby-testutils/drain-and-refill pool-context) fifth-ids (jruby-testutils/drain-and-refill pool-context) original-instances-surviving-first-drain (set/intersection first-ids second-ids) new-instance-after-first-drain (set/difference second-ids first-ids) original-instances-surviving-second-drain (set/intersection first-ids third-ids) original-instances-surviving-third-drain (set/intersection first-ids fourth-ids)] (testing "Initial instances are flushed each splay interval" (is (= 2 (count original-instances-surviving-first-drain))) (is (= 1 (count original-instances-surviving-second-drain))) (is (= 0 (count original-instances-surviving-third-drain)))) (testing "New instances are flushed at max-borrows" (is (not-empty (set/intersection new-instance-after-first-drain third-ids))) (is (not-empty (set/intersection new-instance-after-first-drain fourth-ids))) (is (empty? (set/intersection new-instance-after-first-drain fifth-ids)))))))) (deftest flush-jruby-after-max-borrows (testing "JRubyInstance is not flushed if it has not exceeded max borrows" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2) (let [instance (jruby-core/borrow-from-pool pool-context :test []) id (:id instance)] (jruby-core/return-to-pool pool-context instance :test []) (let [instance (jruby-core/borrow-from-pool pool-context :test [])] (is (= id (:id instance))) (jruby-core/return-to-pool pool-context instance :test []))))) (testing "JRubyInstance is flushed after exceeding max borrows" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2) (jruby-testutils/wait-for-jrubies-from-pool-context pool-context) (is (= 1 (count (jruby-core/registered-instances pool-context)))) (let [instance (jruby-core/borrow-from-pool pool-context :test []) id (:id instance)] (jruby-core/return-to-pool pool-context instance :test []) (jruby-core/borrow-from-pool pool-context :test []) (jruby-core/return-to-pool pool-context instance :test []) (let [instance (jruby-core/borrow-from-pool pool-context :test [])] (is (not= id (:id instance))) (jruby-core/return-to-pool pool-context instance :test [])) (testing "instance is removed from registered elements after flushing" (is (= 1 (count (jruby-core/registered-instances pool-context)))))) (testing "Can lock pool after a flush via max borrows" (let [timeout 1 new-pool-context (assoc-in pool-context [:config :borrow-timeout] timeout)] (pool-protocol/lock new-pool-context) (is (nil? @(future (jruby-core/borrow-from-pool-with-timeout new-pool-context :test [])))) (pool-protocol/unlock new-pool-context) (let [instance @(future (jruby-core/borrow-from-pool-with-timeout new-pool-context :test []))] (is (not (nil? instance))) (jruby-core/return-to-pool pool-context instance :test [])))))) (testing "JRubyInstance is not flushed if max borrows setting is set to 0" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 0) (let [instance (jruby-core/borrow-from-pool pool-context :test []) id (:id instance)] (jruby-core/return-to-pool pool-context instance :test []) (let [instance (jruby-core/borrow-from-pool pool-context :test [])] (is (= id (:id instance))) (jruby-core/return-to-pool pool-context instance :test []))))) (testing "Can flush a JRubyInstance that is not the first one in the pool" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2 3) (let [instance1 (jruby-core/borrow-from-pool pool-context :test []) instance2 (jruby-core/borrow-from-pool pool-context :test []) id (:id instance2)] (jruby-core/return-to-pool pool-context instance2 :test []) ;; borrow it a second time and confirm we get the same instance (let [instance2 (jruby-core/borrow-from-pool pool-context :test [])] (is (= id (:id instance2))) (jruby-core/return-to-pool pool-context instance2 :test [])) ;; borrow it a third time and confirm that we get a different instance (let [instance2 (jruby-core/borrow-from-pool pool-context :test [])] (is (not= id (:id instance2))) (jruby-core/return-to-pool pool-context instance2 :test [])) (jruby-core/return-to-pool pool-context instance1 :test []))))) (deftest return-pill-to-pool-test (testing "Returning a pill to the pool does not throw" ; Essentially this test is insurance to make sure we aren't doing anything ; funky when we return a pill to the pool. Instances have some internal ; data that gets manipulated during a return that poison pills don't have, ; and we'd get null pointer exceptions if this code path tried to access ; those non-existent properties on the pill object (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2) (let [pool (jruby-core/get-pool pool-context) pill (ShutdownPoisonPill. pool)] ; Returning a pill should be a noop (jruby-core/return-to-pool pool-context pill :test []))))) jruby_ref_pool_test.clj000066400000000000000000000304301464330533300351650ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-ref-pool-test (:require [clojure.test :refer :all] [slingshot.test :refer :all] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] [puppetlabs.services.protocols.jruby-pool :as pool-protocol] [puppetlabs.services.jruby-pool-manager.impl.jruby-pool-manager-core :as jruby-pool-manager-core] [puppetlabs.services.jruby-pool-manager.impl.jruby-agents :as jruby-agents] [puppetlabs.trapperkeeper.testutils.logging :as logutils]) (:import (puppetlabs.services.jruby_pool_manager.jruby_schemas ShutdownPoisonPill) (java.util.concurrent TimeoutException))) (defn jruby-test-config ([max-borrows] (jruby-test-config max-borrows 1)) ([max-borrows max-instances] (jruby-testutils/jruby-config {:max-active-instances max-instances :multithreaded true :max-borrows-per-instance max-borrows})) ([max-borrows max-instances options] (jruby-testutils/jruby-config (merge {:max-active-instances max-instances :multithreaded true :max-borrows-per-instance max-borrows} options)))) (deftest worker-id-persists (testing "worker id is the same at borrow time and return time" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2) (let [[instance borrowed-id] (pool-protocol/borrow pool-context) returned-id (pool-protocol/return pool-context instance)] (is (= (.getId (Thread/currentThread)) borrowed-id)) (is (= (.getId (Thread/currentThread)) returned-id)))))) (deftest borrow-while-no-instances-available-test (testing "when all instances are in use, borrow blocks until an instance becomes available" (let [pool-size 2] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 0 pool-size) ;; borrow both instances from the pool (let [drained-instances (jruby-testutils/drain-pool pool-context pool-size)] (try (is (= 2 (count drained-instances))) ;; attempt a borrow, which will block because no instances are free (let [borrow-instance (future (jruby-core/borrow-from-pool-with-timeout pool-context :borrow-with-no-free-instances-test []))] (is (not (realized? borrow-instance))) ;; return an instance to the pool (jruby-core/return-to-pool pool-context (first drained-instances) :borrow-with-no-free-instances-test []) ;; now the borrow can complete (is (some? @borrow-instance))) (finally (jruby-testutils/fill-drained-pool pool-context drained-instances)))))))) (defn add-watch-for-flush-complete [pool-context] (let [flush-complete (promise)] (add-watch (:borrow-count pool-context) :flush-callback (fn [k a old-count new-count] (when (and (= k :flush-callback ) (< new-count old-count)) (remove-watch a :flush-callback) (deliver flush-complete true)))) flush-complete)) (deftest flush-jruby-after-max-borrows (testing "JRubyInstance is not flushed if it has not exceeded max borrows" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2 1) (let [instance (jruby-core/borrow-from-pool pool-context :test []) id (:id instance)] (jruby-core/return-to-pool pool-context instance :test []) (let [instance (jruby-core/borrow-from-pool pool-context :test []) flush-complete (add-watch-for-flush-complete pool-context)] (is (not (realized? flush-complete))) (is (= id (:id instance))) ;; This return will trigger the flush (jruby-core/return-to-pool pool-context instance :test []) (is @flush-complete))) (testing "Can lock pool after a flush via max borrows" (let [timeout 1 new-pool-context (assoc-in pool-context [:config :borrow-timeout] timeout)] (pool-protocol/lock new-pool-context) (is (nil? @(future (jruby-core/borrow-from-pool-with-timeout new-pool-context :test [])))) (pool-protocol/unlock new-pool-context) (let [instance (jruby-core/borrow-from-pool new-pool-context :test [])] (is (= 2 (:id instance))) (jruby-core/return-to-pool pool-context instance :test [])))))) (testing "flushing due to max borrows succeeds when we've over-borrowed" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services ;; We can check out the instance more times than `max-borrows` and return ;; them each in sequence, and nothing blocks or fails (jruby-test-config 2 3) (let [instance1 (jruby-core/borrow-from-pool pool-context :test []) instance2 (jruby-core/borrow-from-pool pool-context :test []) instance3 (jruby-core/borrow-from-pool pool-context :test []) flush-complete (add-watch-for-flush-complete pool-context)] (jruby-core/return-to-pool pool-context instance1 :test []) (is (not (realized? flush-complete))) ;; This return triggers the flush (jruby-core/return-to-pool pool-context instance2 :test []) ;; But it doesn't finish till the other borrow is returned (is (not (realized? flush-complete))) (jruby-core/return-to-pool pool-context instance3 :test []) (is @flush-complete)) (let [instance (jruby-core/borrow-from-pool pool-context :test [])] (is (= 2 (:id instance))) (jruby-core/return-to-pool pool-context instance :test [])))) (testing "JRubyInstance is not flushed if max borrows setting is set to 0" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 0) (let [instance (jruby-core/borrow-from-pool pool-context :test []) id (:id instance)] (jruby-core/return-to-pool pool-context instance :test []) (let [instance (jruby-core/borrow-from-pool pool-context :test [])] (is (= id (:id instance))) (jruby-core/return-to-pool pool-context instance :test [])))))) (deftest flush-times-out-on-return (testing "Attempt to flush times out if flush-timeout is reached" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services ;; Set max-instances higher than max-borrows in order to over-borrow (jruby-test-config 1 2 {:flush-timeout 1}) ;; Borrow instance1 so we can trigger a refresh when we return it ;; Borrow instance2 to prevent lock from being acquired when we return instance1 (let [instance1 (first (pool-protocol/borrow pool-context)) instance2 (first (pool-protocol/borrow pool-context)) flush-complete (add-watch-for-flush-complete pool-context)] (logutils/with-test-logging ;; Return to trigger flush, which should block since instance2 is still borrowed (pool-protocol/return pool-context instance1) (is (not (realized? flush-complete))) ;; Wait for 2 seconds to make sure the flush really times out before proceeding (Thread/sleep 2000) (is (logged? "Max borrows reached, but JRubyPool could not be flushed because lock could not be acquired. Will try again later." :warn)) ;; Return instance2 so that flush can proceed (pool-protocol/return pool-context instance2) (is @flush-complete)))))) (deftest flush-times-out (testing "Attempt to flush times out if flush-timeout is reached" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2 1 {:flush-timeout 0}) ;; Borrow an instance so that the lock can't be acquired (let [instance (first (pool-protocol/borrow pool-context))] (is (thrown+? [:kind :puppetlabs.services.jruby-pool-manager.impl.jruby-internal/jruby-lock-timeout :msg "An attempt to lock the JRubyPool failed with a timeout"] (pool-protocol/flush-pool pool-context))) ;; Return the instance so that shutdown can proceed (otherwise this hangs) (pool-protocol/return pool-context instance))))) (deftest return-pill-to-pool-test (testing "Returning a pill to the pool does not throw" ; Essentially this test is insurance to make sure we aren't doing anything ; funky when we return a pill to the pool. Instances have some internal ; data that gets manipulated during a return that poison pills don't have, ; and we'd get null pointer exceptions if this code path tried to access ; those non-existent properties on the pill object (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 2) (let [pool (jruby-core/get-pool pool-context) pill (ShutdownPoisonPill. pool)] ; Returning a pill should be a noop (jruby-core/return-to-pool pool-context pill :test []))))) (deftest shutdown-prevents-flush (testing "Triggering a flush after shutdown has been requested does not break shutdown" (let [config (jruby-test-config 0 2) ;; Set up the pool manually, since we trigger a shutdown in this test, ;; and the macro would also try to shutdown. It's not a valid operation ;; to shutdown twice. pool-context (jruby-pool-manager-core/create-pool-context config) _ (jruby-agents/prime-pool! pool-context) pool (jruby-core/get-pool pool-context) _ (jruby-testutils/wait-for-jrubies-from-pool-context pool-context) ;; Set up test instance (first (pool-protocol/borrow pool-context)) shutdown-complete? (promise) _ (future (pool-protocol/shutdown pool-context) (deliver shutdown-complete? true)) _ (jruby-testutils/wait-for-pool-to-be-locked pool) ;; When the shutdown inserts the pill, it should interrupt the ;; pending lock from the flush. So we catch that exception and return it. flush-thread (future (try (pool-protocol/flush-pool pool-context) (catch InterruptedException e e)))] ;; Shutdown should be blocked because an instance has been borrowed (is (not (realized? shutdown-complete?))) (is (not (realized? flush-thread))) (pool-protocol/return pool-context instance) @shutdown-complete? (let [exception @flush-thread] (is (= InterruptedException (type exception))) (is (= "Lock can't be granted because a pill has been inserted" (.getMessage exception))))))) (deftest shutdown-times-out (testing "Attempt to shutdown times out if flush-timeout is reached" (let [config (jruby-test-config 3 1 {:flush-timeout 0}) ;; Set up the pool manually, since we trigger a shutdown in this test, ;; and the macro would also try to shutdown. It's not a valid operation ;; to shutdown twice. pool-context (jruby-pool-manager-core/create-pool-context config) _ (jruby-agents/prime-pool! pool-context) _ (jruby-core/get-pool pool-context) _ (jruby-testutils/wait-for-jrubies-from-pool-context pool-context)] ;; Borrow an instance so that the lock can't be acquired (pool-protocol/borrow pool-context) (is (thrown+? [:kind :puppetlabs.services.jruby-pool-manager.impl.jruby-internal/jruby-lock-timeout :msg "An attempt to lock the JRubyPool failed with a timeout"] (pool-protocol/shutdown pool-context)))))) jruby_service_test.clj000066400000000000000000000217041464330533300350240ustar00rootroot00000000000000puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager(ns puppetlabs.services.jruby-pool-manager.jruby-service-test (:require [clojure.test :refer :all] [puppetlabs.services.jruby-pool-manager.jruby-testutils :as jruby-testutils] [puppetlabs.trapperkeeper.core :as tk] [puppetlabs.trapperkeeper.services :as services] [clojure.stacktrace :as stacktrace] [puppetlabs.trapperkeeper.testutils.bootstrap :as tk-bootstrap] [puppetlabs.trapperkeeper.testutils.logging :as logging] [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [schema.test :as schema-test]) (:import (puppetlabs.services.jruby_pool_manager.jruby_schemas JRubyInstance))) (use-fixtures :once schema-test/validate-schemas) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Trapperkeeper Service (tk/defservice jruby-pooled-test-service [[:ConfigService get-config] [:ShutdownService shutdown-on-error] [:PoolManagerService create-pool]] (init [this context] (let [initial-config (get-config) service-id (services/service-id this) agent-shutdown-fn (partial shutdown-on-error service-id) config (jruby-core/initialize-config (assoc-in initial-config [:lifecycle :shutdown-on-error] agent-shutdown-fn))] (let [pool-context (create-pool config)] (-> context (assoc :pool-context pool-context) (assoc :event-callbacks (atom [])))))) (stop [this context] (let [{:keys [pool-context]} (services/service-context this)] (jruby-core/flush-pool-for-shutdown! pool-context)) context)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Tests (defn jruby-test-config [pool-size] (jruby-testutils/jruby-config {:max-active-instances pool-size})) (deftest test-error-during-init (testing (str "If there is an exception while putting a JRubyInstance in " "the pool the application should shut down.") (logging/with-test-logging (let [got-expected-exception (atom false)] (try (tk-bootstrap/with-app-with-config app (conj jruby-testutils/default-services jruby-pooled-test-service) (jruby-testutils/jruby-config {:max-active-instances 1 :lifecycle {:initialize-pool-instance (fn [_] (throw (Exception. "42")))}}) (tk/run-app app)) (catch Exception e (let [cause (stacktrace/root-cause e)] (is (= (.getMessage cause) "42")) (reset! got-expected-exception true)))) (is (true? @got-expected-exception) "Did not get expected exception."))))) (deftest test-pool-size (testing "The pool is created and the size is correctly reported" (let [pool-size 2] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config pool-size) (let [pool (jruby-core/get-pool pool-context) all-the-instances (mapv (fn [_] (jruby-core/borrow-from-pool-with-timeout pool-context :test-pool-size [])) (range pool-size))] (is (= 0 (jruby-core/free-instance-count pool))) (is (= pool-size (count all-the-instances))) (doseq [instance all-the-instances] (is (not (nil? instance)) "One of the JRubyInstances retrieved from the pool is nil") (jruby-core/return-to-pool pool-context instance :test-pool-size [])) (is (= pool-size (jruby-core/free-instance-count pool)))))))) (deftest test-with-jruby-instance (testing "the `with-jruby-instance macro`" (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances 1}) (jruby-core/with-jruby-instance jruby-instance pool-context :test-with-jruby-instance (is (instance? JRubyInstance jruby-instance)) (is (= 0 (jruby-core/free-instance-count (jruby-core/get-pool pool-context))))) (is (= 1 (jruby-core/free-instance-count (jruby-core/get-pool pool-context)))) ;; borrow and return one more time: we're using `with-jruby-instance` ;; here even though it looks a bit strange, because that is what this ;; test is intended to cover. (jruby-core/with-jruby-instance jruby-instance pool-context :test-with-jruby-instance) (let [jruby (jruby-core/borrow-from-pool-with-timeout pool-context :test-with-jruby-instance [])] ;; the counter gets incremented when the instance is returned to the ;; pool, so right now it should be at 2 since we've called ;; `with-jruby-instance` twice. (is (= 2 (:borrow-count (jruby-core/get-instance-state jruby)))) (jruby-core/return-to-pool pool-context jruby :test-with-jruby-instance []))))) (deftest test-jruby-events (testing "jruby service sends event notifications" (let [counter (atom 0) requested (atom {}) borrowed (atom {}) returned (atom {}) callback (fn [{:keys [type reason requested-event instance] :as event}] (case type :instance-requested (reset! requested {:sequence (swap! counter inc) :event event :reason reason}) :instance-borrowed (reset! borrowed {:sequence (swap! counter inc) :reason reason :requested-event requested-event :instance instance}) :instance-returned (reset! returned {:sequence (swap! counter inc) :reason reason :instance instance})))] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-test-config 1) (jruby-core/register-event-handler pool-context callback) ;; We're making an empty call to `with-jruby-instance` here, because ;; we want to trigger a borrow/return via the same code path that ;; would be used in production. (jruby-core/with-jruby-instance jruby-instance pool-context :test-jruby-events) (is (= {:sequence 1 :reason :test-jruby-events} (dissoc @requested :event))) (is (= {:sequence 2 :reason :test-jruby-events} (dissoc @borrowed :instance :requested-event))) (is (jruby-schemas/jruby-instance? (:instance @borrowed))) (is (identical? (:event @requested) (:requested-event @borrowed))) (is (= {:sequence 3 :reason :test-jruby-events} (dissoc @returned :instance))) (is (= (:instance @borrowed) (:instance @returned))) (jruby-core/with-jruby-instance jruby-instance pool-context :test-jruby-events) (is (= 4 (:sequence @requested))) (is (= 5 (:sequence @borrowed))) (is (= 6 (:sequence @returned))))))) (deftest test-borrow-timeout-configuration (testing "configured :borrow-timeout is honored by the borrow-instance-with-timeout function" (let [timeout 250 pool-size 1] (jruby-testutils/with-pool-context pool-context jruby-testutils/default-services (jruby-testutils/jruby-config {:max-active-instances pool-size :borrow-timeout timeout}) (let [jrubies (jruby-testutils/drain-pool pool-context pool-size)] (is (= 1 (count jrubies))) (is (every? jruby-schemas/jruby-instance? jrubies)) (let [test-start-in-millis (System/currentTimeMillis)] (is (nil? (jruby-core/borrow-from-pool-with-timeout pool-context :test-borrow-timeout-configuration []))) (is (>= (- (System/currentTimeMillis) test-start-in-millis) timeout)) (is (= (:borrow-timeout (:config pool-context)) timeout))) ; Test cleanup. This instance needs to be returned so that the stop can complete. (doseq [inst jrubies] (jruby-core/return-to-pool pool-context inst :test [])))))) (testing (str ":borrow-timeout defaults to " jruby-core/default-borrow-timeout " milliseconds") (let [initial-config {:ruby-load-path ["foo"] :gem-home "bar"} config (jruby-core/initialize-config initial-config)] (is (= (:borrow-timeout config) jruby-core/default-borrow-timeout))))) puppetlabs-jruby-utils-ca5d27b/test/unit/puppetlabs/services/jruby_pool_manager/jruby_testutils.clj000066400000000000000000000130251464330533300344410ustar00rootroot00000000000000(ns puppetlabs.services.jruby-pool-manager.jruby-testutils (:require [puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core] [puppetlabs.services.jruby-pool-manager.jruby-schemas :as jruby-schemas] [puppetlabs.services.jruby-pool-manager.impl.jruby-internal :as jruby-internal] [puppetlabs.services.jruby-pool-manager.jruby-pool-manager-service :as pool-manager] [puppetlabs.trapperkeeper.app :as tk-app] [clojure.tools.logging :as log] [schema.core :as schema] [puppetlabs.services.protocols.pool-manager :as pool-manager-protocol] [puppetlabs.trapperkeeper.testutils.bootstrap :as tk-bootstrap]) (:import (clojure.lang IFn))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Constants (def ruby-load-path []) (def gem-home "./target/jruby-gem-home") (def default-services [pool-manager/jruby-pool-manager-service]) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; JRuby Test util functions (schema/defn ^:always-validate jruby-config :- jruby-schemas/JRubyConfig "Create a JRubyConfig for testing. The optional map argument `options` may contain a map, which, if present, will be merged into the final JRubyConfig map. (This function differs from `jruby-tk-config` in that it returns a map that complies with the JRubyConfig schema, which differs slightly from the raw format that would be read from config files on disk.)" ([] (jruby-core/initialize-config {:ruby-load-path ruby-load-path :gem-home gem-home})) ([options] (jruby-core/initialize-config (merge {:ruby-load-path ruby-load-path :gem-home gem-home :borrow-timeout 300000} options)))) (defn drain-pool "Drains the JRuby pool and returns each instance in a vector." [pool-context size] (mapv (fn [_] (jruby-core/borrow-from-pool pool-context :test [])) (range size))) (defn fill-drained-pool "Returns a list of JRubyInstances back to their pool." [pool-context instance-list] (doseq [instance instance-list] (jruby-core/return-to-pool pool-context instance :test []))) (schema/defn ^:always-validate drain-and-refill :- #{schema/Int} "Borrow all instances from a pool and then hand them back. Return a set (order is not important) of the instance ids borrowed/returned" [pool-context :- jruby-schemas/PoolContext] (let [instance-count (get-in pool-context [:config :max-active-instances]) instances (drain-pool pool-context instance-count) ids (into #{} (map :id instances))] (fill-drained-pool pool-context instances) ids)) (defn reduce-over-jrubies! "Utility function; takes a JRuby pool and size, and a function f from integer to string. For each JRubyInstance in the pool, f will be called, passing in an integer offset into the jruby array (0..size), and f is expected to return a string containing a script to run against the jruby instance. Returns a vector containing the results of executing the scripts against the JRubyInstances." [pool-context size f] (let [jrubies (drain-pool pool-context size) result (reduce (fn [acc jruby-offset] (let [sc (:scripting-container (nth jrubies jruby-offset)) script (f jruby-offset) result (.runScriptlet sc script)] (conj acc result))) [] (range size))] (fill-drained-pool pool-context jrubies) result)) (defn wait-for-jrubies-from-pool-context "Wait for all jrubies to land in the pool" [pool-context] (let [num-jrubies (-> pool-context jruby-core/get-pool-state :size)] (while (< (count (jruby-core/registered-instances pool-context)) num-jrubies) (Thread/yield)))) (defn timed-await [agent] (await-for 240000 agent)) (schema/defn wait-for-predicate :- schema/Bool "Wait for some predicate to return true defaults to 20 iterations of 500 ms, ~10 seconds Returns true if f ever returns true, false otherwise" ([f :- IFn] (wait-for-predicate f 20 500)) ([f :- IFn max-iterations :- schema/Int sleep-time :- schema/Int] (loop [count 0] (cond (f) true (>= count max-iterations) (do (log/debugf "Waiting for predicate failed after %s tries" max-iterations) false) :default (do (Thread/sleep sleep-time) (recur (inc count))))))) (schema/defn wait-for-instances :- schema/Bool [pool :- jruby-schemas/pool-queue-type num-instances :- schema/Int] (wait-for-predicate #(= num-instances (jruby-core/free-instance-count pool)))) (schema/defn wait-for-pool-to-be-locked :- schema/Bool [pool :- jruby-schemas/pool-queue-type] (wait-for-predicate #(.isLocked pool))) (defmacro with-pool-context [pool-context services config & body] `(tk-bootstrap/with-app-with-config app# ~services {} (let [pool-manager-service# (tk-app/get-service app# :PoolManagerService) ~pool-context (pool-manager-protocol/create-pool pool-manager-service# ~config)] (try ~@body (finally (jruby-core/flush-pool-for-shutdown! ~pool-context)))))) (defmacro with-scripting-container [container config & body] `(let [~container (jruby-internal/create-scripting-container ~config)] (try ~@body (finally (.terminate ~container)))))