mcollective-server-provisioner-0.0.1~git20110120/0000755000175000017500000000000011600466611021651 5ustar roaksoaxroaksoaxmcollective-server-provisioner-0.0.1~git20110120/Rakefile0000644000175000017500000000460011600465614023320 0ustar roaksoaxroaksoax# Rakefile to build a project using HUDSON require 'rake/packagetask' require 'rake/clean' require 'find' PROJ_DOC_TITLE = "MCollective Server Provisioner" PROJ_VERSION = "1.0.0" PROJ_RELEASE = "1" PROJ_NAME = "mcprovision" PROJ_RPM_NAMES = [PROJ_NAME] PROJ_FILES = ["#{PROJ_NAME}.spec", "#{PROJ_NAME}.rb", "#{PROJ_NAME}.init", "COPYING", "lib", "etc", "agent"] ENV["RPM_VERSION"] ? CURRENT_VERSION = ENV["RPM_VERSION"] : CURRENT_VERSION = PROJ_VERSION ENV["BUILD_NUMBER"] ? CURRENT_RELEASE = ENV["BUILD_NUMBER"] : CURRENT_RELEASE = PROJ_RELEASE CLEAN.include("build") def announce(msg='') STDERR.puts "================" STDERR.puts msg STDERR.puts "================" end def init FileUtils.mkdir("build") unless File.exist?("build") end desc "Build tar balls and rpms" task :default => [:clean, :package, :rpm] desc "Create a tarball for this release" task :package => [:clean] do announce "Creating #{PROJ_NAME}-#{CURRENT_VERSION}.tgz" FileUtils.mkdir_p("build/#{PROJ_NAME}-#{CURRENT_VERSION}") system("cp -R #{PROJ_FILES.join(' ')} build/#{PROJ_NAME}-#{CURRENT_VERSION}") system("cd build && tar --exclude .svn --exclude .git -cvzf #{PROJ_NAME}-#{CURRENT_VERSION}.tgz #{PROJ_NAME}-#{CURRENT_VERSION}") end desc "Creates a RPM" task :rpm => [:clean, :package] do announce("Building RPM for #{PROJ_NAME}-#{CURRENT_VERSION}-#{CURRENT_RELEASE}") sourcedir = `rpm --eval '%_sourcedir'`.chomp specsdir = `rpm --eval '%_specdir'`.chomp srpmsdir = `rpm --eval '%_srcrpmdir'`.chomp rpmdir = `rpm --eval '%_rpmdir'`.chomp lsbdistrel = `lsb_release -r -s | cut -d . -f1`.chomp lsbdistro = `lsb_release -i -s`.chomp case lsbdistro when 'CentOS' rpmdist = ".el#{lsbdistrel}" else rpmdist = "" end system %{cp build/#{PROJ_NAME}-#{CURRENT_VERSION}.tgz #{sourcedir}} system %{cat #{PROJ_NAME}.spec|sed -e s/%{rpm_release}/#{CURRENT_RELEASE}/g | sed -e s/%{version}/#{CURRENT_VERSION}/g > #{specsdir}/#{PROJ_NAME}.spec} system %{cd #{specsdir} && rpmbuild -D 'version #{CURRENT_VERSION}' -D 'rpm_release #{CURRENT_RELEASE}' -D 'dist #{rpmdist}' -ba #{PROJ_NAME}.spec} system %{cp #{srpmsdir}/#{PROJ_NAME}-#{CURRENT_VERSION}-#{CURRENT_RELEASE}#{rpmdist}.src.rpm build/} system %{cp #{rpmdir}/*/#{PROJ_NAME}*-#{CURRENT_VERSION}-#{CURRENT_RELEASE}#{rpmdist}.*.rpm build/} end # vi:tabstop=4:expandtab:ai mcollective-server-provisioner-0.0.1~git20110120/mcprovision.init0000755000175000017500000000454511600465614025123 0ustar roaksoaxroaksoax#!/bin/sh # # mcprovision Server Provisioner for The Marionette Collective # # chkconfig: 345 24 76 # # description: Automated the provisioning of servers # ### BEGIN INIT INFO # Provides: mcprovision # Required-Start: $remote_fs # Required-Stop: $remote_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start daemon at boot time # Description: Enable service provided by daemon. ### END INIT INFO mcprovision="/usr/sbin/mcprovision" pidfile="/var/run/mcprovision.pid" # Lockfile if [ -d /var/lock/subsys ]; then # RedHat/CentOS/etc who use subsys lock="/var/lock/subsys/mcprovision" else # The rest of them lock="/var/lock/mcprovision" fi # Source function library. . /etc/init.d/functions # Load options, set things like PSK's and SSL keys in here if [ -f /etc/sysconfig/mcprovision ]; then . /etc/sysconfig/mcprovision fi # Check that binary exists if ! [ -f $mcprovision ] then echo "$mcprovision binary not found" exit 0 fi # See how we were called. case "$1" in start) echo -n "Starting mcprovision: " if [ -f ${lock} ]; then # we were not shut down correctly if [ -s ${pidfile} ]; then kill `cat ${pidfile}` >/dev/null 2>&1 fi rm -f ${pidfile} rm -f ${lock} sleep 2 fi rm -f ${pidfile} ${mcprovision} /etc/mcollective/mcprovision.yaml if [ $? = 0 ]; then success $"mcprovision" touch $lock echo exit 0 else failure echo exit 1 fi ;; stop) echo -n "Shutting down mcprovision: " if [ -s ${pidfile} ]; then kill `cat ${pidfile}` >/dev/null 2>&1 fi rm -f ${pidfile} success $"mcprovision" rm -f $lock echo ;; restart) $0 stop sleep 2 $0 start ;; condrestart) if [ -f $lock ]; then $0 stop # avoid race sleep 2 $0 start fi ;; status) if [ -f ${lock} ]; then if [ -s ${pidfile} ]; then if [ -e /proc/`cat ${pidfile}` ]; then echo "mcprovision (`cat ${pidfile}`) is running" exit 0 else echo "mcprovision (`cat ${pidfile}`) is NOT running" exit 1 fi fi else echo "mcprovision: service not started" exit 1 fi ;; force-reload) echo "not implemented" ;; *) echo "Usage: mcprovision {start|stop|restart|condrestart|status}" exit 1 ;; esac exit 0 mcollective-server-provisioner-0.0.1~git20110120/lib/0000755000175000017500000000000011600465614022421 5ustar roaksoaxroaksoaxmcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision.rb0000644000175000017500000000241311600465614025316 0ustar roaksoaxroaksoaxrequire 'mcollective' require 'yaml' require 'logger' require 'pp' include MCollective::RPC module MCProvision autoload :Config, "mcprovision/config" autoload :PuppetMaster, "mcprovision/puppetmaster" autoload :Nodes, "mcprovision/nodes" autoload :Node, "mcprovision/node" autoload :Util, "mcprovision/util" autoload :Runner, "mcprovision/runner" autoload :Notifier, "mcprovision/notifier" VERSION = "1.0.0" def self.logfile(logfile, loglevel) @@logfile = logfile @@logger = Logger.new(logfile, 5, 102400) case loglevel when "debug" @@logger.level = Logger::DEBUG when "warn" @@logger.level = Logger::WARN else @@logger.level = Logger::INFO end end def self.version VERSION end def self.warn(msg) MCProvision.log(Logger::WARN, msg) end def self.info(msg) MCProvision.log(Logger::INFO, msg) end def self.debug(msg) MCProvision.log(Logger::DEBUG, msg) end def self.log(severity, msg) begin from = File.basename(caller[1]) @@logger.add(severity) { "#{$$} #{from}: #{msg}" } rescue Exception => e end end end mcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/0000755000175000017500000000000011600465614024771 5ustar roaksoaxroaksoaxmcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/runner.rb0000644000175000017500000001615011600465614026632 0ustar roaksoaxroaksoaxmodule MCProvision class Runner attr_reader :config def initialize(configfile) @config = MCProvision::Config.new(configfile) @master = MCProvision::PuppetMaster.new(@config) @notifier = Notifier.new(@config) Signal.trap("INT") do MCProvision.info("Received INT signal, exiting.") exit! end end def run begin MCProvision.info("Starting runner") loop do MCProvision.info("Looking for machines to provision") provisionable = Nodes.new(@config.settings["target"]["agent"], @config.settings["target"]["filter"], @config) provisionable.nodes.each do |server| begin provision(server) rescue Exception => e MCProvision.warn("Could not provision node #{server.hostname}: #{e.class}: #{e}") MCProvision.warn(e.backtrace.join("\n\t")) if @config.settings["loglevel"] == "debug" end end sleep @config.settings["sleeptime"] || 5 end rescue SignalException => e rescue Exception => e MCProvision.warn("Runner failed: #{e.class}: #{e}") MCProvision.warn(e.backtrace.join("\n\t")) if @config.settings["loglevel"] == "debug" sleep 2 retry end end # Main provisioner body, does the following: # # - Find the node ip address based on target/ipaddress_fact # - picks a puppet master based on configured criteria # - determines the ip address of the picked master # - creates a lock file on the node so no other provisioner threads will interfere with it # - calls to the set_puppet_hostname action which typically adds 'puppet' to /etc/hosts # - checks if the node already has a cert # - if it doesnt # - clean the cert from all masters # - instructs the client to do a run which would request the cert # - signs it on all masters # - call puppet_bootstrap_stage which could run a small bootstrap environment client # - call puppet_final_run which would do a normal puppet run, this steps block till completed # - deletes the lock file # - sends a notification to administrators def provision(node) node_inventory = node.inventory node_ipaddress_fact = @config.settings["target"]["ipaddress_fact"] || "ipaddress" master_ipaddress_fact = @config.settings["master"]["ipaddress_fact"] || "ipaddress" begin raise "Could not determine node ip address from fact #{node_ipaddress_fact}" unless node_inventory[:facts].include?(node_ipaddress_fact) rescue StandardError => e raise "Node didn't reply on allocated time" end steps = @config.settings["steps"].keys.select{|s| @config.settings["steps"][s] } chosen_master, master_inventory = pick_master_from(@config.settings["master"]["criteria"], node_inventory[:facts]) begin raise "Could not determine master ip address from fact #{master_ipaddress_fact}" unless master_inventory[:facts].include?(master_ipaddress_fact) rescue StandardError => e raise "Node didn't reply on allocated time" end master_ip = master_inventory[:facts][master_ipaddress_fact] MCProvision.info("Potential provisioning for #{node.hostname}") # Only do certificate management if the node is clean and doesnt already have a cert unless node.has_cert? MCProvision.info("Provisioning #{node.hostname} / #{node_inventory[:facts][node_ipaddress_fact]} with steps #{steps.join ' '}") MCProvision.info("Provisioning node against #{chosen_master.hostname} / #{master_ip}") node.lock if @config.settings["steps"]["lock"] node.stop_puppet if @config.settings["steps"]["stop_puppet"] node.set_puppet_host(master_ip) if @config.settings["steps"]["set_puppet_hostname"] node.clean_cert if @config.settings["steps"]["clean_node_certname"] @master.clean_cert(node.hostname.downcase) if @config.settings["steps"]["clean_node_certname"] node.send_csr if @config.settings["steps"]["send_node_csr"] @master.sign(node.hostname.downcase) if @config.settings["steps"]["sign_node_csr"] node.cycle_puppet_run if @config.settings["steps"]["cycle_puppet_run"] node.bootstrap if @config.settings["steps"]["puppet_bootstrap_stage"] node.run_puppet if @config.settings["steps"]["puppet_final_run"] node.start_puppet if @config.settings["steps"]["start_puppet"] node.fact_mod("provision-status","provisioned") if @config.settings["steps"]["set_role_provisioned"] node.unlock if @config.settings["steps"]["unlock"] MCProvision.info("Node #{node.hostname} provisioned") else MCProvision.info("Node is already provisioned") end @notifier.notify("Provisioned #{node.hostname} against #{chosen_master.hostname}", "New Node") if @config.settings["steps"]["notify"] end private # Take an array of facts and the node facts. # Discovers all masters and go through their inventories # till we find a match, else return the first one. def pick_master_from(facts, node) masters = @master.find_all chosen_master = masters.first master_inventories = {} # build up a list of the master inventories masters.each do |master| master_inventories[master.hostname] = master.inventory end # For every configured fact begin facts.each do |fact| # Check if the node has it if node.include?(fact) # Now check every master masters.each do |master| master_facts = master_inventories[master.hostname][:facts] if master_facts.include?(fact) # if they match, we have a winner if master_facts[fact] == node[fact] MCProvision.info("Picking #{master.hostname} for puppetmaster based on #{fact} == #{node[fact]}") chosen_master = master end end end end end rescue end raise "Could not find any masters" if chosen_master.nil? return [chosen_master, master_inventories[chosen_master.hostname]] end end end mcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/config.rb0000644000175000017500000000101711600465614026562 0ustar roaksoaxroaksoaxmodule MCProvision class Config attr_reader :settings def initialize(configfile = "/etc/mcollective/provisioner.yaml") Util.log("Loading config from #{configfile}") @settings = YAML.load_file(configfile) if @settings.include?("logfile") && @settings.include?("loglevel") MCProvision.logfile(@settings["logfile"], @settings["loglevel"]) else MCProvision.logfile("/dev/stderr", "debug") end end end end mcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/node.rb0000644000175000017500000001262511600465614026251 0ustar roaksoaxroaksoaxmodule MCProvision class Node attr_reader :hostname, :inventory def initialize(hostname, config, agent) @config = config @hostname = hostname @agent = agent setup @inventory = fetch_inventory end def lock MCProvision.info("Creating lock file on node") request("lock_deploy") end def unlock MCProvision.info("Removing lock file on node") request("unlock_deploy") end # Check if the lock file exist def locked? MCProvision.info("Checking if the deploy is locked on this node") result = request("is_locked") result[:data][:locked] end # Do we already have a puppet cert? def has_cert? MCProvision.info("Finding out if we already have a certificate") result = request("has_cert") result[:data][:has_cert] end # Do we already have a puppet cert? def provisioned? MCProvision.info("Finding out if we already finished provisioning") result = request("provisioned") result[:data][:provisioned] end # sets the ip of the puppet master host using the # set_puppet_host action on the node def set_puppet_host(ipaddress) MCProvision.info("Calling set_puppet_host with ip #{ipaddress}") request("set_puppet_host", {:ipaddress => ipaddress}) end # calls the request_certificate action on the node being provisioned def send_csr MCProvision.info("Calling request_certificate") request("request_certificate") end # calls the bootstrap_puppet action to do initial puppet run def bootstrap MCProvision.info("Calling bootstrap_puppet") result = request("bootstrap_puppet") check_puppet_output(result[:data][:output].split("\n")) end # calls the clean_cert to clean certificate on remote side def clean_cert MCProvision.info("Calling clean_cert") result = request("clean_cert") end # calls stop_puppet to stop puppet service def stop_puppet MCProvision.info("Calling stop_puppet") result = request("stop_puppet") end # calls start_puppet to start puppet service def start_puppet MCProvision.info("Calling start_puppet") result = request("start_puppet") end # Do the final run of the client by calling run_puppet def run_puppet MCProvision.info("Calling run_puppet") result = request("run_puppet") end # Do cycle puppet run def cycle_puppet_run MCProvision.info("Calling cycle_puppet_run") result = request("cycle_puppet_run") end # Modify or add facts to client def fact_mod(fact,value) MCProvision.info("Calling fact_add with fact #{fact} and value #{value}") result = request("fact_mod", {:fact => fact, :value => value}) end private # Wrapper that calls to a node, checks the result structure and status messages and return # the result structure for the node def request(action, arguments={}) begin result = @node.custom_request(action, arguments, @hostname, {"identity" => @hostname}) rescue StandardError => e action = "unlock_deploy" arguments = "" @node.custom_request(action, arguments, @hostname, {"identity" => @hostname}) end raise "Uknown result from remote node: #{result.pretty_inspect}" unless result.is_a?(Array) #MCProvision.info("debug response: #{result.pretty_inspect}" raise "Did not receive a response from #{@hostname} in the allowed time" if result.empty? result = result.first unless result[:statuscode] == 0 raise "Request to #{@hostname}##{action} failed: #{result[:statusmsg]}" end result end # checks output from puppetd that ran with --summarize for errors def check_puppet_output(output) output.each do |o| if o =~ /^\s+Failed: (\d+)/ raise "Puppet failed due to #{$1} failed resource(s)" unless $1 == "0" end if o =~ /^\s+Skipped: (\d+)/ raise "Puppet failed due to #{$1} skipped resource(s)" unless $1 == "0" end end end # Gets the inventory from the discovery agent on the node def fetch_inventory result = {} # Does a MC::Client request to the main discovery agent, we should use the # rpcutil agent for this @node.client.req("inventory", "discovery", @node.client.options, 1) do |resp| result[:agents] = resp[:body][:agents] result[:facts] = resp[:body][:facts] result[:classes] = resp[:body][:classes] end result end def setup @node = rpcclient(@agent) @node.identity_filter @hostname @node.progress = false #MCProvision.info(@node.options.pretty_inspect) end end end mcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/nodes.rb0000644000175000017500000000113111600465614026422 0ustar roaksoaxroaksoaxmodule MCProvision class Nodes attr_reader :nodes, :filter, :agent def initialize(agent, filter, config) @filter = filter @agent = agent @config = config setup find_all end private def find_all @nodes = @rpc.discover.map do |node| Node.new(node, @config, @agent) end end def setup @rpc = rpcclient(@agent) @rpc.filter = Util.parse_filter(@agent, @filter) @rpc.progress = false end end end mcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/notifier.rb0000644000175000017500000000247211600465614027142 0ustar roaksoaxroaksoaxmodule MCProvision class Notifier def initialize(config) @config = config if @config.settings.include?("notify") setup end end def notify(msg, subject) if @config.settings.include?("notify") raise "No notification targets specified" unless @config.settings["notify"].include?("targets") raise "No notification targets specified" if @config.settings["notify"]["targets"].empty? @config.settings["notify"]["targets"].each do |recipient| MCProvision.info("Notifying #{recipient} of new node") raise "Could not find any instances of the '#{@config.settings['notify']['agent']}' agent" if @rpc.discover.empty? @rpc.sendmsg(:message => msg, :subject => subject, :recipient => recipient) end end end private def setup agent = @config.settings["notify"]["agent"] || "angelianotify" @rpc = rpcclient(agent) @rpc.progress = false # in environments with many notifiers running only speak to 1 @rpc.limit_targets = 1 @rpc.filter = Util.parse_filter(agent, @config.settings["notify"]["filter"]) end end end mcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/util.rb0000644000175000017500000000175211600465614026300 0ustar roaksoaxroaksoaxmodule MCProvision::Util # Parses a -W style filter and return a std filter option def self.parse_filter(agent, filter) result = MCollective::Util.empty_filter filter.split(" ").each do |f| if f =~ /^(.+?)=(.+)/ result["fact"] << {:fact => $1, :value => $2} else result["cf_class"] << f end end result["agent"] << agent result end # Daemonize the current process def self.daemonize fork do Process.setsid exit if fork Dir.chdir('/tmp') STDIN.reopen('/dev/null') STDOUT.reopen('/dev/null', 'a') STDERR.reopen('/dev/null', 'a') if File.directory?("/var/run") File.open("/var/run/mcprovision.pid", 'w') {|f| f.write(Process.pid) } rescue true end yield end end def self.log(msg) MCProvision.debug(msg) end end mcollective-server-provisioner-0.0.1~git20110120/lib/mcprovision/puppetmaster.rb0000644000175000017500000000316711600465614030056 0ustar roaksoaxroaksoaxmodule MCProvision class PuppetMaster def initialize(config) @config = config setup end # Find all nodes with the configured master agent def find_all masters = Nodes.new(@config.settings["master"]["agent"], @config.settings["master"]["filter"], @config) masters.nodes end # Cleans a cert from all masters def clean_cert(certname) MCProvision.info("Clean certificate #{certname} from all masters") @puppetca.clean(:certname => certname) end # Signs a cert on all masters def sign(certname) MCProvision.info("Signing certificate #{certname} on all masters") @puppetca.list.each do |list| if list[:data][:requests].include?(certname) @puppetca.sign(:certname => certname) return end end raise "Could not find certificate #{certname} to sign on any master" end # resets the rpc client def reset @puppetca.reset end private def setup unless @config.settings["master"].include?("agent") && @config.settings["master"].include?("filter") raise "The config file master section needs 'agent' and 'filter'" end agent = @config.settings["master"]["agent"] @puppetca = rpcclient(agent) @puppetca.progress = false @puppetca.filter = MCProvision::Util.parse_filter(agent, @config.settings["master"]["filter"]) end end end mcollective-server-provisioner-0.0.1~git20110120/mcprovision.rb0000644000175000017500000000073411600465614024554 0ustar roaksoaxroaksoax#!/usr/bin/ruby require 'mcprovision' if ARGV.size > 0 configfile = ARGV[0] else configfile = "/etc/mcollective/provisioner.yaml" end runner = MCProvision::Runner.new(configfile) begin if runner.config.settings["daemonize"] || false MCProvision::Util.daemonize do runner.run end else runner.run end rescue Exception => e MCProvision.info("Runner failed unexpectedly: #{e.class}: #{e}") sleep 5 retry end mcollective-server-provisioner-0.0.1~git20110120/etc/0000755000175000017500000000000011600465614022426 5ustar roaksoaxroaksoaxmcollective-server-provisioner-0.0.1~git20110120/etc/provisioner.yaml.dist0000644000175000017500000000106111600465614026631 0ustar roaksoaxroaksoax--- logfile: /var/log/mcprovision.log loglevel: debug daemonize: true steps: lock: false set_puppet_hostname: false clean_node_certname: false send_node_csr: false sign_node_csr: false puppet_bootstrap_stage: false puppet_final_run: false unlock: false notify: false master: criteria: - ec2_placement_region - country filter: "" agent: puppetca ipaddress_fact: ipaddress target: filter: "" agent: provision ipaddress_fact: ipaddress notify: filter: "/monitor1/" agent: angelianotify targets: - boxcar://rip@devco.net mcollective-server-provisioner-0.0.1~git20110120/etc/provisioner.yaml0000644000175000017500000000123211600465614025667 0ustar roaksoaxroaksoax--- logfile: /var/log/provision.log loglevel: debug daemonize: false steps: lock: false stop_puppet: true set_puppet_hostname: true clean_node_certname: true send_node_csr: true sign_node_csr: true puppet_bootstrap_stage: false puppet_final_run: false start_puppet: true set_role_provisioned: true cycle_puppet_run: false unlock: false notify: false master: criteria: - ec2_placement_region - country - ipaddress filter: "" agent: puppetca ipaddress_fact: ipaddress target: filter: "" agent: provision ipaddress_fact: ipaddress notify: filter: "/monitor1/" agent: angelianotify targets: - boxcar://rip@devco.net mcollective-server-provisioner-0.0.1~git20110120/etc/mcprovision.defaults0000644000175000017500000000033411600465614026527 0ustar roaksoaxroaksoaxexport MCOLLECTIVE_AES_PRIVATE=/etc/mcollective/mcprovision-private.pem export MCOLLECTIVE_AES_PUBLIC=/etc/mcollective/mcprovision.pem export STOMP_USER=mcprovision export STOMP_PASSWORD=secret export STOMP_SERVER=stomp mcollective-server-provisioner-0.0.1~git20110120/man/0000755000175000017500000000000011600465614022426 5ustar roaksoaxroaksoaxmcollective-server-provisioner-0.0.1~git20110120/man/mcprovision.10000644000175000017500000000146611600465614025067 0ustar roaksoaxroaksoax.TH mcprovision 1 "21 June 2011" mcprovision "mcprovision" .SH NAME mcprovision \- Simple agent to manage services on Ubuntu systems. .SH SYNOPSIS \fBmcprovision\fP .SH DESCRIPTION \fBmcprovision\fP is a provisioner daemon based on mcollective. .SH AUTHOR This manpage was written by Marc Cluet for Ubuntu systems (but may be used by others). Permission is granted to copy, distribute and/or modify this document and the utility under the terms of the GNU General Public License, Version 3 published by the Free Software Foundation. The complete text of the GNU General Public License can be found in \fI/usr/share/common-licenses/GPL\fP on Debian/Ubuntu systems, or in \fI/usr/share/doc/fedora-release-*/GPL\fP on Fedora systems, or on the web at \fIhttp://www.gnu.org/licenses/gpl.txt\fP. mcollective-server-provisioner-0.0.1~git20110120/mcprovision.spec0000644000175000017500000000467611600465614025114 0ustar roaksoaxroaksoax%define ruby_sitelib %(ruby -rrbconfig -e "puts Config::CONFIG['sitelibdir']") %define release %{rpm_release}%{?dist} Summary: Server Provisioner for Puppet and MCollective Name: mcprovision Version: %{version} Release: %{release} Group: System Tools License: Apache v2 URL: http://www.devco.net/ Source0: %{name}-%{version}.tgz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires: ruby Requires: mcollective-client Packager: R.I.Pienaar BuildArch: noarch %package agent Summary: MCollective Provisioner Agent Group: System Tools Requires: ruby Requires: mcollective %description Automated the provisioning of servers in a Puppet environment via MCollective %description agent Agent providing services for the MCollective Server Provisioner %prep %setup -q %build %install rm -rf %{buildroot} %{__install} -d -m0755 %{buildroot}/%{ruby_sitelib}/mcprovision %{__install} -d -m0755 %{buildroot}/etc/mcollective %{__install} -d -m0755 %{buildroot}/usr/sbin %{__install} -d -m0755 %{buildroot}/etc/init.d %{__install} -d -m0755 %{buildroot}/etc/sysconfig %{__install} -d -m0755 %{buildroot}/usr/libexec/mcollective/mcollective/agent %{__install} -m0755 mcprovision.rb %{buildroot}/usr/sbin/mcprovision %{__install} -m0755 mcprovision.init %{buildroot}/etc/init.d/mcprovision %{__install} -m0644 etc/provisioner.yaml.dist %{buildroot}/etc/mcollective/mcprovision.yaml %{__install} -m0644 agent/provision.ddl %{buildroot}/usr/libexec/mcollective/mcollective/agent/provision.ddl %{__install} -m0644 agent/provision.rb %{buildroot}/usr/libexec/mcollective/mcollective/agent/provision.rb %{__install} -m0600 etc/mcprovision.defaults %{buildroot}/etc/sysconfig/mcprovision cp -R lib/* %{buildroot}/%{ruby_sitelib}/ %clean rm -rf %{buildroot} %post /sbin/chkconfig --add mcprovision %preun if [ "$1" = 0 ]; then /sbin/service mcprovision stop >/dev/null 2>&1 || :; /sbin/chkconfig --del mcprovision || :; fi :; %postun if [ "$1" -ge 1 ]; then /sbin/service mcprovision condrestart >/dev/null 2>&1 || :; fi; :; %files agent /usr/libexec/mcollective/mcollective/agent/provision.rb %files %{ruby_sitelib}/mcprovision.rb %{ruby_sitelib}/mcprovision %config(noreplace) /etc/mcollective/mcprovision.yaml %config(noreplace) /etc/sysconfig/mcprovision /etc/init.d/mcprovision /usr/sbin/mcprovision /usr/libexec/mcollective/mcollective/agent/provision.ddl %changelog * Thu Feb 11 2011 R.I.Pienaar - 1.0.0 - First release mcollective-server-provisioner-0.0.1~git20110120/COPYING0000644000175000017500000002613611600465614022716 0ustar roaksoaxroaksoax Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2011 R.I.Pienaar Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. mcollective-server-provisioner-0.0.1~git20110120/ChangeLog0000644000175000017500000000027011600466611023422 0ustar roaksoaxroaksoaxmcollective-server-provisioner (0.0.1~git20110120-0ubuntu1) UNRELEASED; urgency=low * Initial Version -- Andres Rodriguez Wed, 22 Jun 2011 18:26:02 -0400 mcollective-server-provisioner-0.0.1~git20110120/agent/0000755000175000017500000000000011600465614022751 5ustar roaksoaxroaksoaxmcollective-server-provisioner-0.0.1~git20110120/agent/provision.ddl0000644000175000017500000001033211600465614025465 0ustar roaksoaxroaksoaxmetadata :name => "Server Provisioning Agent", :description => "Agent to assist in provisioning new servers", :author => "R.I.Pienaar", :license => "Apache 2.0", :version => "1.1", :url => "http://mcollective-plugins.googlecode.com/", :timeout => 60 action "set_puppet_host", :description => "Update /etc/hosts with the master IP" do display :always input :ipaddress, :prompt => "Master IP Address", :description => "IP Adress of the Puppet Master", :type => :string, :validation => '^\d+\.\d+\.\d+\.\d+$', :optional => false, :maxlength => 15 end action "request_certificate", :description => "Send the CSR to the master" do output :output, :description => "Puppetd Output", :display_as => "Output" output :exitcode, :description => "Puppetd Exit Code", :display_as => "Exit Code" end action "bootstrap_puppet", :description => "Runs the Puppet bootstrap environment" do output :output, :description => "Puppetd Output", :display_as => "Output" output :exitcode, :description => "Puppetd Exit Code", :display_as => "Exit Code" end action "run_puppet", :description => "Runs Puppet in the normal environment" do output :output, :description => "Puppetd Output", :display_as => "Output" output :exitcode, :description => "Puppetd Exit Code", :display_as => "Exit Code" end action "cycle_puppet_run", :description => "Runs Puppet cycle" do output :output, :description => "cycle_puppet_run Output", :display_as => "Output" output :exitcode, :description => "cycle_puppet_run Exit Code", :display_as => "Exit Code" end action "has_cert", :description => "Finds out if we already have a Puppet certificate" do output :has_cert, :description => "Have a puppet certificate already been created", :display_as => "Has Certificate" end action "provisioned", :description => "Finds out if we already are provisioned" do output :provisioned, :description => "Is the server provisioned", :display_as => "Is Provisioned" end action "lock_deploy", :description => "Lock the deploy so new ones can not be started" do output :lockfile, :description => "The file that got created", :display_as => "Lock file" end action "is_locked", :description => "Determine if the install is currently locked" do output :locked, :description => "Is the install locked", :display_as => "Locked" end action "unlock_deploy", :description => "Unlock the deploy" do output :unlocked, :description => "Has the file been unlocked", :display_as => "Unlocked" end action "clean_cert", :description => "Clean client cert" do output :output, :description => "clean_cert Output", :display_as => "Output" output :exitcode, :description => "clean_cert Exit Code", :display_as => "Exit Code" end action "stop_puppet", :description => "Stop puppet" do output :output, :description => "stop_puppet Output", :display_as => "Output" output :exitcode, :description => "stop_puppet Exit Code", :display_as => "Exit Code" end action "start_puppet", :description => "Start puppet" do output :output, :description => "start_puppet Output", :display_as => "Output" output :exitcode, :description => "start_puppet Exit Code", :display_as => "Exit Code" end action "fact_mod", :description => "Fact Mod" do input :fact, :prompt => "Fact", :description => "Fact Name", :type => :string, :validation => '.', :optional => false, :maxlength => 90 input :value, :prompt => "Value", :description => "Value Name", :type => :string, :validation => '.', :optional => false, :maxlength => 90 output :output, :description => "fact_mod Output", :display_as => "Output" output :exitcode, :description => "fact_mod Exit Code", :display_as => "Exit Code" end mcollective-server-provisioner-0.0.1~git20110120/agent/provision.rb0000644000175000017500000001347411600465614025337 0ustar roaksoaxroaksoaxmodule MCollective module Agent class Provision "Server Provisioning Agent", :description => "Agent to assist in provisioning new servers", :author => "R.I.Pienaar", :license => "Apache 2.0", :version => "2.0", :url => "http://www.devco.net/", :timeout => 60 def startup_hook config = Config.instance certname = PluginManager["facts_plugin"].get_fact("hostname").downcase certname = config.identity unless certname @puppetcert = config.pluginconf["provision.certfile"] || "/var/lib/puppet/ssl/certs/#{certname}.pem" @lockfile = config.pluginconf["provision.lockfile"] || "/tmp/mcollective_provisioner_lock" @puppetd = config.pluginconf["provision.puppetd"] || "/usr/sbin/puppetd" @fact_add = config.pluginconf["provision.fact_add"] || "/usr/bin/fact-add" end action "set_puppet_host" do validate :ipaddress, :ipv4address begin hosts = File.readlines("/etc/hosts") File.open("/etc/hosts", "w") do |hosts_file| hosts.each do |host| hosts_file.puts host unless host =~ /puppet/ end hosts_file.puts "#{request[:ipaddress]}\tpuppet" end rescue Exception => e fail "Could not add hosts entry: #{e}" end end # Adds a new fact action "fact_mod" do validate :fact, :value reply[:exitcode] = run("#{@fact_add} #{request[:fact]} #{request[:value]}", :stdout => :output, :stderr => :err, :chomp => true) if reply[:exitcode] != 0 File.unlink(@lockfile) fail "Fact returned #{reply[:exitcode]}" end end # does a run of puppet with --tags no_such_tag_here action "request_certificate" do reply[:exitcode] = run("#{@puppetd} --test --tags no_such_tag_here --color=none --summarize", :stdout => :output, :stderr => :err, :chomp => true) reply[:exitcode] = 0 # dont fail here if exitcode isnt 0, it'll always be non zero end # does a run of puppet with --environment bootstrap or similar action "bootstrap_puppet" do reply[:exitcode] = run("#{@puppetd} --test --environment bootstrap --color=none --summarize", :stdout => :output, :stderr => :err, :chomp => true) if reply[:exitcode] == 1 File.unlink(@lockfile) fail "Puppet returned #{reply[:exitcode]}" end end # does a normal puppet run action "run_puppet" do reply[:exitcode] = run("#{@puppetd} --test --color=none --summarize", :stdout => :output, :stderr => :err, :chomp => true) reply[:exitcode] = 0 end # start puppet action "start_puppet" do reply[:exitcode] = run("service puppet start", :stdout => :output, :stderr => :err, :chomp => true) if reply[:exitcode] != 0 File.unlink(@lockfile) fail "Puppet returned #{reply[:exitcode]}" end end # cycle_puppet_run action "cycle_puppet_run" do reply[:exitcode] = run("#{@puppetd} --test --color=none --summarize; #{@puppetd} --test --color=none --summarize; #{@puppetd} --test --color=none --summarize", :stdout => :output, :stderr => :err, :chomp => true) reply[:exitcode] = 0 # Even if this errors (it will), we don't care end # stop puppet action "stop_puppet" do reply[:exitcode] = run("service puppet stop", :stdout => :output, :stderr => :err, :chomp => true) if reply[:exitcode] != 0 File.unlink(@lockfile) fail "Puppet returned #{reply[:exitcode]}" end end # clean client cert action "clean_cert" do reply[:exitcode] = run("find /var/lib/puppet/ssl -type f -exec rm {} \+ ", :stdout => :output, :stderr => :err, :chomp => true) reply[:exitcode] = 0 # Even if this errors we don't care end action "has_cert" do reply[:has_cert] = has_cert? end action "provisioned" do isprovisioned = PluginManager["facts_plugin"].get_fact("provision-status") if isprovisioned == "provisioned" reply[:provisioned] = true else reply[:provisioned] = false end end action "lock_deploy" do reply.fail! "Already locked" if locked? File.open(@lockfile, "w") {|f| f.puts Time.now} reply[:lockfile] = @lockfile reply.fail! "Failed to lock the install" unless locked? end action "is_locked" do reply[:locked] = locked? end action "unlock_deploy" do File.unlink(@lockfile) reply[:unlocked] = locked? reply.fail! "Failed to unlock the install" if locked? end private def has_cert? File.exist?(@puppetcert) end def locked? File.exist?(@lockfile) end end end end mcollective-server-provisioner-0.0.1~git20110120/README.markdown0000644000175000017500000000632711600465614024364 0ustar roaksoaxroaksoaxWhat is it? =========== A tool that will take machines running mcollective and a small provisioning agent and bootstrap them through the process of running puppet. Logic Flow ---------- Each node more or less go through the following steps given the config file: --- logfile: /var/log/mcprovision.log loglevel: debug daemonize: true sleeptime: 10 steps: lock: true set_puppet_hostname: true clean_node_certname: true send_node_csr: true sign_node_csr: true puppet_bootstrap_stage: true puppet_final_run: true unlock: true notify: true master: criteria: - ec2_placement_region - country filter: "" agent: puppetca ipaddress_fact: ipaddress target: filter: "" agent: provision ipaddress_fact: ipaddress notify: filter: "" agent: angelianotify targets: - boxcar://you@example.com 1. Discover all nodes running the 'provision' agent 1. Pick the first discovered node and start provisioning 1. Find a list of all masters running 'puppetca' agent 1. Pick a master based on facts ec2_placement_region and then the country facts. If nothing match, take the first 1. Attempts to create a lock on the node to prevent other threads or provisioners from finding this node 1. Check if the node already has a cert and skips certificate steps if it does 1. Calls the 'set_puppet_host' actions giving it the ip of the chosen master 1. Cleans the certificate from all masters matching the identity of the machine being provisioned 1. Gets the node to request a new certificate using the 'request_certificate' action 1. Signs the certificate on all masters 1. Does an initial puppet run with the 'bootstrap_puppet' action 1. Does a 2nd puppet run via 'run_puppet' action. This should remove the provision agent from the node 1. Removes the lock file on the node 1. Notifies my iPhone via boxcar To do this we re-use a lot of existing agents: 1. Your masters all need the puppetca agent 1. Your nodes being provisioned need the provision agent, see agents subdir 1. You need to have angelia deployed and the node should run the angelia agent 1. You need some angelia plugins the boxcar and gcal ones are opensource Customizing ----------- The basic flow is probably generic enough for most bootstrapping scenarios, cloud or non cloud, the specifics of the process should be captured in your agent. There's a sample agent in the agent subdir. You can enable and disable individual steps that doesn't fit your needs in the steps section of the config file Changelog --------- - 2011/02/04 - Improved error handling - 2011/02/04 - Make the facts used to determine ip address connfigurable - 2011/02/04 - Log backtraces when run in debug mode - 2011/02/04 - Make the agent names for all the components configurable - 2011/02/06 - Check if a machine already has a cert and skip cert related steps if it does - 2011/02/06 - Optimise performance of obtaining the node inventory - 2011/02/06 - Add lock and unlock stepts - 2011/02/06 - Make the node code more DRY - 2011/02/06 - When notifying communicate with only 1 of the discovered nodes providing a notification service License ------- Apache 2.0 Contact ------- Contact R.I.Pienaar / @ripienaar / www.devco.net with questions