[svn] commit: r2093 - in /experiments/fujiwara-stats: ./ src/bin/ src/bin/bind10/ src/bin/stats/ src/bin/stats/test/
BIND 10 source code commits
bind10-changes at lists.isc.org
Tue Jun 8 11:54:51 UTC 2010
Author: fujiwara
Date: Tue Jun 8 11:54:50 2010
New Revision: 2093
Log:
Statistics test branch
Now it is controlled by config daemon
Added:
experiments/fujiwara-stats/
- copied from r2090, trunk/
experiments/fujiwara-stats/src/bin/stats/run_statsd.sh.in (with props)
experiments/fujiwara-stats/src/bin/stats/statsd.py.in
experiments/fujiwara-stats/src/bin/stats/statsd.spec
experiments/fujiwara-stats/src/bin/stats/statsd.xml
experiments/fujiwara-stats/src/bin/stats/test/example_agent.py.in
Modified:
experiments/fujiwara-stats/configure.ac
experiments/fujiwara-stats/src/bin/Makefile.am
experiments/fujiwara-stats/src/bin/bind10/bind10.py.in
experiments/fujiwara-stats/src/bin/bind10/run_bind10.sh.in
experiments/fujiwara-stats/src/bin/stats/statsd.py
Modified: experiments/fujiwara-stats/configure.ac
==============================================================================
--- experiments/fujiwara-stats/configure.ac (original)
+++ experiments/fujiwara-stats/configure.ac Tue Jun 8 11:54:50 2010
@@ -399,6 +399,7 @@
src/bin/xfrout/Makefile
src/bin/xfrout/tests/Makefile
src/bin/usermgr/Makefile
+ src/bin/stats/Makefile
src/lib/Makefile
src/lib/cc/Makefile
src/lib/python/Makefile
@@ -432,6 +433,7 @@
src/bin/bind10/bind10.py
src/bin/bind10/tests/bind10_test
src/bin/bind10/run_bind10.sh
+ src/bin/stats/run_statsd.sh
src/bin/bindctl/run_bindctl.sh
src/bin/bindctl/bindctl-source.py
src/bin/bindctl/tests/bindctl_test
@@ -444,6 +446,7 @@
src/bin/msgq/run_msgq.sh
src/bin/auth/auth.spec.pre
src/bin/auth/spec_config.h.pre
+ src/bin/stats/statsd.py
src/lib/config/tests/data_def_unittests_config.h
src/lib/python/isc/config/tests/config_test
src/lib/python/isc/cc/tests/cc_test
@@ -456,6 +459,7 @@
chmod +x src/bin/xfrin/run_b10-xfrin.sh
chmod +x src/bin/xfrout/run_b10-xfrout.sh
chmod +x src/bin/bind10/run_bind10.sh
+ chmod +x src/bin/stats/run_statsd.sh
chmod +x src/bin/cmdctl/tests/cmdctl_test
chmod +x src/bin/xfrin/tests/xfrin_test
chmod +x src/bin/xfrout/tests/xfrout_test
Modified: experiments/fujiwara-stats/src/bin/Makefile.am
==============================================================================
--- experiments/fujiwara-stats/src/bin/Makefile.am (original)
+++ experiments/fujiwara-stats/src/bin/Makefile.am Tue Jun 8 11:54:50 2010
@@ -1,1 +1,1 @@
-SUBDIRS = bind10 bindctl cfgmgr loadzone msgq host cmdctl auth xfrin xfrout usermgr
+SUBDIRS = bind10 bindctl cfgmgr loadzone msgq host cmdctl auth xfrin xfrout usermgr stats
Modified: experiments/fujiwara-stats/src/bin/bind10/bind10.py.in
==============================================================================
--- experiments/fujiwara-stats/src/bin/bind10/bind10.py.in (original)
+++ experiments/fujiwara-stats/src/bin/bind10/bind10.py.in Tue Jun 8 11:54:50 2010
@@ -326,6 +326,25 @@
if self.verbose:
sys.stdout.write("[bind10] Started b10-xfrin (PID %d)\n" % xfrind.pid)
+ # start b10-statsd
+ statsd_args = ['b10-statsd']
+ if self.verbose:
+ sys.stdout.write("Starting b10-statsd\n")
+ statsd_args += ['-v']
+ try:
+ statsd = ProcessInfo("b10-statsd", statsd_args,
+ c_channel_env)
+ except Exception as e:
+ c_channel.process.kill()
+ bind_cfgd.process.kill()
+ xfrout.process.kill()
+ xfrind.process.kill()
+ auth.process.kill()
+ return "Unable to start b10-statsd; " + str(e)
+ self.processes[statsd.pid] = statsd
+ if self.verbose:
+ sys.stdout.write("Started b10-statsd (PID %d)\n" % statsd.pid)
+
# start the b10-cmdctl
# XXX: we hardcode port 8080
cmdctl_args = ['b10-cmdctl']
@@ -341,6 +360,7 @@
xfrout.process.kill()
auth.process.kill()
xfrind.process.kill()
+ statsd.process.kill()
return "Unable to start b10-cmdctl; " + str(e)
self.processes[cmd_ctrld.pid] = cmd_ctrld
if self.verbose:
Modified: experiments/fujiwara-stats/src/bin/bind10/run_bind10.sh.in
==============================================================================
--- experiments/fujiwara-stats/src/bin/bind10/run_bind10.sh.in (original)
+++ experiments/fujiwara-stats/src/bin/bind10/run_bind10.sh.in Tue Jun 8 11:54:50 2010
@@ -20,7 +20,7 @@
BIND10_PATH=@abs_top_builddir@/src/bin/bind10
-PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:$PATH
+PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:@abs_top_builddir@/src/bin/stats:$PATH
export PATH
PYTHONPATH=@abs_top_srcdir@/src/lib/python:@abs_top_builddir@/src/lib/python:@abs_top_builddir@/src/lib/dns/.libs:@abs_top_builddir@/src/lib/xfr/.libs
Modified: experiments/fujiwara-stats/src/bin/stats/statsd.py
==============================================================================
--- experiments/fujiwara-stats/src/bin/stats/statsd.py (original)
+++ experiments/fujiwara-stats/src/bin/stats/statsd.py Tue Jun 8 11:54:50 2010
@@ -1,151 +1,159 @@
-#!/usr/bin/python
+#!/usr/local/bin/python3.1
+
+# Copyright (C) 2010 Internet Systems Consortium.
#
-# This program collects 'counters' from 'statistics' channel.
-# It accepts one command: 'Boss' group 'shutdown'
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""\
+This file implements the statistics collecting program.
+
+This program collects 'counters' from 'statistics' channel.
+"""
+
+import sys; sys.path.append ('@@PYTHONPATH@@')
+import os
+
+# If B10_FROM_SOURCE is set in the environment, we use data files
+# from a directory relative to that, otherwise we use the ones
+# installed on the system
+if "B10_FROM_SOURCE" in os.environ:
+ SPECFILE_LOCATION = os.environ["B10_FROM_SOURCE"] + "/src/bin/stats/statsd.spec"
+else:
+ PREFIX = "/usr/local/bind10"
+ DATAROOTDIR = "${prefix}/share"
+ SPECFILE_LOCATION = "${datarootdir}/bind10-devel/statsd.spec".replace("${datarootdir}", DATAROOTDIR).replace("${prefix}", PREFIX)
+
+import time
+import select
+import errno
+import signal
+import re
import isc.cc
-import time
-import select
-import os
+__version__ = "v20100602"
-bossgroup = 'Boss'
-myname = 'statsd'
-debug = 0
+##############################################################
+# for debugging
+##############################################################
+print ("Started statsd")
-def total(s):
- def totalsub(d,s):
- for k in s.keys():
- if (k == 'component' or k == 'version'
- or k == 'timestamp' or k == 'from'):
- continue
- if (k in d):
- if (isinstance(s[k], dict)):
- totalsub(d[k], s[k])
- else:
- d[k] = s[k] + d[k]
+# TODO: start up statistics thingy
+
+class Statistics:
+ """Statistics Class."""
+
+ def __init__(self, verbose = True):
+ self.debug = 1
+ self.stats = {}
+ self.output_interval = 10
+ self.output_path = '/tmp/stats.xml'
+ self.output_generation = 100
+ self.myname = 'statistics'
+ self.verbose = verbose
+ self.shutdown = 0
+ self.wrote_time = 0
+
+ def stats_update(self, arg):
+ if self.debug:
+ print ("[statsd] stats_update: "+str(arg))
+ if (not ('component' in arg and 'from' in arg)):
+ return 1
+ self.stats[arg['component']] = arg
+ if self.debug:
+ print ("received from=",arg['from'], " component=", arg['component'])
+ print (str(arg))
+ return 0
+ #if last_recvd_time > wrote_time:
+ # if debug:
+ # print ("dump stats")
+ # dump_stats(statpath, statcount, statstotal, stats)
+
+ def config_handler(self, new_config):
+ if self.debug:
+ print ("[statistics] handling new config: ")
+ print (str(new_config))
+ answer = isc.config.ccsession.create_answer(0)
+ return answer
+ # TODO
+
+ def command_handler(self, command, args):
+ if self.debug:
+ print ("[statsd] got command: "+str(command))
+ answer = isc.config.ccsession.create_answer(1, "command not implemented")
+ if type(command) != str:
+ answer = isc.config.ccsession.create_answer(1, "bad command")
+ else:
+ cmd = command
+ if cmd == "stats_update":
+ result = self.stats_update(args)
+ if self.debug:
+ print ("self.stats_update returned "+str(result))
+ answer = isc.config.ccsession.create_answer(result)
+ elif cmd == "shutdown":
+ if self.debug:
+ print ("[statsd] shutdown")
+ self.shutdown = 1
+ answer = isc.config.ccsession.create_answer(0)
+ elif cmd == "clear_counters":
+ self.stats = {}
+ if self.debug:
+ print ("[statsd] clear_counters")
+ answer = isc.config.ccsession.create_answer(0)
+ elif cmd == "show_statistics":
+ answer = isc.config.ccsession.create_answer(0, self.stats)
+ elif cmd == "print_settings":
+ full_config = self.ccs.get_full_config()
+ if self.debug:
+ print ("[stats] Full Config:")
+ for item in full_config:
+ print (item + ": " + str(full_config[item]))
+ answer = isc.config.ccsession.create_answer(0)
else:
- d[k] = s[k]
+ answer = isc.config.ccsession.create_answer(1, "Unknown command")
+ if self.debug:
+ print ("answer="+str(answer))
+ print (str(self)+str(command)+str( args)+str( answer))
+ return answer
- if (len(s) == 0):
- return {}
- if (len(s) == 1):
- for k in s.keys():
- out = s[k]
- out['components'] = 1
- out['timestamp2'] = out['timestamp']
- del out['from']
- return out
- _time1 = 0
- _time2 = 0
- out = {}
- for i in s.values():
- if (_time1 == 0 or _time1 < i['timestamp']):
- _time1 = i['timestamp']
- if (_time2 == 0 or _time2 > i['timestamp']):
- _time2 = i['timestamp']
- totalsub(out, i)
- out['components'] = len(s)
- out['timestamp'] = _time1;
- out['timestamp2'] = _time2;
- return out
+ def main(self):
+ if self.debug:
+ print ("statsd: Statistics.main()")
+ self.ccs = isc.config.ModuleCCSession(SPECFILE_LOCATION, self.config_handler, self.command_handler)
+ self.ccs.start()
+ if self.debug:
+ print ("[statsd] ccsession started")
+ while not self.shutdown:
+ if self.debug:
+ print ("loop")
+ self.ccs.check_command()
+ exit (1)
-def dicttoxml(stats, level = 0):
- def dicttoxmlsub(s, level):
- output = ''
- spaces = ' ' * level
- for k in s.keys():
- if (isinstance(s[k], dict)):
- output += spaces + ('<%s>\n' %k) \
- + dicttoxmlsub(s[k], level+1) \
- + spaces + '</%s>\n' %k
- else:
- output += spaces + '<%s>%s</%s>\n' % (k, s[k], k)
- return output
-
- for k in stats.keys():
- space = ' ' * level
- output = space + '<component component="%s">\n' % k
- s = stats[k]
- if ('component' in s or 'components' in s):
- output += dicttoxmlsub(s, level+1)
- else:
- for l in s.keys():
- output += space + ' <from from="%s">\n' % l \
- + dicttoxmlsub(s[l], level+2) \
- + space + ' </from>\n'
- output += space + '</component>\n'
- return output
-
-def dump_stats(statpath, statcount, stat, statraw):
- newfile = open(statpath + '.new', 'w')
- newfile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
- newfile.write('<!-- created at '+time.strftime('%Y%m%d %H%M%S')+' -->\n')
- newfile.write('<isc version="0.0">\n')
- newfile.write(' <bind10>\n')
- newfile.write(' <total>\n')
- newfile.write(dicttoxml(stat, 3))
- newfile.write(' </total>\n')
- newfile.write(' <each>\n')
- newfile.write(dicttoxml(statraw, 3))
- newfile.write(' </each>\n')
- newfile.write(' </bind10>\n')
- newfile.write('</isc>\n')
- newfile.close()
- loop = statcount
- while(loop > 0):
- old = statpath + '.%d' % loop
- loop -= 1
- new = statpath + '.%d' % loop
- if (os.access(new, os.F_OK)):
- os.rename(new, old)
- if (os.access(statpath, os.F_OK)):
- os.rename(statpath, new)
- os.rename(statpath + '.new', statpath)
-
-def collector(statgroup,step,statpath,statcount):
- cc = isc.cc.Session()
- if debug:
- print ("cc.lname=",cc.lname)
- cc.group_subscribe(statgroup)
- cc.group_subscribe(bossgroup, myname)
- wrote_time = -1
- last_wrote_time = -1
- last_recvd_time = -1
- stats = {}
- statstotal = {}
- while 1:
- wait = wrote_time + step - time.time()
- if wait <= 0 and last_recvd_time > wrote_time:
- if debug:
- print ("dump stats")
- dump_stats(statpath, statcount, statstotal, stats)
- last_wrote_time = wrote_time;
- wrote_time = time.time();
- wait = last_wrote_time + step - time.time()
- if wait < 0:
- wait = step
- r,w,e = select.select([cc._socket],[],[], wait)
- for sock in r:
- if sock == cc._socket:
- data,envelope = cc.group_recvmsg(False)
- if (envelope['group'] == bossgroup):
- if ('shutdown' in data):
- exit()
- if (envelope['group'] == statgroup):
- # Check received data
- if (not('component' in data and 'version' in data
- and 'stats' in data)):
- continue
- component = data['component']
- _from = envelope['from']
- data['from'] = _from
- if debug:
- print ("received from ",_from)
- if (not (component in stats)):
- stats[component] = {}
- (stats[component])[_from] = data;
- statstotal[component] = total(stats[component])
- last_recvd_time = time.time()
+def main():
+ try:
+ parser = OptionParser(version = __version__)
+ set_cmd_options(parser)
+ (options, args) = parser.parse_args()
+ statistics = Statistics()
+ statistics.main()
+ except KeyboardInterrupt:
+ log_error("exit b10-statsd")
+ except isc.cc.session.SessionError as e:
+ log_error(str(e))
+ log_error('Error happened! is the command channel daemon running?')
+ except Exception as e:
+ log_error(str(e))
if __name__ == '__main__':
- collector('statistics', 10, '/tmp/stats.xml', 100)
+ statistics = Statistics()
+ statistics.main()
More information about the bind10-changes
mailing list