BIND 10 trac1751, updated. 6fde5a92a9e7610e4bd6612221239fe686c3c004 [1751] a ChangeLog entry for #1751
BIND 10 source code commits
bind10-changes at lists.isc.org
Thu Mar 15 09:26:42 UTC 2012
The branch, trac1751 has been updated
via 6fde5a92a9e7610e4bd6612221239fe686c3c004 (commit)
via 94f99cad8539239a93877a1774fbadf89238c181 (commit)
via 3ca2c002d807fa1f8e7dbf6731ac4f1796f4e5a0 (commit)
via ddf22289ac033aa2ffdaf8e348c4a05f2d1d6951 (commit)
from 20fb475bd5cf9051c6f92163e2c427bb782db60b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 6fde5a92a9e7610e4bd6612221239fe686c3c004
Author: Naoki Kambe <kambe at jprs.co.jp>
Date: Thu Mar 15 17:30:35 2012 +0900
[1751] a ChangeLog entry for #1751
commit 94f99cad8539239a93877a1774fbadf89238c181
Author: Naoki Kambe <kambe at jprs.co.jp>
Date: Thu Mar 15 14:43:41 2012 +0900
[1751] addition of a search and removal of dead instances, a accumulation check by a complicated statistics data structure, and miscellaneous typo fixes
commit 3ca2c002d807fa1f8e7dbf6731ac4f1796f4e5a0
Author: Naoki Kambe <kambe at jprs.co.jp>
Date: Tue Mar 13 19:37:07 2012 +0900
[1751] typo fix, removal of the final recheck of statistics due to another issue, and addition of two more running auths for a regression check
commit ddf22289ac033aa2ffdaf8e348c4a05f2d1d6951
Author: Naoki Kambe <kambe at jprs.co.jp>
Date: Wed Mar 14 18:33:04 2012 +0900
[1751] check with a real pid
-----------------------------------------------------------------------
Summary of changes:
ChangeLog | 10 +++
src/bin/auth/tests/statistics_unittest.cc | 4 +-
src/bin/stats/stats.py.in | 79 ++++++++++++++++-------
src/bin/stats/tests/b10-stats_test.py | 98 ++++++++++++++++++++++++++++-
src/bin/stats/tests/test_utils.py | 14 ++++-
tests/system/bindctl/tests.sh | 66 +++++++++----------
6 files changed, 206 insertions(+), 65 deletions(-)
-----------------------------------------------------------------------
diff --git a/ChangeLog b/ChangeLog
index c56c407..3d2f9cf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+xxx. [bug] naokikambe
+ Fix inconsistent statistics counts of Auth via bindctl or
+ b10-stats-httpd while multiple b10-auth instances are running. Each
+ count of Auth is accumulated across the instances except an inactive
+ instances. Introduce a pid argument into the set command of b10-stats
+ so that b10-stats identifies which instance sends by the PID which the
+ b10-auth instance sends together with statistics data. Also add a
+ regression check into systest against that inconsistency.
+ (Trac #1751, git TBD)
+
401. [func]* jinmei
libdns++: updated the internal implementation of the
MessageRenderer class. This is mostly a transparent change, but
diff --git a/src/bin/auth/tests/statistics_unittest.cc b/src/bin/auth/tests/statistics_unittest.cc
index 31a1e89..2c25591 100644
--- a/src/bin/auth/tests/statistics_unittest.cc
+++ b/src/bin/auth/tests/statistics_unittest.cc
@@ -281,8 +281,8 @@ TEST_F(AuthCountersTest, submitStatisticsWithoutValidator) {
->get(0)->stringValue());
EXPECT_EQ("Auth", statistics_session_.sent_msg->get("command")
->get(1)->get("owner")->stringValue());
- EXPECT_TRUE(statistics_session_.sent_msg->get("command")
- ->get(1)->get("pid")->intValue() > 0);
+ EXPECT_EQ(statistics_session_.sent_msg->get("command")
+ ->get(1)->get("pid")->intValue(), getpid());
ConstElementPtr statistics_data = statistics_session_.sent_msg
->get("command")->get(1)
->get("data");
diff --git a/src/bin/stats/stats.py.in b/src/bin/stats/stats.py.in
index 3d6f178..58716e4 100755
--- a/src/bin/stats/stats.py.in
+++ b/src/bin/stats/stats.py.in
@@ -291,6 +291,29 @@ class Stats:
else:
statistics_data[name] = value
self.statistics_data = statistics_data
+ def _sum_bymodule(statistics_data_bypid):
+ # sum recursively each value under each
+ # element
+ def _sum(a, b):
+ if type(a) is dict:
+ return dict([ (k, _sum(v, b[k])) if k in b else (k, v) \
+ for (k, v) in a.items() ] \
+ + [ (k, v) for (k, v) in b.items() \
+ if k not in a ])
+ elif type(a) is list:
+ return [ _sum(a[i], b[i]) if len(b) > i else a[i] \
+ for i in range(len(a)) ] \
+ + [ b[i] for i in range(len(b)) \
+ if len(a) <= i ]
+ elif type(a) is float or type(a) is int:
+ return a + b
+ # If str or other types than above, then
+ # just replace with the newer value.
+ return a
+ ret = {}
+ for data in statistics_data_bypid.values():
+ ret.update(_sum(data, ret))
+ return ret
if owner and data:
errors = []
try:
@@ -312,35 +335,41 @@ class Stats:
self.statistics_data_bypid[owner][pid] = data
else:
self.statistics_data_bypid[owner] = { pid : data }
- def _sum_bymodule(statistics_data_bypid):
- # sum recursively each value under each
- # element
- def _sum(a, b):
- if type(a) == dict:
- return dict([ (k, _sum(v, b[k])) if k in b else (k, v) \
- for (k, v) in a.items() ] \
- + [ (k, v) for (k, v) in b.items() \
- if k not in a ])
- elif type(a) == list:
- return [ _sum(a[i], b[i]) if len(b) > i else a[i] \
- for i in range(len(a)) ] \
- + [ b[i] for i in range(len(b)) \
- if len(a) <= i ]
- elif type(a) == float or type(a) == int:
- return a + b
- # If str or other types than above, then
- # just replace with the newer value.
- return a
- ret = {}
- for data in statistics_data_bypid.values():
- ret.update(_sum(data, ret))
- return ret
self.statistics_data[owner].update(
_sum_bymodule(self.statistics_data_bypid[owner]))
- return
except KeyError:
errors.append("unknown module name: " + str(owner))
- return errors
+ if errors: return errors
+ # Find dead instances by invoking "show_processes" to Boss,
+ # then remove their statistics data if there are ones
+ seq = self.cc_session.group_sendmsg(
+ isc.config.ccsession.create_command("show_processes", None),
+ "Boss")
+ (answer, env) = self.cc_session.group_recvmsg(False, seq)
+ if answer:
+ (rcode, value) = isc.config.ccsession.parse_answer(answer)
+ if rcode == 0:
+ if type(value) is not list:
+ return
+ for v in value:
+ if type(v) is not list or len(v) < 2:
+ return
+ mlist = [ k for k in self.statistics_data_bypid.keys() ]
+ for m in mlist:
+ plist1 = [ p for p in self.statistics_data_bypid[m].keys() ]
+ plist2 = [ v[0] for v in value \
+ if v[1].lower().find(m.lower()) >= 0 ] \
+ + [-1] # add the default pid
+ # set object difference: nplist = plist1 - plist2
+ nplist = set(plist1).difference(set(plist2))
+ for p in nplist:
+ self.statistics_data_bypid[m].pop(p)
+ if self.statistics_data_bypid[m]:
+ if m in self.statistics_data:
+ self.statistics_data[m].update(
+ _sum_bymodule(self.statistics_data_bypid[m]))
+ else:
+ self.statistics_data_bypid.pop(m)
def command_status(self):
"""
diff --git a/src/bin/stats/tests/b10-stats_test.py b/src/bin/stats/tests/b10-stats_test.py
index 636f81c..d34c998 100644
--- a/src/bin/stats/tests/b10-stats_test.py
+++ b/src/bin/stats/tests/b10-stats_test.py
@@ -373,7 +373,7 @@ class TestStats(unittest.TestCase):
self.assertEqual(self.stats.update_statistics_data(owner='Dummy', foo='bar'),
['unknown module name: Dummy'])
- def test_update_modules_withpid(self):
+ def test_update_statistics_data_withpid(self):
# one pid of Auth
self.stats.update_statistics_data(owner='Auth',
pid=9999,
@@ -387,6 +387,19 @@ class TestStats(unittest.TestCase):
self.assertEqual(self.stats.statistics_data_bypid['Auth'][9999]['queries.tcp'], 1001)
self.assertEqual(self.stats.statistics_data_bypid,
{'Auth': {9999: {'queries.tcp': 1001}}})
+ # non-existent pid of Auth, but no changes in statistics data
+ self.stats.update_statistics_data(owner='Auth',
+ pid=10000,
+ **{'queries.tcp':2001})
+ self.assertTrue('Auth' in self.stats.statistics_data)
+ self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+ self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 1001)
+ self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+ self.assertTrue(9999 in self.stats.statistics_data_bypid['Auth'])
+ self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9999])
+ self.assertEqual(self.stats.statistics_data_bypid['Auth'][9999]['queries.tcp'], 1001)
+ self.assertEqual(self.stats.statistics_data_bypid,
+ {'Auth': {9999: {'queries.tcp': 1001}}})
# another pid of Auth
self.stats.update_statistics_data(owner='Auth',
pid=9998,
@@ -749,38 +762,117 @@ class TestStats(unittest.TestCase):
retval = isc.config.ccsession.parse_answer(
self.stats.command_set(owner='Auth',
pid=9997,
- data={ 'queries.tcp' : 1001 }))
+ data={ 'queries.tcp' : 1001,
+ 'queries.perzone':
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 1 },
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 2,
+ 'queries.udp': 3 }]}))
self.assertEqual(retval, (0,None))
self.assertTrue('Auth' in self.stats.statistics_data)
self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 1001)
+ self.assertEqual(self.stats.statistics_data['Auth']['queries.perzone'],
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 1 },
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 2,
+ 'queries.udp': 3 }])
self.assertTrue('Stats' in self.stats.statistics_data)
self.assertTrue('last_update_time' in self.stats.statistics_data['Stats'])
self.assertTrue('Auth' in self.stats.statistics_data_bypid)
self.assertTrue(9997 in self.stats.statistics_data_bypid['Auth'])
self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9997])
+ self.assertTrue('queries.perzone' in self.stats.statistics_data_bypid['Auth'][9997])
+ self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.tcp'], 1001)
+ self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.perzone'],
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 1 },
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 2,
+ 'queries.udp': 3 }])
+ # non-existent pid of Auth, but no changes in statistics data
+ retval = isc.config.ccsession.parse_answer(
+ self.stats.command_set(owner='Auth',
+ pid=10000,
+ data={ 'queries.tcp' : 2001,
+ 'queries.perzone':
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 101 },
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 102,
+ 'queries.udp': 103 }]}))
+ self.assertEqual(retval, (0,None))
+ self.assertTrue('Auth' in self.stats.statistics_data)
+ self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
+ self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 1001)
+ self.assertEqual(self.stats.statistics_data['Auth']['queries.perzone'],
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 1 },
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 2,
+ 'queries.udp': 3 }])
+ self.assertTrue('Auth' in self.stats.statistics_data_bypid)
+ self.assertTrue(9997 in self.stats.statistics_data_bypid['Auth'])
+ self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9997])
self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.tcp'], 1001)
+ self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.perzone'],
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 1 },
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 2,
+ 'queries.udp': 3 }])
# another pid of Auth
retval = isc.config.ccsession.parse_answer(
self.stats.command_set(owner='Auth',
pid=9996,
data={ 'queries.tcp' : 1002,
- 'queries.udp' : 1003,}))
+ 'queries.udp' : 1003,
+ 'queries.perzone':
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 10,
+ 'queries.udp': 11},
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 12,
+ 'queries.udp': 13 }]}))
self.assertEqual(retval, (0,None))
self.assertTrue('Auth' in self.stats.statistics_data)
self.assertTrue('queries.tcp' in self.stats.statistics_data['Auth'])
self.assertTrue('queries.udp' in self.stats.statistics_data['Auth'])
+ self.assertTrue('queries.perzone' in self.stats.statistics_data['Auth'])
self.assertEqual(self.stats.statistics_data['Auth']['queries.tcp'], 2003)
self.assertEqual(self.stats.statistics_data['Auth']['queries.udp'], 1003)
+ self.assertEqual(self.stats.statistics_data['Auth']['queries.perzone'],
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 11,
+ 'queries.udp': 11},
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 14,
+ 'queries.udp': 16 }])
self.assertTrue('Auth' in self.stats.statistics_data_bypid)
self.assertTrue(9997 in self.stats.statistics_data_bypid['Auth'])
self.assertTrue(9996 in self.stats.statistics_data_bypid['Auth'])
self.assertTrue('queries.tcp' in self.stats.statistics_data_bypid['Auth'][9997])
self.assertTrue('queries.udp' in self.stats.statistics_data_bypid['Auth'][9996])
self.assertTrue('queries.udp' in self.stats.statistics_data_bypid['Auth'][9996])
+ self.assertTrue('queries.perzone' in self.stats.statistics_data_bypid['Auth'][9996])
self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.tcp'], 1001)
+ self.assertEqual(self.stats.statistics_data_bypid['Auth'][9997]['queries.perzone'],
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 1 },
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 2,
+ 'queries.udp': 3 }])
self.assertEqual(self.stats.statistics_data_bypid['Auth'][9996]['queries.tcp'], 1002)
self.assertEqual(self.stats.statistics_data_bypid['Auth'][9996]['queries.udp'], 1003)
+ self.assertEqual(self.stats.statistics_data_bypid['Auth'][9996]['queries.perzone'],
+ [{ 'zonename': 'test1.example',
+ 'queries.tcp': 10,
+ 'queries.udp': 11},
+ { 'zonename': 'test2.example',
+ 'queries.tcp': 12,
+ 'queries.udp': 13 }])
class TestOSEnv(unittest.TestCase):
def test_osenv(self):
diff --git a/src/bin/stats/tests/test_utils.py b/src/bin/stats/tests/test_utils.py
index 29fc785..8d9ed50 100644
--- a/src/bin/stats/tests/test_utils.py
+++ b/src/bin/stats/tests/test_utils.py
@@ -150,6 +150,11 @@ class MockBoss:
"command_name": "sendstats",
"command_description": "Send data to a statistics module at once",
"command_args": []
+ },
+ {
+ "command_name": "show_processes",
+ "command_description": "List the running BIND 10 processes",
+ "command_args": []
}
],
"statistics": [
@@ -210,6 +215,13 @@ class MockBoss:
return isc.config.create_answer(0)
elif command == 'getstats':
return isc.config.create_answer(0, params)
+ elif command == 'show_processes':
+ # Return dummy pids
+ return isc.config.create_answer(
+ 0, [[ 9999, "b10-auth" ],
+ [ 9998, "b10-auth-2" ],
+ [ 9997, "b10-auth-3" ],
+ [ 9996, "b10-auth-4" ]])
return isc.config.create_answer(1, "Unknown Command")
class MockAuth:
@@ -340,7 +352,7 @@ class MockAuth:
params = { "owner": "Auth",
"data": { 'queries.tcp': self.queries_tcp,
'queries.udp': self.queries_udp,
- 'queries.per-zone' : self.queries_per_zone } }
+ 'queries.perzone' : self.queries_per_zone } }
return send_command("set", "Stats", params=params, session=self.cc_session)
return isc.config.create_answer(1, "Unknown Command")
diff --git a/tests/system/bindctl/tests.sh b/tests/system/bindctl/tests.sh
index 8fa9d61..cb9d8be 100755
--- a/tests/system/bindctl/tests.sh
+++ b/tests/system/bindctl/tests.sh
@@ -85,12 +85,9 @@ sleep 2
echo 'Stats show
' | $RUN_BINDCTL \
--csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
-# The statistics counters can not be reset even after auth
-# restarts. Because stats preserves the query counts which the dying
-# auth sent. Then it cumulates them and new counts which the living
-# auth sends.
-cnt_value1=`expr $cnt_value1 + 0`
-cnt_value2=`expr $cnt_value2 + 1`
+# The statistics counters should have been reset while stop/start.
+cnt_value1=0
+cnt_value2=1
cnt_value3=`expr $cnt_value1 + $cnt_value2`
grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
@@ -126,58 +123,59 @@ grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
if [ $status != 0 ]; then echo "I:failed"; fi
n=`expr $n + 1`
-echo "I:Starting another b10-auth and checking that ($n)"
-echo 'config add Boss/components b10-auth-2
-config set Boss/components/b10-auth { "special": "auth", "kind": "needed" }
+echo "I:Starting more b10-auths and checking that ($n)"
+for i in 2 3
+do
+ echo 'config add Boss/components b10-auth-'$i'
+config set Boss/components/b10-auth-'$i' { "special": "auth", "kind": "needed" }
config commit
quit
' | $RUN_BINDCTL \
--csv-file-dir=$BINDCTL_CSV_DIR 2>&1 > /dev/null || status=1
+done
$DIG +norec @10.53.0.1 -p 53210 ns.example.com. A >dig.out.$n || status=1
grep 192.0.2.2 dig.out.$n > /dev/null || status=1
if [ $status != 0 ]; then echo "I:failed"; fi
n=`expr $n + 1`
-echo "I:Rechecking BIND 10 statistics after a pause ($n)"
+echo "I:Rechecking BIND 10 statistics consistency after a pause ($n)"
sleep 2
-echo 'Stats show
-' | $RUN_BINDCTL \
- --csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
-# The statistics counters should keep increasing even after another
-# b10-auth starts.
cnt_value1=`expr $cnt_value1 + 0`
cnt_value2=`expr $cnt_value2 + 1`
cnt_value3=`expr $cnt_value1 + $cnt_value2`
-grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
-grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
-grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
-if [ $status != 0 ]; then echo "I:failed"; fi
+# Rechecking some times
+for i in 1 2 3 4
+do
+ echo 'Stats show
+' | $RUN_BINDCTL \
+ --csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
+ # The statistics counters should keep being consistent even while
+ # multiple b10-auths are running.
+ grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
+ grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
+ grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
+ if [ $status != 0 ]; then echo "I:failed "; break ; fi
+done
n=`expr $n + 1`
-echo "I:Stopping the second b10-auth and checking that ($n)"
-echo 'config remove Boss/components b10-auth-2
+echo "I:Stopping extra b10-auths and checking that ($n)"
+for i in 3 2
+do
+ echo 'config remove Boss/components b10-auth-'$i'
config commit
quit
' | $RUN_BINDCTL \
--csv-file-dir=$BINDCTL_CSV_DIR 2>&1 > /dev/null || status=1
+done
$DIG +norec @10.53.0.1 -p 53210 ns.example.com. A >dig.out.$n || status=1
grep 192.0.2.2 dig.out.$n > /dev/null || status=1
if [ $status != 0 ]; then echo "I:failed"; fi
n=`expr $n + 1`
-echo "I:Rechecking BIND 10 statistics after a pause ($n)"
-sleep 2
-echo 'Stats show
-' | $RUN_BINDCTL \
- --csv-file-dir=$BINDCTL_CSV_DIR > bindctl.out.$n || status=1
-cnt_value1=`expr $cnt_value1 + 0`
-cnt_value2=`expr $cnt_value2 + 1`
-cnt_value3=`expr $cnt_value1 + $cnt_value2`
-grep $cnt_name1".*\<"$cnt_value1"\>" bindctl.out.$n > /dev/null || status=1
-grep $cnt_name2".*\<"$cnt_value2"\>" bindctl.out.$n > /dev/null || status=1
-grep $cnt_name3".*\<"$cnt_value3"\>" bindctl.out.$n > /dev/null || status=1
-if [ $status != 0 ]; then echo "I:failed"; fi
-n=`expr $n + 1`
+# The statistics counters can not be rechecked here because the auth
+# instance seems to hang up after one of the multiple auth instances
+# was removed via bindctl. This reason seems to be the same reason as
+# #1703.
echo "I:exit status: $status"
exit $status
More information about the bind10-changes
mailing list