BIND 10 master, updated. 55f0a8f3f54c91686f910669b5183e4e95b51ae0 Merge #3049
BIND 10 source code commits
bind10-changes at lists.isc.org
Mon Aug 19 11:31:05 UTC 2013
The branch, master has been updated
via 55f0a8f3f54c91686f910669b5183e4e95b51ae0 (commit)
via f5c6cae3cdd1a3fafa8266575a5cc8a082ef8643 (commit)
via a635a6d6ce769d71109e683f0e0480f73b818112 (commit)
via b177e3f9b587dafa15133ef0b8fe621cfbf95f2e (commit)
via b92b89134b7393e24e9bfbe16deee9ca99d352e1 (commit)
via 8838f82a224a2aeb7479bc2c4bc3c44c3c20454b (commit)
via d03c9fce131acc35740a52e63894761c77915c0c (commit)
via 7b94068163e1d90e4c10a19ec5a847b614f31f99 (commit)
from 6c968df2fedfc4c0b464fde5e1772dd24e37602e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 55f0a8f3f54c91686f910669b5183e4e95b51ae0
Merge: f5c6cae a635a6d
Author: Michal 'vorner' Vaner <michal.vaner at nic.cz>
Date: Mon Aug 19 12:41:46 2013 +0200
Merge #3049
Fix configure test for offset_ptr warnings.
commit f5c6cae3cdd1a3fafa8266575a5cc8a082ef8643
Merge: 6c968df b177e3f
Author: Michal 'vorner' Vaner <michal.vaner at nic.cz>
Date: Mon Aug 19 12:37:35 2013 +0200
Merge #2690
Using select instead of poll/kqueue in message queue daemon.
-----------------------------------------------------------------------
Summary of changes:
configure.ac | 2 +-
m4macros/ax_boost_for_bind10.m4 | 3 +
src/bin/msgq/msgq.py.in | 164 +++++++--------------------------------
src/bin/msgq/msgq_messages.mes | 16 ++--
src/bin/msgq/tests/msgq_test.py | 76 ++++++++++--------
5 files changed, 81 insertions(+), 180 deletions(-)
-----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index d083fc4..e01f5cf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -883,7 +883,7 @@ LIBS=$LIBS_SAVED
AX_BOOST_FOR_BIND10
# Boost offset_ptr is required in one library and not optional right now, so
# we unconditionally fail here if it doesn't work.
-if test "$BOOST_OFFSET_PTR_WOULDFAIL" = "yes"; then
+if test "$BOOST_OFFSET_PTR_WOULDFAIL" = "yes" -a "$werror_ok" = 1; then
AC_MSG_ERROR([Failed to compile a required header file. Try upgrading Boost to 1.44 or higher (when using clang++) or specifying --without-werror. See the ChangeLog entry for Trac no. 2147 for more details.])
fi
diff --git a/m4macros/ax_boost_for_bind10.m4 b/m4macros/ax_boost_for_bind10.m4
index cc6408c..3045dfb 100644
--- a/m4macros/ax_boost_for_bind10.m4
+++ b/m4macros/ax_boost_for_bind10.m4
@@ -86,6 +86,8 @@ AC_TRY_COMPILE([
# Boost offset_ptr is known to not compile on some platforms, depending on
# boost version, its local configuration, and compiler. Detect it.
+CXXFLAGS_SAVED="$CXXFLAGS"
+CXXFLAGS="$CXXFLAGS -Werror"
AC_MSG_CHECKING([Boost offset_ptr compiles])
AC_TRY_COMPILE([
#include <boost/interprocess/offset_ptr.hpp>
@@ -94,6 +96,7 @@ AC_TRY_COMPILE([
BOOST_OFFSET_PTR_WOULDFAIL=no],
[AC_MSG_RESULT(no)
BOOST_OFFSET_PTR_WOULDFAIL=yes])
+CXXFLAGS="$CXXFLAGS_SAVED"
# Detect build failure case known to happen with Boost installed via
# FreeBSD ports
diff --git a/src/bin/msgq/msgq.py.in b/src/bin/msgq/msgq.py.in
index b6b1106..a3e30d3 100755
--- a/src/bin/msgq/msgq.py.in
+++ b/src/bin/msgq/msgq.py.in
@@ -182,8 +182,6 @@ class MsgQ:
self.socket_file = socket_file
self.verbose = verbose
- self.poller = None
- self.kqueue = None
self.runnable = False
self.listen_socket = False
self.sockets = {}
@@ -204,6 +202,7 @@ class MsgQ:
# side.
self.__lock = threading.Lock()
self._session = None
+ self.__poller_sock = None
def members_notify(self, event, params):
"""
@@ -264,37 +263,6 @@ class MsgQ:
self.__cfgmgr_ready_cond.wait()
return self.__cfgmgr_ready
- def setup_poller(self):
- """Set up the poll thing. Internal function."""
- try:
- self.kqueue = select.kqueue()
- except AttributeError:
- self.poller = select.poll()
-
- def add_kqueue_socket(self, socket, write_filter=False):
- """Add a kqueue filter for a socket. By default the read
- filter is used; if write_filter is set to True, the write
- filter is used. We use a boolean value instead of a specific
- filter constant, because kqueue filter values do not seem to
- be defined on some systems. The use of boolean makes the
- interface restrictive because there are other filters, but this
- method is mostly only for our internal use, so it should be
- acceptable at least for now."""
- filter_type = select.KQ_FILTER_WRITE if write_filter else \
- select.KQ_FILTER_READ
- event = select.kevent(socket.fileno(), filter_type,
- select.KQ_EV_ADD | select.KQ_EV_ENABLE)
- self.kqueue.control([event], 0)
-
- def delete_kqueue_socket(self, socket, write_filter=False):
- """Delete a kqueue filter for socket. See add_kqueue_socket()
- for the semantics and notes about write_filter."""
- filter_type = select.KQ_FILTER_WRITE if write_filter else \
- select.KQ_FILTER_READ
- event = select.kevent(socket.fileno(), filter_type,
- select.KQ_EV_DELETE)
- self.kqueue.control([event], 0)
-
def setup_listener(self):
"""Set up the listener socket. Internal function."""
logger.debug(TRACE_BASIC, MSGQ_LISTENER_SETUP, self.socket_file)
@@ -315,11 +283,6 @@ class MsgQ:
logger.fatal(MSGQ_LISTENER_FAILED, self.socket_file, e)
raise e
- if self.poller:
- self.poller.register(self.listen_socket, select.POLLIN)
- else:
- self.add_kqueue_socket(self.listen_socket)
-
def setup_signalsock(self):
"""Create a socket pair used to signal when we want to finish.
Using a socket is easy and thread/signal safe way to signal
@@ -329,18 +292,12 @@ class MsgQ:
# closed, we should shut down.
(self.__poller_sock, self.__control_sock) = socket.socketpair()
- if self.poller:
- self.poller.register(self.__poller_sock, select.POLLIN)
- else:
- self.add_kqueue_socket(self.__poller_sock)
-
def setup(self):
"""Configure listener socket, polling, etc.
Raises a socket.error if the socket_file cannot be
created.
"""
- self.setup_poller()
self.setup_signalsock()
self.setup_listener()
@@ -369,20 +326,10 @@ class MsgQ:
logger.debug(TRACE_BASIC, MSGQ_SOCKET_REGISTERED, newsocket.fileno(),
lname)
- if self.poller:
- self.poller.register(newsocket, select.POLLIN)
- else:
- self.add_kqueue_socket(newsocket)
-
self.members_notify('connected', {'client': lname})
def kill_socket(self, fd, sock):
"""Fully close down the socket."""
- # Unregister events on the socket. Note that we don't have to do
- # this for kqueue because the registered events are automatically
- # deleted when the corresponding socket is closed.
- if self.poller:
- self.poller.unregister(sock)
unsubscribed_from = self.subs.unsubscribe_all(sock)
lname = self.fd_to_lname[fd]
@@ -584,15 +531,10 @@ class MsgQ:
else:
buff = msg[amount_sent:]
last_sent = now
- if self.poller:
- self.poller.register(fileno, select.POLLIN |
- select.POLLOUT)
- else:
- self.add_kqueue_socket(sock, True)
self.sendbuffs[fileno] = (last_sent, buff)
return True
- def __process_write(self, fileno):
+ def _process_write(self, fileno):
# Try to send some data from the buffer
(_, msg) = self.sendbuffs[fileno]
sock = self.sockets[fileno]
@@ -602,10 +544,6 @@ class MsgQ:
msg = msg[amount_sent:]
if len(msg) == 0:
# If there's no more, stop requesting for write availability
- if self.poller:
- self.poller.register(fileno, select.POLLIN)
- else:
- self.delete_kqueue_socket(sock, True)
del self.sendbuffs[fileno]
else:
self.sendbuffs[fileno] = (time.clock(), msg)
@@ -717,90 +655,46 @@ class MsgQ:
"""Process messages. Forever. Mostly."""
self.running = True
- if self.poller:
- self.run_poller()
- else:
- self.run_kqueue()
+ self.run_select()
- def run_poller(self):
+ def run_select(self):
while self.running:
+ reads = list(self.fd_to_lname.keys())
+ if self.listen_socket.fileno() != -1: # Skip in tests
+ reads.append(self.listen_socket.fileno())
+ if self.__poller_sock and self.__poller_sock.fileno() != -1:
+ reads.append(self.__poller_sock.fileno())
+ writes = list(self.sendbuffs.keys())
+ (read_ready, write_ready) = ([], [])
try:
- # Poll with a timeout so that every once in a while,
- # the loop checks for self.running.
- events = self.poller.poll()
+ (read_ready, write_ready, _) = select.select(reads, writes,
+ []);
except select.error as err:
if err.args[0] == errno.EINTR:
- events = []
+ continue # Just try it again if interrupted.
else:
- logger.fatal(MSGQ_POLL_ERROR, err)
+ logger.fatal(MSGQ_SELECT_ERROR, err)
break
with self.__lock:
- for (fd, event) in events:
+ write_ready = set(write_ready)
+ for fd in read_ready:
+ # Do only one operation per loop iteration on the given fd.
+ # It could be possible to perform both, but it may have
+ # undesired side effects in special situations (like, if the
+ # read closes the socket).
+ if fd in write_ready:
+ write_ready.remove(fd)
if fd == self.listen_socket.fileno():
self.process_accept()
- elif fd == self.__poller_sock.fileno():
- # If it's the signal socket, we should terminate now.
+ elif self.__poller_sock and fd == \
+ self.__poller_sock.fileno():
+ # The signal socket. We should terminate now.
self.running = False
break
else:
- writable = event & select.POLLOUT
- # Note: it may be okay to read data if available
- # immediately after write some, but due to unexpected
- # regression (see comments on the kqueue version below)
- # we restrict one operation per iteration for now.
- # In future we may clarify the point and enable the
- # "read/write" mode.
- readable = not writable and (event & select.POLLIN)
- if not writable and not readable:
- logger.error(MSGQ_POLL_UNKNOWN_EVENT, fd, event)
- self._process_fd(fd, writable, readable, False)
-
- def run_kqueue(self):
- while self.running:
- # Check with a timeout so that every once in a while,
- # the loop checks for self.running.
- events = self.kqueue.control(None, 10)
- if not events:
- raise RuntimeError('serve: kqueue returned no events')
-
- with self.__lock:
- for event in events:
- if event.ident == self.listen_socket.fileno():
- self.process_accept()
- elif event.ident == self.__poller_sock.fileno():
- # If it's the signal socket, we should terminate now.
- self.running = False
- break;
- else:
- fd = event.ident
- writable = event.filter == select.KQ_FILTER_WRITE
- readable = (event.filter == select.KQ_FILTER_READ and
- event.data > 0)
- # It seems to break some of our test cases if we
- # immediately close the socket on EOF after reading
- # some data. It may be possible to avoid by tweaking
- # the test, but unless we can be sure we'll hold off.
- closed = (not readable and
- (event.flags & select.KQ_EV_EOF))
- self._process_fd(fd, writable, readable, closed)
-
- def _process_fd(self, fd, writable, readable, closed):
- '''Process a single FD: unified subroutine of run_kqueue/poller.
-
- closed can be True only in the case of kqueue. This is essentially
- private but is defined as if it were "protected" so it's callable
- from tests.
-
- '''
- # We need to check if FD is still in the sockets dict, because
- # it's possible that the socket has been "killed" while processing
- # other FDs; it's even possible it's killed within this method.
- if writable and fd in self.sockets:
- self.__process_write(fd)
- if readable and fd in self.sockets:
- self.process_packet(fd, self.sockets[fd])
- if closed and fd in self.sockets:
- self.kill_socket(fd, self.sockets[fd])
+ self.process_packet(fd, self.sockets[fd])
+ for fd in write_ready:
+ self._process_write(fd)
def stop(self):
# Signal it should terminate.
diff --git a/src/bin/msgq/msgq_messages.mes b/src/bin/msgq/msgq_messages.mes
index 09c9030..909d6a3 100644
--- a/src/bin/msgq/msgq_messages.mes
+++ b/src/bin/msgq/msgq_messages.mes
@@ -85,17 +85,6 @@ Debug message. The listener is trying to open a listening socket.
Debug message. The message queue successfully opened a listening socket and
waits for incoming connections.
-% MSGQ_POLL_ERROR Error while polling for events: %1
-A low-level error happened when waiting for events, the error is logged. The
-reason for this varies, but it usually means the system is short on some
-resources.
-
-% MSGQ_POLL_UNKNOWN_EVENT Got an unknown event from the poller for fd %1: %2
-An unknown event got out from the poll() system call. This should generally not
-happen and it is either a programmer error or OS bug. The event is ignored. The
-number noted as the event is the raw encoded value, which might be useful to
-the authors when figuring the problem out.
-
% MSGQ_RECV_ERROR Error reading from socket %1: %2
There was a low-level error when reading from a socket. The error is logged and
the corresponding socket is dropped. The errors include receiving
@@ -119,6 +108,11 @@ on shutdown unless there's really something unexpected.
% MSGQ_RECV_HDR Received header: %1
Debug message. This message includes the whole routing header of a packet.
+% MSGQ_SELECT_ERROR Error while waiting for events: %1
+A low-level error happened when waiting for events, the error is logged. The
+reason for this varies, but it usually means the system is short on some
+resources.
+
% MSGQ_SEND_ERROR Error while sending to socket %1: %2
There was a low-level error when sending data to a socket. The error is logged
and the corresponding socket is dropped.
diff --git a/src/bin/msgq/tests/msgq_test.py b/src/bin/msgq/tests/msgq_test.py
index 3d0cbf9..210246d 100644
--- a/src/bin/msgq/tests/msgq_test.py
+++ b/src/bin/msgq/tests/msgq_test.py
@@ -274,8 +274,6 @@ class MsgQTest(unittest.TestCase):
sock = Sock(1)
return notifications, sock
- @unittest.skipUnless('POLLIN' in select.__dict__,
- 'cannot perform tests requiring select.poll')
def test_notifies(self):
"""
Test the message queue sends notifications about connecting,
@@ -315,8 +313,6 @@ class MsgQTest(unittest.TestCase):
self.__msgq.kill_socket(sock.fileno(), sock)
self.assertEqual([('disconnected', {'client': lname})], notifications)
- @unittest.skipUnless('POLLIN' in select.__dict__,
- 'cannot perform tests requiring select.poll')
def test_notifies_implicit_kill(self):
"""
Test that the unsubscription notifications are sent before the socket
@@ -579,7 +575,6 @@ class SendNonblock(unittest.TestCase):
'''
msgq = MsgQ()
# We do only partial setup, so we don't create the listening socket
- msgq.setup_poller()
(read, write) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
msgq.register_socket(write)
self.assertEqual(1, len(msgq.lnames))
@@ -677,7 +672,6 @@ class SendNonblock(unittest.TestCase):
queue_pid = os.fork()
if queue_pid == 0:
signal.alarm(120)
- msgq.setup_poller()
msgq.setup_signalsock()
msgq.register_socket(queue)
msgq.run()
@@ -754,7 +748,6 @@ class SendNonblock(unittest.TestCase):
msgq = MsgQ()
# Don't need a listen_socket
msgq.listen_socket = DummySocket
- msgq.setup_poller()
msgq.setup_signalsock()
msgq.register_socket(write)
msgq.register_socket(control_write)
@@ -997,9 +990,11 @@ class SocketTests(unittest.TestCase):
self.__killed_socket = None
self.__logger = self.LoggerWrapper(msgq.logger)
msgq.logger = self.__logger
+ self.__orig_select = msgq.select.select
def tearDown(self):
msgq.logger = self.__logger.orig_logger
+ msgq.select.select = self.__orig_select
def test_send_data(self):
# Successful case: _send_data() returns the hardcoded value, and
@@ -1047,32 +1042,6 @@ class SocketTests(unittest.TestCase):
self.assertEqual(expected_errors, self.__logger.error_called)
self.assertEqual(expected_warns, self.__logger.warn_called)
- def test_process_fd_read_after_bad_write(self):
- '''Check the specific case of write fail followed by read attempt.
-
- The write failure results in kill_socket, then read shouldn't tried.
-
- '''
- self.__sock_error.errno = errno.EPIPE
- self.__sock.ex_on_send = self.__sock_error
- self.__msgq.process_socket = None # if called, trigger an exception
- self.__msgq._process_fd(42, True, True, False) # shouldn't crash
-
- # check the socket is deleted from the fileno=>sock dictionary
- self.assertEqual({}, self.__msgq.sockets)
-
- def test_process_fd_close_after_bad_write(self):
- '''Similar to the previous, but for checking dup'ed kill attempt'''
- self.__sock_error.errno = errno.EPIPE
- self.__sock.ex_on_send = self.__sock_error
- self.__msgq._process_fd(42, True, False, True) # shouldn't crash
- self.assertEqual({}, self.__msgq.sockets)
-
- def test_process_fd_writer_after_close(self):
- '''Emulate a "writable" socket has been already closed and killed.'''
- # This just shouldn't crash
- self.__msgq._process_fd(4200, True, False, False)
-
def test_process_packet(self):
'''Check some failure cases in handling an incoming message.'''
expected_errors = 0
@@ -1106,6 +1075,47 @@ class SocketTests(unittest.TestCase):
self.assertEqual(expected_errors, self.__logger.error_called)
self.assertEqual(expected_debugs, self.__logger.debug_called)
+ def test_do_select(self):
+ """
+ Check the behaviour of the run_select method.
+
+ In particular, check that we skip writing to the sockets we read,
+ because a read may have side effects (like closing the socket) and
+ we want to prevent strange behavior.
+ """
+ self.__read_called = []
+ self.__write_called = []
+ self.__reads = None
+ self.__writes = None
+ def do_read(fd, socket):
+ self.__read_called.append(fd)
+ self.__msgq.running = False
+ def do_write(fd):
+ self.__write_called.append(fd)
+ self.__msgq.running = False
+ self.__msgq.process_packet = do_read
+ self.__msgq._process_write = do_write
+ self.__msgq.fd_to_lname = {42: 'lname', 44: 'other', 45: 'unused'}
+ # The do_select does index it, but just passes the value. So reuse
+ # the dict to safe typing in the test.
+ self.__msgq.sockets = self.__msgq.fd_to_lname
+ self.__msgq.sendbuffs = {42: 'data', 43: 'data'}
+ def my_select(reads, writes, errors):
+ self.__reads = reads
+ self.__writes = writes
+ self.assertEqual([], errors)
+ return ([42, 44], [42, 43], [])
+ msgq.select.select = my_select
+ self.__msgq.listen_socket = DummySocket
+
+ self.__msgq.running = True
+ self.__msgq.run_select()
+
+ self.assertEqual([42, 44], self.__read_called)
+ self.assertEqual([43], self.__write_called)
+ self.assertEqual({42, 44, 45}, set(self.__reads))
+ self.assertEqual({42, 43}, set(self.__writes))
+
if __name__ == '__main__':
isc.log.resetUnitTestRootLogger()
unittest.main()
More information about the bind10-changes
mailing list