viff

changeset 1184:6cd5ceb87542

Added reentrant reactor. Data now is sent faster and incoming data is processed faster.
author Marcel Keller <mkeller@cs.au.dk>
date Tue, 19 May 2009 17:04:07 +0200
parents e89fb02c5e3d
children 89b93bc75476
files apps/aes.py apps/aes_inversion.py apps/beginner.py apps/benchmark.py apps/compare.py apps/divide.py apps/equality.py apps/eval-poly.py apps/gc-test.py apps/int-bit-conversion.py apps/millionaires.py apps/multiply.py apps/online-comparison-benchmark.py apps/paillier.py apps/prss-and-open.py apps/seq-mul.py apps/shamir-share-open.py apps/share-open.py apps/sort.py apps/sum.py apps/two-fields.py apps/xor-all.py viff/active.py viff/passive.py viff/reactor.py viff/runtime.py viff/test/__init__.py viff/test/util.py
diffstat 28 files changed, 186 insertions(+), 1 deletions(-) [+]
line diff
     1.1 --- a/apps/aes.py	Thu May 14 12:01:54 2009 +0200
     1.2 +++ b/apps/aes.py	Tue May 19 17:04:07 2009 +0200
     1.3 @@ -23,6 +23,8 @@
     1.4  import time
     1.5  from optparse import OptionParser
     1.6  
     1.7 +import viff.reactor
     1.8 +viff.reactor.install()
     1.9  from twisted.internet import reactor
    1.10  
    1.11  from viff.field import GF256
     2.1 --- a/apps/aes_inversion.py	Thu May 14 12:01:54 2009 +0200
     2.2 +++ b/apps/aes_inversion.py	Tue May 19 17:04:07 2009 +0200
     2.3 @@ -24,6 +24,8 @@
     2.4  import time
     2.5  from optparse import OptionParser
     2.6  
     2.7 +import viff.reactor
     2.8 +viff.reactor.install()
     2.9  from twisted.internet import reactor
    2.10  
    2.11  from viff.field import GF256
     3.1 --- a/apps/beginner.py	Thu May 14 12:01:54 2009 +0200
     3.2 +++ b/apps/beginner.py	Tue May 19 17:04:07 2009 +0200
     3.3 @@ -37,6 +37,8 @@
     3.4  # Some useful imports.
     3.5  import sys
     3.6  
     3.7 +import viff.reactor
     3.8 +viff.reactor.install()
     3.9  from twisted.internet import reactor
    3.10  
    3.11  from viff.field import GF
     4.1 --- a/apps/benchmark.py	Thu May 14 12:01:54 2009 +0200
     4.2 +++ b/apps/benchmark.py	Tue May 19 17:04:07 2009 +0200
     4.3 @@ -60,6 +60,8 @@
     4.4  import operator
     4.5  from pprint import pformat
     4.6  
     4.7 +import viff.reactor
     4.8 +viff.reactor.install()
     4.9  from twisted.internet import reactor
    4.10  
    4.11  from viff.field import GF, GF256, FakeGF
     5.1 --- a/apps/compare.py	Thu May 14 12:01:54 2009 +0200
     5.2 +++ b/apps/compare.py	Tue May 19 17:04:07 2009 +0200
     5.3 @@ -20,6 +20,8 @@
     5.4  import sys
     5.5  import random
     5.6  
     5.7 +import viff.reactor
     5.8 +viff.reactor.install()
     5.9  from twisted.internet import reactor
    5.10  from twisted.internet.defer import gatherResults
    5.11  
     6.1 --- a/apps/divide.py	Thu May 14 12:01:54 2009 +0200
     6.2 +++ b/apps/divide.py	Tue May 19 17:04:07 2009 +0200
     6.3 @@ -32,6 +32,8 @@
     6.4  """
     6.5  
     6.6  from optparse import OptionParser
     6.7 +import viff.reactor
     6.8 +viff.reactor.install()
     6.9  from twisted.internet import reactor
    6.10  
    6.11  from viff.field import GF
     7.1 --- a/apps/equality.py	Thu May 14 12:01:54 2009 +0200
     7.2 +++ b/apps/equality.py	Tue May 19 17:04:07 2009 +0200
     7.3 @@ -31,6 +31,8 @@
     7.4  """
     7.5  
     7.6  from optparse import OptionParser
     7.7 +import viff.reactor
     7.8 +viff.reactor.install()
     7.9  from twisted.internet import reactor
    7.10  
    7.11  from viff.field import GF
     8.1 --- a/apps/eval-poly.py	Thu May 14 12:01:54 2009 +0200
     8.2 +++ b/apps/eval-poly.py	Tue May 19 17:04:07 2009 +0200
     8.3 @@ -26,6 +26,8 @@
     8.4  from time import time
     8.5  
     8.6  from optparse import OptionParser
     8.7 +import viff.reactor
     8.8 +viff.reactor.install()
     8.9  from twisted.internet import reactor
    8.10  
    8.11  from viff.field import GF
     9.1 --- a/apps/gc-test.py	Thu May 14 12:01:54 2009 +0200
     9.2 +++ b/apps/gc-test.py	Tue May 19 17:04:07 2009 +0200
     9.3 @@ -26,6 +26,8 @@
     9.4  import sys
     9.5  from time import time
     9.6  
     9.7 +import viff.reactor
     9.8 +viff.reactor.install()
     9.9  from twisted.internet import reactor
    9.10  
    9.11  from viff.field import GF
    10.1 --- a/apps/int-bit-conversion.py	Thu May 14 12:01:54 2009 +0200
    10.2 +++ b/apps/int-bit-conversion.py	Tue May 19 17:04:07 2009 +0200
    10.3 @@ -19,6 +19,8 @@
    10.4  
    10.5  import sys
    10.6  
    10.7 +import viff.reactor
    10.8 +viff.reactor.install()
    10.9  from twisted.internet import reactor
   10.10  
   10.11  from viff.field import GF, GF256
    11.1 --- a/apps/millionaires.py	Thu May 14 12:01:54 2009 +0200
    11.2 +++ b/apps/millionaires.py	Tue May 19 17:04:07 2009 +0200
    11.3 @@ -32,6 +32,8 @@
    11.4  # the example with '--help' for help with the command line options.
    11.5  
    11.6  from optparse import OptionParser
    11.7 +import viff.reactor
    11.8 +viff.reactor.install()
    11.9  from twisted.internet import reactor
   11.10  
   11.11  from viff.field import GF
    12.1 --- a/apps/multiply.py	Thu May 14 12:01:54 2009 +0200
    12.2 +++ b/apps/multiply.py	Tue May 19 17:04:07 2009 +0200
    12.3 @@ -18,6 +18,8 @@
    12.4  # License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
    12.5  
    12.6  from optparse import OptionParser
    12.7 +import viff.reactor
    12.8 +viff.reactor.install()
    12.9  from twisted.internet import reactor
   12.10  
   12.11  from viff.field import GF
    13.1 --- a/apps/online-comparison-benchmark.py	Thu May 14 12:01:54 2009 +0200
    13.2 +++ b/apps/online-comparison-benchmark.py	Tue May 19 17:04:07 2009 +0200
    13.3 @@ -21,6 +21,8 @@
    13.4  import signal
    13.5  from optparse import OptionParser
    13.6  
    13.7 +import viff.reactor
    13.8 +viff.reactor.install()
    13.9  from twisted.internet import reactor
   13.10  from twisted.internet.defer import DeferredList, gatherResults
   13.11  
    14.1 --- a/apps/paillier.py	Thu May 14 12:01:54 2009 +0200
    14.2 +++ b/apps/paillier.py	Tue May 19 17:04:07 2009 +0200
    14.3 @@ -26,6 +26,8 @@
    14.4  
    14.5  import sys
    14.6  
    14.7 +import viff.reactor
    14.8 +viff.reactor.install()
    14.9  from twisted.internet import reactor
   14.10  
   14.11  from viff.field import GF
    15.1 --- a/apps/prss-and-open.py	Thu May 14 12:01:54 2009 +0200
    15.2 +++ b/apps/prss-and-open.py	Tue May 19 17:04:07 2009 +0200
    15.3 @@ -19,6 +19,8 @@
    15.4  
    15.5  import sys
    15.6  
    15.7 +import viff.reactor
    15.8 +viff.reactor.install()
    15.9  from twisted.internet import reactor
   15.10  from twisted.internet.defer import gatherResults
   15.11  
    16.1 --- a/apps/seq-mul.py	Thu May 14 12:01:54 2009 +0200
    16.2 +++ b/apps/seq-mul.py	Tue May 19 17:04:07 2009 +0200
    16.3 @@ -18,6 +18,8 @@
    16.4  # License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
    16.5  
    16.6  from optparse import OptionParser
    16.7 +import viff.reactor
    16.8 +viff.reactor.install()
    16.9  from twisted.internet import reactor
   16.10  
   16.11  from viff.field import GF
    17.1 --- a/apps/shamir-share-open.py	Thu May 14 12:01:54 2009 +0200
    17.2 +++ b/apps/shamir-share-open.py	Tue May 19 17:04:07 2009 +0200
    17.3 @@ -19,6 +19,8 @@
    17.4  
    17.5  import sys
    17.6  
    17.7 +import viff.reactor
    17.8 +viff.reactor.install()
    17.9  from twisted.internet import reactor
   17.10  
   17.11  from viff.field import GF
    18.1 --- a/apps/share-open.py	Thu May 14 12:01:54 2009 +0200
    18.2 +++ b/apps/share-open.py	Tue May 19 17:04:07 2009 +0200
    18.3 @@ -19,6 +19,8 @@
    18.4  
    18.5  import sys
    18.6  
    18.7 +import viff.reactor
    18.8 +viff.reactor.install()
    18.9  from twisted.internet import reactor
   18.10  
   18.11  from viff.field import GF256
    19.1 --- a/apps/sort.py	Thu May 14 12:01:54 2009 +0200
    19.2 +++ b/apps/sort.py	Tue May 19 17:04:07 2009 +0200
    19.3 @@ -51,6 +51,8 @@
    19.4  
    19.5  from math import log, floor
    19.6  from optparse import OptionParser
    19.7 +import viff.reactor
    19.8 +viff.reactor.install()
    19.9  from twisted.internet import reactor
   19.10  
   19.11  from progressbar import ProgressBar, Percentage, Bar, ETA, ProgressBarWidget
    20.1 --- a/apps/sum.py	Thu May 14 12:01:54 2009 +0200
    20.2 +++ b/apps/sum.py	Tue May 19 17:04:07 2009 +0200
    20.3 @@ -18,6 +18,8 @@
    20.4  # License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
    20.5  
    20.6  from optparse import OptionParser
    20.7 +import viff.reactor
    20.8 +viff.reactor.install()
    20.9  from twisted.internet import reactor
   20.10  
   20.11  from viff.field import GF
    21.1 --- a/apps/two-fields.py	Thu May 14 12:01:54 2009 +0200
    21.2 +++ b/apps/two-fields.py	Tue May 19 17:04:07 2009 +0200
    21.3 @@ -19,6 +19,8 @@
    21.4  
    21.5  import sys
    21.6  
    21.7 +import viff.reactor
    21.8 +viff.reactor.install()
    21.9  from twisted.internet import reactor
   21.10  
   21.11  from viff.field import GF
    22.1 --- a/apps/xor-all.py	Thu May 14 12:01:54 2009 +0200
    22.2 +++ b/apps/xor-all.py	Tue May 19 17:04:07 2009 +0200
    22.3 @@ -19,6 +19,8 @@
    22.4  
    22.5  import sys
    22.6  
    22.7 +import viff.reactor
    22.8 +viff.reactor.install()
    22.9  from twisted.internet import reactor
   22.10  
   22.11  from viff.field import GF256
    23.1 --- a/viff/active.py	Thu May 14 12:01:54 2009 +0200
    23.2 +++ b/viff/active.py	Tue May 19 17:04:07 2009 +0200
    23.3 @@ -70,6 +70,9 @@
    23.4              for protocol in self.protocols.itervalues():
    23.5                  protocol.sendData(pc, data_type, message)
    23.6  
    23.7 +            # do actual communication
    23.8 +            self.activate_reactor()
    23.9 +
   23.10          def echo_received(message, peer_id):
   23.11              # This is called when we receive an echo message. It
   23.12              # updates the echo count for the message and enters the
   23.13 @@ -133,6 +136,8 @@
   23.14              d_send = Deferred().addCallback(send_received)
   23.15              self._expect_data(sender, SEND, d_send)
   23.16  
   23.17 +        # do actual communication
   23.18 +        self.activate_reactor()
   23.19  
   23.20          return result
   23.21  
   23.22 @@ -261,6 +266,9 @@
   23.23                  # first T shares.
   23.24                  return rvec[:T]
   23.25  
   23.26 +            # do actual communication
   23.27 +            self.activate_reactor()
   23.28 +
   23.29          result = gather_shares(svec[T:])
   23.30          self.schedule_callback(result, exchange)
   23.31          return result
   23.32 @@ -361,6 +369,9 @@
   23.33                  # first T shares.
   23.34                  return (rvec1[:T], rvec2[:T])
   23.35  
   23.36 +            # do actual communication
   23.37 +            self.activate_reactor()
   23.38 +
   23.39          result = gather_shares([gather_shares(svec1[T:]), gather_shares(svec2[T:])])
   23.40          self.schedule_callback(result, exchange)
   23.41          return result
    24.1 --- a/viff/passive.py	Thu May 14 12:01:54 2009 +0200
    24.2 +++ b/viff/passive.py	Tue May 19 17:04:07 2009 +0200
    24.3 @@ -104,6 +104,10 @@
    24.4  
    24.5          result = share.clone()
    24.6          self.schedule_callback(result, exchange)
    24.7 +
    24.8 +        # do actual communication
    24.9 +        self.activate_reactor()
   24.10 +
   24.11          if self.id in receivers:
   24.12              return result
   24.13  
   24.14 @@ -204,6 +208,10 @@
   24.15          result = gather_shares([share_a, share_b])
   24.16          result.addCallback(lambda (a, b): a * b)
   24.17          self.schedule_callback(result, share_recombine)
   24.18 +
   24.19 +        # do actual communication
   24.20 +        self.activate_reactor()
   24.21 +
   24.22          return result
   24.23  
   24.24      def pow(self, share, exponent):
   24.25 @@ -501,6 +509,9 @@
   24.26              else:
   24.27                  results.append(self._expect_share(peer_id, field))
   24.28  
   24.29 +        # do actual communication
   24.30 +        self.activate_reactor()
   24.31 +
   24.32          # Unpack a singleton list.
   24.33          if len(results) == 1:
   24.34              return results[0]
    25.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    25.2 +++ b/viff/reactor.py	Tue May 19 17:04:07 2009 +0200
    25.3 @@ -0,0 +1,55 @@
    25.4 +# -*- coding: utf-8 -*-
    25.5 +#
    25.6 +# Copyright 2009 VIFF Development Team.
    25.7 +#
    25.8 +# This file is part of VIFF, the Virtual Ideal Functionality Framework.
    25.9 +#
   25.10 +# VIFF is free software: you can redistribute it and/or modify it
   25.11 +# under the terms of the GNU Lesser General Public License (LGPL) as
   25.12 +# published by the Free Software Foundation, either version 3 of the
   25.13 +# License, or (at your option) any later version.
   25.14 +#
   25.15 +# VIFF is distributed in the hope that it will be useful, but WITHOUT
   25.16 +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
   25.17 +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
   25.18 +# Public License for more details.
   25.19 +#
   25.20 +# You should have received a copy of the GNU Lesser General Public
   25.21 +# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
   25.22 +
   25.23 +"""VIFF reactor to have control over the scheduling."""
   25.24 +
   25.25 +__docformat__ = "restructuredtext"
   25.26 +
   25.27 +from twisted.internet.selectreactor import SelectReactor
   25.28 +
   25.29 +
   25.30 +class ViffReactor(SelectReactor):
   25.31 +    """VIFF reactor.
   25.32 +
   25.33 +    The only difference to the SelectReactor is the loop call.
   25.34 +    From there, doIteration() can be called recursively."""
   25.35 +
   25.36 +    def __init__(self):
   25.37 +        SelectReactor.__init__(self)
   25.38 +        self.loopCall = lambda: None
   25.39 +   
   25.40 +    def setLoopCall(self, f):
   25.41 +        self.loopCall = f
   25.42 +
   25.43 +    def doIteration(self, t):
   25.44 +        # Do the same as in mainLoop() first.
   25.45 +        self.runUntilCurrent()
   25.46 +        t2 = self.timeout()
   25.47 +
   25.48 +        if (t2 is not None):
   25.49 +            t = min(t, self.running and t2)
   25.50 +
   25.51 +        SelectReactor.doIteration(self, t)
   25.52 +        self.loopCall()
   25.53 +
   25.54 +def install():
   25.55 +    """Use the VIFF reactor."""
   25.56 +    reactor = ViffReactor()
   25.57 +    from twisted.internet.main import installReactor
   25.58 +    installReactor(reactor)
    26.1 --- a/viff/runtime.py	Thu May 14 12:01:54 2009 +0200
    26.2 +++ b/viff/runtime.py	Tue May 19 17:04:07 2009 +0200
    26.3 @@ -318,7 +318,10 @@
    26.4                      deferred = deq.popleft()
    26.5                      if not deq:
    26.6                          del self.waiting_deferreds[key]
    26.7 -                    deferred.callback(data)
    26.8 +
    26.9 +                    # Just queue, callbacks will be executed
   26.10 +                    # in process_deferred_queue().
   26.11 +                    self.factory.runtime.queue_deferred(deferred, data)
   26.12                  else:
   26.13                      deq = self.incoming_data.setdefault(key, deque())
   26.14                      deq.append(data)
   26.15 @@ -533,6 +536,14 @@
   26.16          # communicating with ourselves.
   26.17          self.add_player(player, None)
   26.18  
   26.19 +        #: Queue of deferreds and data.
   26.20 +        self.deferred_queue = []
   26.21 +        #: Counter for calls of activate_reactor().
   26.22 +        self.activation_counter = 0
   26.23 +        #: Record the recursion depth.
   26.24 +        self.depth_counter = 0
   26.25 +        self.max_depth = 0
   26.26 +
   26.27      def add_player(self, player, protocol):
   26.28          self.players[player.id] = player
   26.29          self.num_players = len(self.players)
   26.30 @@ -761,6 +772,39 @@
   26.31          Python integer."""
   26.32          raise NotImplemented("Override this abstract method in a subclass.")
   26.33  
   26.34 +    def queue_deferred(self, deferred, data):
   26.35 +        """Put deferred and data into the queue."""
   26.36 +
   26.37 +        self.deferred_queue.append((deferred, data))
   26.38 +
   26.39 +    def process_deferred_queue(self):
   26.40 +        """Execute the callbacks of the deferreds in the queue."""
   26.41 +
   26.42 +        while(self.deferred_queue):
   26.43 +            deferred, data = self.deferred_queue.pop(0)
   26.44 +            deferred.callback(data)
   26.45 +
   26.46 +    def activate_reactor(self):
   26.47 +        """Activate the reactor to do actual communcation.
   26.48 +
   26.49 +        This is where the recursion happens."""
   26.50 +
   26.51 +        self.activation_counter += 1
   26.52 +
   26.53 +        # setting the number to n makes the reactor called 
   26.54 +        # only every n-th time
   26.55 +        if (self.activation_counter >= 2):
   26.56 +            self.depth_counter += 1
   26.57 +
   26.58 +            if (self.depth_counter > self.max_depth):
   26.59 +                # Record the maximal depth reached.
   26.60 +                self.max_depth = self.depth_counter
   26.61 +
   26.62 +            reactor.doIteration(0)
   26.63 +
   26.64 +            self.depth_counter -= 1
   26.65 +            self.activation_counter = 0
   26.66 +
   26.67  
   26.68  def make_runtime_class(runtime_class=None, mixins=None):
   26.69      """Creates a new runtime class with *runtime_class* as a base
   26.70 @@ -912,6 +956,9 @@
   26.71              print "Will connect to %s" % player
   26.72              connect(player.host, player.port)
   26.73  
   26.74 +    # Process the deferred queue after every reactor iteration.
   26.75 +    reactor.setLoopCall(runtime.process_deferred_queue)
   26.76 +
   26.77      return result
   26.78  
   26.79  if __name__ == "__main__":
    27.1 --- a/viff/test/__init__.py	Thu May 14 12:01:54 2009 +0200
    27.2 +++ b/viff/test/__init__.py	Tue May 19 17:04:07 2009 +0200
    27.3 @@ -14,3 +14,6 @@
    27.4  #
    27.5  # You should have received a copy of the GNU Lesser General Public
    27.6  # License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
    27.7 +
    27.8 +import viff.reactor
    27.9 +viff.reactor.install()
    28.1 --- a/viff/test/util.py	Thu May 14 12:01:54 2009 +0200
    28.2 +++ b/viff/test/util.py	Tue May 19 17:04:07 2009 +0200
    28.3 @@ -19,6 +19,7 @@
    28.4  
    28.5  from twisted.internet.defer import Deferred, gatherResults, maybeDeferred
    28.6  from twisted.trial.unittest import TestCase
    28.7 +from twisted.internet import reactor
    28.8  
    28.9  from viff.passive import PassiveRuntime
   28.10  from viff.runtime import Share, ShareExchanger, ShareExchangerFactory
   28.11 @@ -127,6 +128,7 @@
   28.12          self.close_sentinels = []
   28.13  
   28.14          self.runtimes = []
   28.15 +        self.real_runtimes = []
   28.16          for id in reversed(range(1, self.num_players+1)):
   28.17              _, players = load_config(configs[id])
   28.18              self.create_loopback_runtime(id, players)
   28.19 @@ -164,6 +166,18 @@
   28.20          # the Runtime, since we want everybody to wait until all
   28.21          # runtimes are ready.
   28.22          self.runtimes.append(result)
   28.23 +        self.real_runtimes.append(runtime)
   28.24 +        self.i = 0
   28.25 +
   28.26 +        # This loop call should ensure the queues of the parties are
   28.27 +        # processed in a more or less fair manner. This is necessary
   28.28 +        # because we have only one reactor for all parties here.
   28.29 +        def loop_call():
   28.30 +            for runtime in self.real_runtimes[self.i:] + self.real_runtimes[:self.i]:
   28.31 +                runtime.process_deferred_queue()
   28.32 +                self.i = (self.i + 1) % len(self.real_runtimes)
   28.33 +
   28.34 +        reactor.setLoopCall(loop_call)
   28.35  
   28.36          for peer_id in players:
   28.37              if peer_id != id: