viff

changeset 1335:ed3f546283ec

benchmark: shorter name for benchmark util module
author Martin Geisler <mg@cs.au.dk>
date Tue, 27 Oct 2009 11:33:22 +0100
parents 0d3460e28e84
children 13f6a14cafe0
files apps/benchmark.py apps/benchmark_classes.py apps/benchutil.py
diffstat 3 files changed, 249 insertions(+), 249 deletions(-) [+]
line diff
     1.1 --- a/apps/benchmark.py	Tue Oct 27 11:20:25 2009 +0100
     1.2 +++ b/apps/benchmark.py	Tue Oct 27 11:33:22 2009 +0100
     1.3 @@ -74,7 +74,7 @@
     1.4  from viff.config import load_config
     1.5  from viff.util import find_prime
     1.6  
     1.7 -from benchmark_classes import SelfcontainedBenchmarkStrategy, \
     1.8 +from benchutil import SelfcontainedBenchmarkStrategy, \
     1.9      NeededDataBenchmarkStrategy, ParallelBenchmark, SequentialBenchmark, BinaryOperation, NullaryOperation
    1.10  
    1.11  # Hack in order to avoid Maximum recursion depth exceeded
     2.1 --- a/apps/benchmark_classes.py	Tue Oct 27 11:20:25 2009 +0100
     2.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.3 @@ -1,248 +0,0 @@
     2.4 -# Copyright 2009 VIFF Development Team.
     2.5 -#
     2.6 -# This file is part of VIFF, the Virtual Ideal Functionality Framework.
     2.7 -#
     2.8 -# VIFF is free software: you can redistribute it and/or modify it
     2.9 -# under the terms of the GNU Lesser General Public License (LGPL) as
    2.10 -# published by the Free Software Foundation, either version 3 of the
    2.11 -# License, or (at your option) any later version.
    2.12 -#
    2.13 -# VIFF is distributed in the hope that it will be useful, but WITHOUT
    2.14 -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
    2.15 -# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
    2.16 -# Public License for more details.
    2.17 -#
    2.18 -# You should have received a copy of the GNU Lesser General Public
    2.19 -# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
    2.20 -
    2.21 -import sys
    2.22 -import time
    2.23 -
    2.24 -from pprint import pformat
    2.25 -
    2.26 -from twisted.internet.defer import gatherResults
    2.27 -
    2.28 -from viff.runtime import gather_shares
    2.29 -from viff.util import rand
    2.30 -
    2.31 -start = 0
    2.32 -
    2.33 -
    2.34 -def record_start(what):
    2.35 -    global start
    2.36 -    start = time.time()
    2.37 -    print "*" * 64
    2.38 -    print "Started", what
    2.39 -
    2.40 -
    2.41 -def record_stop(x, what, count):
    2.42 -    stop = time.time()
    2.43 -    print
    2.44 -    print "Total time used: %.3f sec" % (stop-start)
    2.45 -    print "Time per %s operation: %.0f ms" % (what, 1000*(stop-start) / count)
    2.46 -    print "Throughput: %d per second" % (count / (stop-start))
    2.47 -    print "*" * 64
    2.48 -    return x
    2.49 -
    2.50 -
    2.51 -class Benchmark(object):
    2.52 -    """Abstract base class for all Benchmarks.
    2.53 -
    2.54 -    For concrete classes see the `ParallelBenchmark` and
    2.55 -    `SequentialBenchmark` classes. A concrete class must be mixed with
    2.56 -    a `BenchmarkStrategy` and an `Operator`.
    2.57 -    """
    2.58 -
    2.59 -    def __init__(self, rt, operation, field, count):
    2.60 -        self.rt = rt
    2.61 -        self.operation = getattr(rt, operation)
    2.62 -        self.pc = None
    2.63 -        self.field = field
    2.64 -        self.count = count
    2.65 -
    2.66 -    def preprocess(self, needed_data):
    2.67 -        print "Preprocess", needed_data
    2.68 -        if needed_data:
    2.69 -            print "Starting preprocessing"
    2.70 -            record_start("preprocessing")
    2.71 -            preproc = self.rt.preprocess(needed_data)
    2.72 -            preproc.addCallback(record_stop, "preprocessing", self.count)
    2.73 -            return preproc
    2.74 -        else:
    2.75 -            print "Need no preprocessing"
    2.76 -            return None
    2.77 -
    2.78 -    def test(self, d, termination_function):
    2.79 -        self.rt.schedule_callback(d, self.generate_operation_arguments)
    2.80 -        self.rt.schedule_callback(d, self.sync_test)
    2.81 -        self.rt.schedule_callback(d, self.run_test)
    2.82 -        self.rt.schedule_callback(d, self.sync_test)
    2.83 -        self.rt.schedule_callback(d, self.finished, termination_function)
    2.84 -        return d
    2.85 -
    2.86 -    def sync_test(self, x):
    2.87 -        print "Synchronizing test start."
    2.88 -        sys.stdout.flush()
    2.89 -        sync = self.rt.synchronize()
    2.90 -        self.rt.schedule_callback(sync, lambda y: x)
    2.91 -        return sync
    2.92 -
    2.93 -    def run_test(self, _):
    2.94 -        raise NotImplementedError
    2.95 -
    2.96 -    def finished(self, needed_data, termination_function):
    2.97 -        sys.stdout.flush()
    2.98 -
    2.99 -        if self.rt._needed_data:
   2.100 -            print "Missing pre-processed data:"
   2.101 -            for (func, args), pcs in needed_data.iteritems():
   2.102 -                print "* %s%s:" % (func, args)
   2.103 -                print "  " + pformat(pcs).replace("\n", "\n  ")
   2.104 -
   2.105 -        return termination_function(needed_data)
   2.106 -
   2.107 -
   2.108 -class ParallelBenchmark(Benchmark):
   2.109 -    """This class implements a benchmark where run_test executes all
   2.110 -    operations in parallel."""
   2.111 -
   2.112 -    def run_test(self, shares):
   2.113 -        # print "rt", self.rt.program_counter, self.pc
   2.114 -        if self.pc != None:
   2.115 -            self.rt.program_counter = self.pc
   2.116 -        else:
   2.117 -            self.pc = list(self.rt.program_counter)
   2.118 -        c_shares = []
   2.119 -        record_start("parallel test")
   2.120 -        while not self.is_operation_done():
   2.121 -            c_shares.append(self.do_operation())
   2.122 -
   2.123 -        done = gatherResults(c_shares)
   2.124 -        done.addCallback(record_stop, "parallel test", self.count)
   2.125 -        def f(x):
   2.126 -            needed_data = self.rt._needed_data
   2.127 -            self.rt._needed_data = {}
   2.128 -            return needed_data
   2.129 -        done.addCallback(f)
   2.130 -        return done
   2.131 -
   2.132 -
   2.133 -class SequentialBenchmark(Benchmark):
   2.134 -    """A benchmark where the operations are executed one after each
   2.135 -    other."""
   2.136 -
   2.137 -    def run_test(self, _, termination_function, d):
   2.138 -        record_start("sequential test")
   2.139 -        self.single_operation(None, termination_function)
   2.140 -
   2.141 -    def single_operation(self, _, termination_function):
   2.142 -        if not self.is_operation_done():
   2.143 -            c = self.do_operation()
   2.144 -            self.rt.schedule_callback(c, self.single_operation, termination_function)
   2.145 -        else:
   2.146 -            record_stop(None, "sequential test", self.count)
   2.147 -            self.finished(None, termination_function)
   2.148 -
   2.149 -
   2.150 -class Operation(object):
   2.151 -    """An abstract mixin which encapsulate the behaviour of an operation.
   2.152 -
   2.153 -    An operation can be nullary, unary, binary, etc.
   2.154 -    """
   2.155 -
   2.156 -    def generate_operation_arguments(self, _):
   2.157 -        """Generate the input need for performing the operation.
   2.158 -
   2.159 -        Returns: None.
   2.160 -        """
   2.161 -        raise NotImplementedError
   2.162 -
   2.163 -    def is_operation_done(self):
   2.164 -        """Returns true if there are no more operations to perform.
   2.165 -        Used in sequential tests.
   2.166 -
   2.167 -        Returns: Boolean.
   2.168 -        """
   2.169 -        raise NotImplementedError
   2.170 -
   2.171 -    def do_operation(self):
   2.172 -        """Perform the operation.
   2.173 -
   2.174 -        Returns: A share containing the result of the operation.
   2.175 -        """
   2.176 -        raise NotImplementedError
   2.177 -
   2.178 -class BinaryOperation(Operation):
   2.179 -    """A binary operation."""
   2.180 -
   2.181 -    def generate_operation_arguments(self, _):
   2.182 -        # print "Generate operation arguments", self.rt.program_counter
   2.183 -        print "Runtime ready, generating shares"
   2.184 -        self.a_shares = []
   2.185 -        self.b_shares = []
   2.186 -        for i in range(self.count):
   2.187 -            inputter = (i % len(self.rt.players)) + 1
   2.188 -            if inputter == self.rt.id:
   2.189 -                a = rand.randint(0, self.field.modulus)
   2.190 -                b = rand.randint(0, self.field.modulus)
   2.191 -            else:
   2.192 -                a, b = None, None
   2.193 -            self.a_shares.append(self.rt.input([inputter], self.field, a))
   2.194 -            self.b_shares.append(self.rt.input([inputter], self.field, b))
   2.195 -        shares_ready = gather_shares(self.a_shares + self.b_shares)
   2.196 -        return shares_ready
   2.197 -
   2.198 -    def is_operation_done(self):
   2.199 -        return not (self.a_shares and self.b_shares)
   2.200 -
   2.201 -    def do_operation(self):
   2.202 -        a = self.a_shares.pop()
   2.203 -        b = self.b_shares.pop()
   2.204 -        return self.operation(a, b)
   2.205 -
   2.206 -
   2.207 -class NullaryOperation(Operation):
   2.208 -    """A nullary operation."""
   2.209 -
   2.210 -    def generate_operation_arguments(self, _):
   2.211 -        self.nullary_tests = self.count
   2.212 -        return None
   2.213 -
   2.214 -    def is_operation_done(self):
   2.215 -        return self.nullary_tests == 0
   2.216 -
   2.217 -    def do_operation(self):
   2.218 -        self.nullary_tests -= 1
   2.219 -        return self.operation(self.field)
   2.220 -
   2.221 -
   2.222 -class BenchmarkStrategy(object):
   2.223 -    """A benchmark strategy defines how the benchmark is done."""
   2.224 -
   2.225 -    def benchmark(self, *args):
   2.226 -        raise NotImplementedError
   2.227 -
   2.228 -
   2.229 -class SelfcontainedBenchmarkStrategy(BenchmarkStrategy):
   2.230 -    """In a self contained benchmark strategy, all the needed data is
   2.231 -    generated on the fly."""
   2.232 -
   2.233 -    def benchmark(self, *args):
   2.234 -        sys.stdout.flush()
   2.235 -        sync = self.rt.synchronize()
   2.236 -        self.test(sync, lambda x: x)
   2.237 -        self.rt.schedule_callback(sync, self.preprocess)
   2.238 -        self.test(sync, lambda x: self.rt.shutdown())
   2.239 -
   2.240 -
   2.241 -class NeededDataBenchmarkStrategy(BenchmarkStrategy):
   2.242 -    """In a needed data benchmark strategy, all the needed data has to
   2.243 -    have been generated before the test is run."""
   2.244 -
   2.245 -    def benchmark(self, needed_data, pc, *args):
   2.246 -        self.pc = pc
   2.247 -        sys.stdout.flush()
   2.248 -        sync = self.rt.synchronize()
   2.249 -        self.rt.schedule_callback(sync, lambda x: needed_data)
   2.250 -        self.rt.schedule_callback(sync, self.preprocess)
   2.251 -        self.test(sync, lambda x: self.rt.shutdown())
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/apps/benchutil.py	Tue Oct 27 11:33:22 2009 +0100
     3.3 @@ -0,0 +1,248 @@
     3.4 +# Copyright 2009 VIFF Development Team.
     3.5 +#
     3.6 +# This file is part of VIFF, the Virtual Ideal Functionality Framework.
     3.7 +#
     3.8 +# VIFF is free software: you can redistribute it and/or modify it
     3.9 +# under the terms of the GNU Lesser General Public License (LGPL) as
    3.10 +# published by the Free Software Foundation, either version 3 of the
    3.11 +# License, or (at your option) any later version.
    3.12 +#
    3.13 +# VIFF is distributed in the hope that it will be useful, but WITHOUT
    3.14 +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
    3.15 +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General
    3.16 +# Public License for more details.
    3.17 +#
    3.18 +# You should have received a copy of the GNU Lesser General Public
    3.19 +# License along with VIFF. If not, see <http://www.gnu.org/licenses/>.
    3.20 +
    3.21 +import sys
    3.22 +import time
    3.23 +
    3.24 +from pprint import pformat
    3.25 +
    3.26 +from twisted.internet.defer import gatherResults
    3.27 +
    3.28 +from viff.runtime import gather_shares
    3.29 +from viff.util import rand
    3.30 +
    3.31 +start = 0
    3.32 +
    3.33 +
    3.34 +def record_start(what):
    3.35 +    global start
    3.36 +    start = time.time()
    3.37 +    print "*" * 64
    3.38 +    print "Started", what
    3.39 +
    3.40 +
    3.41 +def record_stop(x, what, count):
    3.42 +    stop = time.time()
    3.43 +    print
    3.44 +    print "Total time used: %.3f sec" % (stop-start)
    3.45 +    print "Time per %s operation: %.0f ms" % (what, 1000*(stop-start) / count)
    3.46 +    print "Throughput: %d per second" % (count / (stop-start))
    3.47 +    print "*" * 64
    3.48 +    return x
    3.49 +
    3.50 +
    3.51 +class Benchmark(object):
    3.52 +    """Abstract base class for all Benchmarks.
    3.53 +
    3.54 +    For concrete classes see the `ParallelBenchmark` and
    3.55 +    `SequentialBenchmark` classes. A concrete class must be mixed with
    3.56 +    a `BenchmarkStrategy` and an `Operator`.
    3.57 +    """
    3.58 +
    3.59 +    def __init__(self, rt, operation, field, count):
    3.60 +        self.rt = rt
    3.61 +        self.operation = getattr(rt, operation)
    3.62 +        self.pc = None
    3.63 +        self.field = field
    3.64 +        self.count = count
    3.65 +
    3.66 +    def preprocess(self, needed_data):
    3.67 +        print "Preprocess", needed_data
    3.68 +        if needed_data:
    3.69 +            print "Starting preprocessing"
    3.70 +            record_start("preprocessing")
    3.71 +            preproc = self.rt.preprocess(needed_data)
    3.72 +            preproc.addCallback(record_stop, "preprocessing", self.count)
    3.73 +            return preproc
    3.74 +        else:
    3.75 +            print "Need no preprocessing"
    3.76 +            return None
    3.77 +
    3.78 +    def test(self, d, termination_function):
    3.79 +        self.rt.schedule_callback(d, self.generate_operation_arguments)
    3.80 +        self.rt.schedule_callback(d, self.sync_test)
    3.81 +        self.rt.schedule_callback(d, self.run_test)
    3.82 +        self.rt.schedule_callback(d, self.sync_test)
    3.83 +        self.rt.schedule_callback(d, self.finished, termination_function)
    3.84 +        return d
    3.85 +
    3.86 +    def sync_test(self, x):
    3.87 +        print "Synchronizing test start."
    3.88 +        sys.stdout.flush()
    3.89 +        sync = self.rt.synchronize()
    3.90 +        self.rt.schedule_callback(sync, lambda y: x)
    3.91 +        return sync
    3.92 +
    3.93 +    def run_test(self, _):
    3.94 +        raise NotImplementedError
    3.95 +
    3.96 +    def finished(self, needed_data, termination_function):
    3.97 +        sys.stdout.flush()
    3.98 +
    3.99 +        if self.rt._needed_data:
   3.100 +            print "Missing pre-processed data:"
   3.101 +            for (func, args), pcs in needed_data.iteritems():
   3.102 +                print "* %s%s:" % (func, args)
   3.103 +                print "  " + pformat(pcs).replace("\n", "\n  ")
   3.104 +
   3.105 +        return termination_function(needed_data)
   3.106 +
   3.107 +
   3.108 +class ParallelBenchmark(Benchmark):
   3.109 +    """This class implements a benchmark where run_test executes all
   3.110 +    operations in parallel."""
   3.111 +
   3.112 +    def run_test(self, shares):
   3.113 +        # print "rt", self.rt.program_counter, self.pc
   3.114 +        if self.pc != None:
   3.115 +            self.rt.program_counter = self.pc
   3.116 +        else:
   3.117 +            self.pc = list(self.rt.program_counter)
   3.118 +        c_shares = []
   3.119 +        record_start("parallel test")
   3.120 +        while not self.is_operation_done():
   3.121 +            c_shares.append(self.do_operation())
   3.122 +
   3.123 +        done = gatherResults(c_shares)
   3.124 +        done.addCallback(record_stop, "parallel test", self.count)
   3.125 +        def f(x):
   3.126 +            needed_data = self.rt._needed_data
   3.127 +            self.rt._needed_data = {}
   3.128 +            return needed_data
   3.129 +        done.addCallback(f)
   3.130 +        return done
   3.131 +
   3.132 +
   3.133 +class SequentialBenchmark(Benchmark):
   3.134 +    """A benchmark where the operations are executed one after each
   3.135 +    other."""
   3.136 +
   3.137 +    def run_test(self, _, termination_function, d):
   3.138 +        record_start("sequential test")
   3.139 +        self.single_operation(None, termination_function)
   3.140 +
   3.141 +    def single_operation(self, _, termination_function):
   3.142 +        if not self.is_operation_done():
   3.143 +            c = self.do_operation()
   3.144 +            self.rt.schedule_callback(c, self.single_operation, termination_function)
   3.145 +        else:
   3.146 +            record_stop(None, "sequential test", self.count)
   3.147 +            self.finished(None, termination_function)
   3.148 +
   3.149 +
   3.150 +class Operation(object):
   3.151 +    """An abstract mixin which encapsulate the behaviour of an operation.
   3.152 +
   3.153 +    An operation can be nullary, unary, binary, etc.
   3.154 +    """
   3.155 +
   3.156 +    def generate_operation_arguments(self, _):
   3.157 +        """Generate the input need for performing the operation.
   3.158 +
   3.159 +        Returns: None.
   3.160 +        """
   3.161 +        raise NotImplementedError
   3.162 +
   3.163 +    def is_operation_done(self):
   3.164 +        """Returns true if there are no more operations to perform.
   3.165 +        Used in sequential tests.
   3.166 +
   3.167 +        Returns: Boolean.
   3.168 +        """
   3.169 +        raise NotImplementedError
   3.170 +
   3.171 +    def do_operation(self):
   3.172 +        """Perform the operation.
   3.173 +
   3.174 +        Returns: A share containing the result of the operation.
   3.175 +        """
   3.176 +        raise NotImplementedError
   3.177 +
   3.178 +class BinaryOperation(Operation):
   3.179 +    """A binary operation."""
   3.180 +
   3.181 +    def generate_operation_arguments(self, _):
   3.182 +        # print "Generate operation arguments", self.rt.program_counter
   3.183 +        print "Runtime ready, generating shares"
   3.184 +        self.a_shares = []
   3.185 +        self.b_shares = []
   3.186 +        for i in range(self.count):
   3.187 +            inputter = (i % len(self.rt.players)) + 1
   3.188 +            if inputter == self.rt.id:
   3.189 +                a = rand.randint(0, self.field.modulus)
   3.190 +                b = rand.randint(0, self.field.modulus)
   3.191 +            else:
   3.192 +                a, b = None, None
   3.193 +            self.a_shares.append(self.rt.input([inputter], self.field, a))
   3.194 +            self.b_shares.append(self.rt.input([inputter], self.field, b))
   3.195 +        shares_ready = gather_shares(self.a_shares + self.b_shares)
   3.196 +        return shares_ready
   3.197 +
   3.198 +    def is_operation_done(self):
   3.199 +        return not (self.a_shares and self.b_shares)
   3.200 +
   3.201 +    def do_operation(self):
   3.202 +        a = self.a_shares.pop()
   3.203 +        b = self.b_shares.pop()
   3.204 +        return self.operation(a, b)
   3.205 +
   3.206 +
   3.207 +class NullaryOperation(Operation):
   3.208 +    """A nullary operation."""
   3.209 +
   3.210 +    def generate_operation_arguments(self, _):
   3.211 +        self.nullary_tests = self.count
   3.212 +        return None
   3.213 +
   3.214 +    def is_operation_done(self):
   3.215 +        return self.nullary_tests == 0
   3.216 +
   3.217 +    def do_operation(self):
   3.218 +        self.nullary_tests -= 1
   3.219 +        return self.operation(self.field)
   3.220 +
   3.221 +
   3.222 +class BenchmarkStrategy(object):
   3.223 +    """A benchmark strategy defines how the benchmark is done."""
   3.224 +
   3.225 +    def benchmark(self, *args):
   3.226 +        raise NotImplementedError
   3.227 +
   3.228 +
   3.229 +class SelfcontainedBenchmarkStrategy(BenchmarkStrategy):
   3.230 +    """In a self contained benchmark strategy, all the needed data is
   3.231 +    generated on the fly."""
   3.232 +
   3.233 +    def benchmark(self, *args):
   3.234 +        sys.stdout.flush()
   3.235 +        sync = self.rt.synchronize()
   3.236 +        self.test(sync, lambda x: x)
   3.237 +        self.rt.schedule_callback(sync, self.preprocess)
   3.238 +        self.test(sync, lambda x: self.rt.shutdown())
   3.239 +
   3.240 +
   3.241 +class NeededDataBenchmarkStrategy(BenchmarkStrategy):
   3.242 +    """In a needed data benchmark strategy, all the needed data has to
   3.243 +    have been generated before the test is run."""
   3.244 +
   3.245 +    def benchmark(self, needed_data, pc, *args):
   3.246 +        self.pc = pc
   3.247 +        sys.stdout.flush()
   3.248 +        sync = self.rt.synchronize()
   3.249 +        self.rt.schedule_callback(sync, lambda x: needed_data)
   3.250 +        self.rt.schedule_callback(sync, self.preprocess)
   3.251 +        self.test(sync, lambda x: self.rt.shutdown())