<commit_before>#! /usr/bin/env python
''' main module to read files, start, evaluate and output different experiments
  See -h for further information
'''

import argparse
import time
import mult_weights_solver as mws
import sariel_lp_solver as slpsolv
import fekete_lp_solver as flpsolv
import opt_solver
import highdimgraph
import plotting
import numpy as np

def main():
    args = parse_args()
    experiment = prepare_experiment(args)
    experiment.process()

# constants for options
SOLVER_OPTIONS = [ 'mult_weight', 'sariel_lp', 'fekete_lp', 'opt', 'all']
DATA_DISTRIBUTION_OPTIONS = ['uniform', 'grid']
LINE_OPTIONS = ['all', 'stabbing', 'random']

def parse_args():
    '''
    parse all arguments on the command line and return them
    '''
    parser = argparse.ArgumentParser(description=
            """
            run an parameterized experiment to compute
            a spanning tree with low crossing number
            """)
    parser.add_argument("-s", "--solver", default='mult_weight',
            choices=SOLVER_OPTIONS,
            help="choose an algorithm computing a feasible solution")
    parser.add_argument("-d", "--dimensions", type=int, default=2,
        help="number of dimensions of point set")
    parser.add_argument("-n", "--number", type=int, default=16,
        help="quantify how many points you want")
    parser.add_argument("-g", "--generate", default='uniform',
            choices=DATA_DISTRIBUTION_OPTIONS,
        help="how should the point set be sampled")
    parser.add_argument("-i", "--input", type=str, default=None,
        help="specify a input file")
    parser.add_argument("-l", "--linesampling", default='all',
            choices=LINE_OPTIONS,
        help="structure of the line set")
    parser.add_argument("-m", "--mean", type=int, default=1,
        help="run experiment m times and returns averaged results")
    parser.add_argument("-p", "--plot", action='store_true', default=False,
        help="plots the computed solution into a widget")
    parser.add_argument("-v", "--verbose", action='store_true',
        help="adds verbose outputs to STDOUT")
    args = parser.parse_args()
    return args

def prepare_experiment(args):
    '''
    factory for creation of experiments from arguments
    '''
    return create_experiment(args.solver, args.dimensions, args.number,
                             args.generate, args.linesampling, args.mean,
                             args.plot, args.verbose)

def create_experiment(solver_type, d, n, distribution_type, lines_type, mean,
                      has_plot, verbose, input_filename=None):

    if input_filename == None:
        if verbose:
            print "Start generating graph..."
        graph = generate_graph(d, n, distribution_type)
    else:
        if verbose:
            print "Reading graph from file %s..." % input_filename
        graph = graph_from_file(input_filename)
    if verbose:
        print "Graph has been created."

    if solver_type == 'all':
        experiment = SpanningTreeExperiment(SOLVER_OPTIONS[0], graph,
                                            lines_type, has_plot, verbose)
    else:
        experiment = SpanningTreeExperiment(solver_type, graph, lines_type,
                                            has_plot, verbose)

    if mean > 1:
        experiment = AveragedExperiment(mean, experiment)

    if solver_type == 'all':
        return AllSolversExperiment(experiment)
    else:
        return experiment

def generate_graph(d, n, distribution_type):
    '''
    generic sampling function for different point sets (dimensions,
    distributions)
    '''
    assert distribution_type in DATA_DISTRIBUTION_OPTIONS
    if distribution_type == 'uniform':
        return highdimgraph.create_uniform_graph(n, d)
    elif distribution_type == 'grid':
        return highdimgraph.create_grid_graph(n, d)

def graph_from_file(input_filename):
    pass

def get_solver(solver_type):
    '''
    select a solver by solver_type and use return solver function later
    '''
    assert solver_type in SOLVER_OPTIONS
    if solver_type == 'mult_weight':
        return mws.compute_spanning_tree
    elif solver_type == 'sariel_lp':
        return slpsolv.compute_spanning_tree
    elif solver_type == 'fekete_lp':
        return flpsolv.compute_spanning_tree
    elif solver_type == 'opt':
        return opt_solver.compute_spanning_tree
    else:
        raise StandardError('Not yet supported this |%s| solver type' %
                solver_type)

def generate_lines(graph, line_type, verbose):
    assert line_type in LINE_OPTIONS
    if verbose:
        print "Sampling of lines started..."
    if line_type == 'all':
        graph.create_all_lines()
        graph.preprocess_lines()
    elif line_type == 'stabbing':
        graph.create_stabbing_lines()
    elif line_type == 'random':
        graph.create_random_lines()
        graph.preprocess_lines()
    else:
        raise StandardError('Not yet supported this |%s| line-sampling type' %
                            line_type)
    if verbose:
            print "Sampling of lines finished."
    return

class SpanningTreeExperiment:
    '''
    stores all necessary information, data and options to preprocess point or
    line set before running a solver and takes care of time measuring
    '''
    def __init__(self, solver_type, graph, lines_type, has_plot, verbose):
        assert solver_type != 'all'
        self.graph = graph
        self.lines_type = lines_type
        generate_lines(graph, lines_type, verbose)
        assert list(graph.lines)
        self.solver_type = solver_type
        self.solver = get_solver(solver_type)
        self.has_plot = has_plot
        self.verbose = verbose

        self.elapsed_time = None
        self.crossing_number = None
        self.crossings = None
        self.iterations = None

    def get_name(self):
        return "%s_%s_%s" % (self.solver_type, self.graph.get_name(), self.lines_type)

    def update_point_set(self, d, n, distribution_type, lines_type):
        '''
        set a new point set to distribution_type and updates also the line set
        '''
        graph = generate_graph(d, n, distribution_type)
        generate_lines(graph, lines_type, self.verbose)
        self.graph = graph

    def update_line_set(self, lines_type):
        '''
        set a line set to new lines_type
        '''
        self.lines_type = lines_type
        generate_lines(self.graph, lines_type, self.verbose)
        return

    def update_solver(self, solver_type):
        '''
        exchange a solver in the experiment with a new one
        '''
        assert solver_type != 'all'
        self.solver_type = solver_type
        self.solver = get_solver(solver_type)
        if self.verbose:
            print "Changed solver to %s..." % solver_type

    def process(self):
        self.run()
        self.print_results()
        self.plot()

    def run(self):
        '''
        compute a spanning tree and takes care of
        caching of points, lines, time measuring and storing the crossing
        number
        '''
        if self.verbose:
            print "Start now computing a spanning tree..."
        # have fresh graph for each new computation (overriding solution and edges)
        self.graph = self.graph.copy_graph()
        start = time.time()
        self.iterations = self.solver(self.graph)
        end = time.time()
        if self.verbose:
            print "Computing a spanning tree finished."
        self.elapsed_time = end - start
        (crossing_number, crossings) = self.graph.crossing_tuple()
        self.crossing_number = crossing_number
        self.crossings = crossings

    def print_results(self):
        '''
        print all important statistics to STDOUT: crossing number, elapsed
        CPU time, iterations. If verbose flag followed by:
         - number of points
         - number of lines,
         - sum of all crossings and
         - average crossing number
        '''
        print "CPU time (in sec) %s" % self.elapsed_time
        print "crossing number=%s" % self.crossing_number
        print "iterations=%s" % self.iterations
        if self.verbose:
            print "number of points=%s" % self.graph.n
            no_lines = len(self.graph.lines)
            print "number of lines=%s" % no_lines
            print "all crossings=%s" % self.crossings
            avg_crossing_number = float(self.crossings) / no_lines
            print "average crossing number=%s" % avg_crossing_number

    def plot(self):
        '''
        if plot option is set, plot
        '''
        if self.has_plot:
            if self.verbose:
                print "Start now plotting..."
            plotting.plot(self.graph)
            if self.verbose:
                print "Closed plot."
        return

    def results(self):
        '''
        returns all statistics about the experiment and its results as a tuple:
        (
        # of points,
        # of lines,
        CPU time in seconds,
        iterations,
        crossing number,
        average crossing number,
        crossings (overall)
        )
        '''
        no_lines = len(self.graph.lines)
        results = (self.graph.n, no_lines,
                self.elapsed_time, self.iterations, self.crossing_number,
                (float(self.crossings) / no_lines), self.crossings)
        return results

    def results_csv(self):
        '''
        same as results but as one string separarted by ;
        '''
        res = self.results()
        return ";".join(res)

class AllSolversExperiment:
    def __init__(self, experiment):
        self.experiment = experiment

    def __iter__(self):
        solvers = SOLVER_OPTIONS[:-1]
        for current_solver in solvers:
            self.experiment.update_solver(current_solver)
            yield self.experiment

    def process(self):
        for experiment in self:
            experiment.process()

class AveragedExperiment:
    def __init__(self, rounds, experiment):
        self.experiment = experiment
        self.rounds = rounds
        self.computed_results = []

    def get_name(self):
        return "%s_averaged_%s" % (self.experiment.get_name(), self.rounds)

    def update_point_set(self, d, n, distribution_type, lines_type):
        self.computed_results = []
        self.experiment.update_point_set(d, n, distribution_type, lines_type)

    def update_line_set(self, lines_type):
        self.computed_results = []
        self.experiment.update_line_set(lines_type)

    def update_solver(self, solver_type):
        self.computed_results = []
        self.experiment.update_solver(solver_type)

    def run(self):
        for i in range(self.rounds):
            self.experiment.run()
            self.computed_results.append(self.experiment.results())

    def results(self):
        A = np.array(self.computed_results)
        res = np.mean(A, axis=0)
        return res.tolist()

    def str_results(self):
        (no_points, no_lines, cpu_time,
         iterations, crossing_no, avg_crossing_no, crossings) = self.results()
        return (str(int(no_points)), str(int(no_lines)), str(cpu_time),
                str(iterations), str(crossing_no),
                str(avg_crossing_no), str(crossings))

    def print_results(self):
        (no_points, no_lines, cpu_time,
         iterations, crossing_no, avg_crossing_no, crossings) = self.str_results()
        print "CPU time (in sec) %s" % cpu_time
        print "crossing number=%s" % crossing_no
        print "iterations=%s" % iterations
        if self.experiment.verbose:
            print "number of points=%s" % no_points
            print "number of lines=%s" % no_lines
            print "all crossings=%s" % crossings
            print "average crossing number=%s" % avg_crossing_no

    def process(self):
        self.run()
        self.print_results()
        self.experiment.plot()

if __name__ == '__main__':
    main()
<commit_msg>reading and solving external files<commit_after>#! /usr/bin/env python
''' main module to read files, start, evaluate and output different experiments
  See -h for further information
'''

import argparse
import time
import mult_weights_solver as mws
import sariel_lp_solver as slpsolv
import fekete_lp_solver as flpsolv
import opt_solver
import highdimgraph
import plotting
import numpy as np
import os

def main():
    args = parse_args()
    experiment = prepare_experiment(args)
    experiment.process()

# constants for options
SOLVER_OPTIONS = [ 'mult_weight', 'sariel_lp', 'fekete_lp', 'opt', 'all']
DATA_DISTRIBUTION_OPTIONS = ['uniform', 'grid']
LINE_OPTIONS = ['all', 'stabbing', 'random']

def parse_args():
    '''
    parse all arguments on the command line and return them
    '''
    parser = argparse.ArgumentParser(description=
            """
            run an parameterized experiment to compute
            a spanning tree with low crossing number
            """)
    parser.add_argument("-s", "--solver", default='mult_weight',
            choices=SOLVER_OPTIONS,
            help="choose an algorithm computing a feasible solution")
    parser.add_argument("-d", "--dimensions", type=int, default=2,
        help="number of dimensions of point set")
    parser.add_argument("-n", "--number", type=int, default=16,
        help="quantify how many points you want")
    parser.add_argument("-g", "--generate", default='uniform',
            choices=DATA_DISTRIBUTION_OPTIONS,
        help="how should the point set be sampled")
    parser.add_argument("-i", "--input", type=str, default=None,
        help="specify a input file")
    parser.add_argument("-l", "--linesampling", default='all',
            choices=LINE_OPTIONS,
        help="structure of the line set")
    parser.add_argument("-m", "--mean", type=int, default=1,
        help="run experiment m times and returns averaged results")
    parser.add_argument("-p", "--plot", action='store_true', default=False,
        help="plots the computed solution into a widget")
    parser.add_argument("-v", "--verbose", action='store_true',
        help="adds verbose outputs to STDOUT")
    args = parser.parse_args()
    return args

def prepare_experiment(args):
    '''
    factory for creation of experiments from arguments
    '''
    return create_experiment(args.solver, args.dimensions, args.number,
                             args.generate, args.linesampling, args.mean,
                             args.plot, args.verbose, args.input)

def create_experiment(solver_type, d, n, distribution_type, lines_type, mean,
                      has_plot, verbose, input_filename):

    if input_filename == None:
        if verbose:
            print "Start generating graph..."
        graph = generate_graph(d, n, distribution_type)
    else:
        if verbose:
            print "Reading graph from file %s..." % input_filename
        graph = graph_from_file(input_filename)
    if verbose:
        print "Graph has been created."

    if solver_type == 'all':
        experiment = SpanningTreeExperiment(SOLVER_OPTIONS[0], graph,
                                            lines_type, has_plot, verbose)
    else:
        experiment = SpanningTreeExperiment(solver_type, graph, lines_type,
                                            has_plot, verbose)

    if mean > 1:
        experiment = AveragedExperiment(mean, experiment)

    if solver_type == 'all':
        return AllSolversExperiment(experiment)
    else:
        return experiment

def generate_graph(d, n, distribution_type):
    '''
    generic sampling function for different point sets (dimensions,
    distributions)
    '''
    assert distribution_type in DATA_DISTRIBUTION_OPTIONS
    if distribution_type == 'uniform':
        return highdimgraph.create_uniform_graph(n, d)
    elif distribution_type == 'grid':
        return highdimgraph.create_grid_graph(n, d)

def graph_from_file(input_filename):
    with open(input_filename, 'r') as ifile:
        points = np.loadtxt(ifile)
        assert len(points.shape) == 2
        (n, d) = points.shape
        base = os.path.basename(input_filename)
        dataname_without_ext = os.path.splitext(base)[0]
        return highdimgraph.create_graph(points, n, d, dataname_without_ext)

def get_solver(solver_type):
    '''
    select a solver by solver_type and use return solver function later
    '''
    assert solver_type in SOLVER_OPTIONS
    if solver_type == 'mult_weight':
        return mws.compute_spanning_tree
    elif solver_type == 'sariel_lp':
        return slpsolv.compute_spanning_tree
    elif solver_type == 'fekete_lp':
        return flpsolv.compute_spanning_tree
    elif solver_type == 'opt':
        return opt_solver.compute_spanning_tree
    else:
        raise StandardError('Not yet supported this |%s| solver type' %
                solver_type)

def generate_lines(graph, line_type, verbose):
    assert line_type in LINE_OPTIONS
    if verbose:
        print "Sampling of lines started..."
    if line_type == 'all':
        graph.create_all_lines()
        graph.preprocess_lines()
    elif line_type == 'stabbing':
        graph.create_stabbing_lines()
    elif line_type == 'random':
        graph.create_random_lines()
        graph.preprocess_lines()
    else:
        raise StandardError('Not yet supported this |%s| line-sampling type' %
                            line_type)
    if verbose:
            print "Sampling of lines finished."
    return

class SpanningTreeExperiment:
    '''
    stores all necessary information, data and options to preprocess point or
    line set before running a solver and takes care of time measuring
    '''
    def __init__(self, solver_type, graph, lines_type, has_plot, verbose):
        assert solver_type != 'all'
        self.graph = graph
        self.lines_type = lines_type
        generate_lines(graph, lines_type, verbose)
        assert list(graph.lines)
        self.solver_type = solver_type
        self.solver = get_solver(solver_type)
        self.has_plot = has_plot
        self.verbose = verbose

        self.elapsed_time = None
        self.crossing_number = None
        self.crossings = None
        self.iterations = None

    def get_name(self):
        return "%s_%s_%s" % (self.solver_type, self.graph.get_name(), self.lines_type)

    def update_point_set(self, d, n, distribution_type, lines_type):
        '''
        set a new point set to distribution_type and updates also the line set
        '''
        graph = generate_graph(d, n, distribution_type)
        generate_lines(graph, lines_type, self.verbose)
        self.graph = graph

    def update_line_set(self, lines_type):
        '''
        set a line set to new lines_type
        '''
        self.lines_type = lines_type
        generate_lines(self.graph, lines_type, self.verbose)
        return

    def update_solver(self, solver_type):
        '''
        exchange a solver in the experiment with a new one
        '''
        assert solver_type != 'all'
        self.solver_type = solver_type
        self.solver = get_solver(solver_type)
        if self.verbose:
            print "Changed solver to %s..." % solver_type

    def process(self):
        self.run()
        self.print_results()
        self.plot()

    def run(self):
        '''
        compute a spanning tree and takes care of
        caching of points, lines, time measuring and storing the crossing
        number
        '''
        if self.verbose:
            print "Start now computing a spanning tree..."
        # have fresh graph for each new computation (overriding solution and edges)
        self.graph = self.graph.copy_graph()
        start = time.time()
        self.iterations = self.solver(self.graph)
        end = time.time()
        if self.verbose:
            print "Computing a spanning tree finished."
        self.elapsed_time = end - start
        (crossing_number, crossings) = self.graph.crossing_tuple()
        self.crossing_number = crossing_number
        self.crossings = crossings

    def print_results(self):
        '''
        print all important statistics to STDOUT: crossing number, elapsed
        CPU time, iterations. If verbose flag followed by:
         - number of points
         - number of lines,
         - sum of all crossings and
         - average crossing number
        '''
        print "CPU time (in sec) %s" % self.elapsed_time
        print "crossing number=%s" % self.crossing_number
        print "iterations=%s" % self.iterations
        if self.verbose:
            print "number of points=%s" % self.graph.n
            no_lines = len(self.graph.lines)
            print "number of lines=%s" % no_lines
            print "all crossings=%s" % self.crossings
            avg_crossing_number = float(self.crossings) / no_lines
            print "average crossing number=%s" % avg_crossing_number

    def plot(self):
        '''
        if plot option is set, plot
        '''
        if self.has_plot:
            if self.verbose:
                print "Start now plotting..."
            plotting.plot(self.graph)
            if self.verbose:
                print "Closed plot."
        return

    def results(self):
        '''
        returns all statistics about the experiment and its results as a tuple:
        (
        # of points,
        # of lines,
        CPU time in seconds,
        iterations,
        crossing number,
        average crossing number,
        crossings (overall)
        )
        '''
        no_lines = len(self.graph.lines)
        results = (self.graph.n, no_lines,
                self.elapsed_time, self.iterations, self.crossing_number,
                (float(self.crossings) / no_lines), self.crossings)
        return results

    def results_csv(self):
        '''
        same as results but as one string separarted by ;
        '''
        res = self.results()
        return ";".join(res)

class AllSolversExperiment:
    def __init__(self, experiment):
        self.experiment = experiment

    def __iter__(self):
        solvers = SOLVER_OPTIONS[:-1]
        for current_solver in solvers:
            self.experiment.update_solver(current_solver)
            yield self.experiment

    def process(self):
        for experiment in self:
            experiment.process()

class AveragedExperiment:
    def __init__(self, rounds, experiment):
        self.experiment = experiment
        self.rounds = rounds
        self.computed_results = []

    def get_name(self):
        return "%s_averaged_%s" % (self.experiment.get_name(), self.rounds)

    def update_point_set(self, d, n, distribution_type, lines_type):
        self.computed_results = []
        self.experiment.update_point_set(d, n, distribution_type, lines_type)

    def update_line_set(self, lines_type):
        self.computed_results = []
        self.experiment.update_line_set(lines_type)

    def update_solver(self, solver_type):
        self.computed_results = []
        self.experiment.update_solver(solver_type)

    def run(self):
        for i in range(self.rounds):
            self.experiment.run()
            self.computed_results.append(self.experiment.results())

    def results(self):
        A = np.array(self.computed_results)
        res = np.mean(A, axis=0)
        return res.tolist()

    def str_results(self):
        (no_points, no_lines, cpu_time,
         iterations, crossing_no, avg_crossing_no, crossings) = self.results()
        return (str(int(no_points)), str(int(no_lines)), str(cpu_time),
                str(iterations), str(crossing_no),
                str(avg_crossing_no), str(crossings))

    def print_results(self):
        (no_points, no_lines, cpu_time,
         iterations, crossing_no, avg_crossing_no, crossings) = self.str_results()
        print "CPU time (in sec) %s" % cpu_time
        print "crossing number=%s" % crossing_no
        print "iterations=%s" % iterations
        if self.experiment.verbose:
            print "number of points=%s" % no_points
            print "number of lines=%s" % no_lines
            print "all crossings=%s" % crossings
            print "average crossing number=%s" % avg_crossing_no

    def process(self):
        self.run()
        self.print_results()
        self.experiment.plot()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Access control helper.

The functions in this module can be used to check access control
related requirements. When the specified required conditions are not
met, an exception is raised. This exception contains a views that
either prompts for authentication, or informs the user that they
do not meet the required criteria.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  ]

from google.appengine.api import users

from django.core import urlresolvers
from django.utils.translation import ugettext_lazy

from soc.logic import accounts
from soc.logic import dicts
from soc.logic.models import host as host_logic
from soc.logic.models import notification as notification_logic
from soc.logic.models import user as user_logic
from soc.logic.models import request as request_logic
from soc.views import helper
from soc.views import out_of_band

DEF_NO_USER_LOGIN_MSG_FMT = ugettext_lazy(
  'Please create User Profile'
  ' in order to view this page.')

DEF_DEV_LOGOUT_LOGIN_MSG_FMT = ugettext_lazy(
  'Please sign out'
  ' and sign in'
  ' again as %(role)s to view this page.')

DEF_PAGE_DENIED_MSG = ugettext_lazy(
  'Access to this page has been restricted')

DEF_LOGOUT_MSG_FMT = ugettext_lazy(
    'Please sign out in order to view this page')

def checkAccess(access_type, request, rights, args=None, kwargs=None):
  """Runs all the defined checks for the specified type.

  Args:
    access_type: the type of request (such as 'list' or 'edit')
    request: the Django request object
    rights: a dictionary containing access check functions

  Rights usage:
    The rights dictionary is used to check if the current user is allowed
    to view the page specified. The functions defined in this dictionary
    are always called with the django request object as argument. On any
    request, regardless of what type, the functions in the 'any_access' value
    are called. If the specified type is not in the rights dictionary, all
    the functions in the 'unspecified' value are called. When the specified
    type _is_ in the rights dictionary, all the functions in that access_type's
    value are called.

  Returns:
    True: If all the required access checks have been made successfully
    False: If a check failed, in this case self._response will contain
      the response provided by the failed access check.
  """

  # Call each access checker
  for check in rights['any_access']:
    check(request, args, kwargs)

  if access_type not in rights:
    for check in rights['unspecified']:
      # No checks defined, so do the 'generic' checks and bail out
      check(request, args, kwargs)
    return

  for check in rights[access_type]:
    check(request, args, kwargs)

def allow(request, args, kwargs):
  """Never returns an alternate HTTP response.

  Args:
    request: a Django HTTP request
  """

  return

def deny(request, args, kwargs):
  """Returns an alternate HTTP response.

  Args:
    request: a Django HTTP request

  Returns:
    a subclass of django.http.HttpResponse which contains the
    alternate response that should be returned by the calling view.
  """

  context = {}
  context['title'] = 'Access denied'

  raise out_of_band.AccessViolation(DEF_PAGE_DENIED_MSG, context=context)

def checkIsLoggedIn(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account is not logged in.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the user is logged in, or a subclass of
    django.http.HttpResponse which contains the alternate response
    that should be returned by the calling view.
  """

  if users.get_current_user():
    return

  raise out_of_band.LoginRequest()

def checkNotLoggedIn(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account is not logged in.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the user is logged in, or a subclass of
    django.http.HttpResponse which contains the alternate response
    that should be returned by the calling view.
  """

  if not users.get_current_user():
    return

  raise out_of_band.LoginRequest(message_fmt=DEF_LOGOUT_MSG_FMT)

def checkIsUser(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no User entity.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if User exists for a Google Account, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  checkIsLoggedIn(request, args, kwargs)

  user = user_logic.logic.getForFields(
      {'account': users.get_current_user()}, unique=True)

  if user:
    return

  raise out_of_band.LoginRequest(message_fmt=DEF_NO_USER_LOGIN_MSG_FMT)

def checkIsDeveloper(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account is not a Developer.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Google Account is logged in and logged-in user is a Developer,
    or a subclass of django.http.HttpResponse which contains the alternate
    response should be returned by the calling view.
  """

  checkIsUser(request, args, kwargs)

  if accounts.isDeveloper(account=users.get_current_user()):
    return

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Site Developer '}

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsHost(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no Host entity
     for the specified program.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Host exists for the specified program, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  try:
    # if the current user is invited to create a host profile we allow access
    checkIsInvited(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  user = user_logic.logic.getForFields(
      {'account': users.get_current_user()}, unique=True)

  host = host_logic.logic.getForFields(
      {'user': user}, unique=True)

  if host:
    return

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Program Administrator '}

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsClubAdminForClub(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no Club Admin
     entity for the specified club.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Club Admin exists for the specified club, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  try:
    # if the current user is invited to create a host profile we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  # TODO(srabbelier) implement this

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Club Admin for this Club'}

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsInvited(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no Host entity
     for the specified program.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Host exists for the specified program, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  try:
    # if the current user is a developer we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Program Administrator for this Program'}

  splitpath = request.path.split('/')
  splitpath = splitpath[1:] # cut off leading ''

  if len(splitpath) < 4:
    # TODO: perhaps this needs a better explanation?
    deny(request, args, kwargs)

  role = splitpath[0]
  group_id = splitpath[2]
  user_id = splitpath[3]

  user = user_logic.logic.getForFields(
      {'account': users.get_current_user()}, unique=True)

  if user_id != user.link_id:
    # TODO: perhaps this needs a better explanation?
    deny(request, args, kwargs)

  properties = {
      'link_id': user_id,
      'role': role,
      'scope_path': group_id,
      'group_accepted': True,
      }

  request = request_logic.logic.getForFields(properties, unique=True)

  if request:
    return

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsApplicationAccepted(app_logic):
  """Returns an alternate HTTP response if Google Account has no Club App
     entity for the specified Club.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Club App  exists for the specified program, or a subclass
    of django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  def wrapper(request, args, kwargs):
    try:
      # if the current user is a developer we allow access
      checkIsDeveloper(request, args, kwargs)
      return
    except out_of_band.Error:
      pass

    checkIsUser(request, args, kwargs)

    user = user_logic.logic.getForCurrentAccount()

    properties = {
        'applicant': user,
        'reviewed': True,
        'accepted': True,
        'application_completed': False,
        }

    application = app_logic.logic.getForFields(properties, unique=True)

    if application:
      return

    # TODO(srabbelier) Make this give a proper error message
    deny(request, args, kwargs)

  return wrapper

def checkIsMyNotification(request, args, kwargs):
  """Returns an alternate HTTP response if this request is for a Notification belonging
     to the current user.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the current User is allowed to access this Notification.
  """

  try:
    # if the current user is a developer we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  # Mine the url for params
  try:
    callback, args, kwargs = urlresolvers.resolve(request.path)
  except Exception:
    deny(request, args, kwargs)

  properties = dicts.filter(kwargs, ['link_id', 'scope_path'])

  notification = notification_logic.logic.getForFields(properties, unique=True)
  user = user_logic.logic.getForCurrentAccount()

  # We need to check to see if the key's are equal since the User
  # objects are different and the default __eq__ method does not check
  # if the keys are equal (which is what we want).
  if user.key() == notification.scope.key():
    return None

  # TODO(ljvderijk) Make this give a proper error message
  deny(request, args, kwargs)

def checkIsMyApplication(app_logic):
  """Returns an alternate HTTP response if this request is for a Application belonging
     to the current user.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the current User is allowed to access this Application.
  """

  def wrapper(request, args, kwargs):
    try:
      # if the current user is a developer we allow access
      checkIsDeveloper(request, args, kwargs)
      return
    except out_of_band.Error:
      pass

    checkIsUser(request, args, kwargs)

    properties = dicts.filter(kwargs, ['link_id'])

    application = app_logic.logic.getForFields(properties, unique=True)

    if not application:
      deny(request, args, kwargs)

    user = user_logic.logic.getForCurrentAccount()

    # We need to check to see if the key's are equal since the User
    # objects are different and the default __eq__ method does not check
    # if the keys are equal (which is what we want).
    if user.key() == application.applicant.key():
      return None

    # TODO(srabbelier) Make this give a proper error message
    deny(request, args, kwargs)

  return wrapper

def checkCanInvite(request, args, kwargs):
  """Checks to see if the current user can create an invite.

  Note that if the current url is not in the default 'request' form
  this method either deny()s or performs the wrong access check.

  Args:
    request: a Django HTTP request
  """

  try:
    # if the current user is a developer we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  # Mine the url for params
  try:
    callback, args, kwargs = urlresolvers.resolve(request.path)
  except Exception:
    deny(request, args, kwargs)

  # Construct a new url by reshufling the kwargs
  order = ['role', 'access_type', 'scope_path', 'link_id']
  url_params = dicts.unzip(kwargs, order)
  url = '/'.join([''] + list(url_params))

  # Mine the reshufled url
  try:
    callback, args, kwargs = urlresolvers.resolve(url)
  except Exception:
    deny(request, args, kwargs)

  # Get the everything we need for the access check
  params = callback.im_self.getParams()
  access_type = kwargs['access_type']

  # Perform the access check
  checkAccess(access_type, request, rights=params['rights'])

def checkIsDocumentPublic(request, args, kwargs):
  """Checks whether a document is public.

  Args:
    request: a Django HTTP request
  """

  # TODO(srabbelier): A proper check needs to be done to see if the document
  # is public or not, probably involving analysing it's scope or such.
  allow(request, args, kwargs)
<commit_msg>Take advantage of new "from ... import ... as ..." rules in the style guide.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Access control helper.

The functions in this module can be used to check access control
related requirements. When the specified required conditions are not
met, an exception is raised. This exception contains a views that
either prompts for authentication, or informs the user that they
do not meet the required criteria.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  ]

from google.appengine.api import users

from django.core import urlresolvers
from django.utils.translation import ugettext_lazy

from soc.logic import accounts
from soc.logic import dicts
from soc.logic.models.host import logic as host_logic
from soc.logic.models.notification import logic as notification_logic
from soc.logic.models.request import logic as request_logic
from soc.logic.models.user import logic as user_logic
from soc.views import helper
from soc.views import out_of_band

DEF_NO_USER_LOGIN_MSG_FMT = ugettext_lazy(
  'Please create User Profile'
  ' in order to view this page.')

DEF_DEV_LOGOUT_LOGIN_MSG_FMT = ugettext_lazy(
  'Please sign out'
  ' and sign in'
  ' again as %(role)s to view this page.')

DEF_PAGE_DENIED_MSG = ugettext_lazy(
  'Access to this page has been restricted')

DEF_LOGOUT_MSG_FMT = ugettext_lazy(
    'Please sign out in order to view this page')

def checkAccess(access_type, request, rights, args=None, kwargs=None):
  """Runs all the defined checks for the specified type.

  Args:
    access_type: the type of request (such as 'list' or 'edit')
    request: the Django request object
    rights: a dictionary containing access check functions

  Rights usage:
    The rights dictionary is used to check if the current user is allowed
    to view the page specified. The functions defined in this dictionary
    are always called with the django request object as argument. On any
    request, regardless of what type, the functions in the 'any_access' value
    are called. If the specified type is not in the rights dictionary, all
    the functions in the 'unspecified' value are called. When the specified
    type _is_ in the rights dictionary, all the functions in that access_type's
    value are called.

  Returns:
    True: If all the required access checks have been made successfully
    False: If a check failed, in this case self._response will contain
      the response provided by the failed access check.
  """

  # Call each access checker
  for check in rights['any_access']:
    check(request, args, kwargs)

  if access_type not in rights:
    for check in rights['unspecified']:
      # No checks defined, so do the 'generic' checks and bail out
      check(request, args, kwargs)
    return

  for check in rights[access_type]:
    check(request, args, kwargs)

def allow(request, args, kwargs):
  """Never returns an alternate HTTP response.

  Args:
    request: a Django HTTP request
  """

  return

def deny(request, args, kwargs):
  """Returns an alternate HTTP response.

  Args:
    request: a Django HTTP request

  Returns:
    a subclass of django.http.HttpResponse which contains the
    alternate response that should be returned by the calling view.
  """

  context = {}
  context['title'] = 'Access denied'

  raise out_of_band.AccessViolation(DEF_PAGE_DENIED_MSG, context=context)

def checkIsLoggedIn(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account is not logged in.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the user is logged in, or a subclass of
    django.http.HttpResponse which contains the alternate response
    that should be returned by the calling view.
  """

  if users.get_current_user():
    return

  raise out_of_band.LoginRequest()

def checkNotLoggedIn(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account is not logged in.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the user is logged in, or a subclass of
    django.http.HttpResponse which contains the alternate response
    that should be returned by the calling view.
  """

  if not users.get_current_user():
    return

  raise out_of_band.LoginRequest(message_fmt=DEF_LOGOUT_MSG_FMT)

def checkIsUser(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no User entity.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if User exists for a Google Account, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  checkIsLoggedIn(request, args, kwargs)

  user = user_logic.getForFields({'account': users.get_current_user()},
                                 unique=True)

  if user:
    return

  raise out_of_band.LoginRequest(message_fmt=DEF_NO_USER_LOGIN_MSG_FMT)

def checkIsDeveloper(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account is not a Developer.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Google Account is logged in and logged-in user is a Developer,
    or a subclass of django.http.HttpResponse which contains the alternate
    response should be returned by the calling view.
  """

  checkIsUser(request, args, kwargs)

  if accounts.isDeveloper(account=users.get_current_user()):
    return

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Site Developer '}

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsHost(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no Host entity
     for the specified program.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Host exists for the specified program, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  try:
    # if the current user is invited to create a host profile we allow access
    checkIsInvited(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  user = user_logic.getForFields({'account': users.get_current_user()},
                                 unique=True)

  host = host_logic.getForFields({'user': user}, unique=True)

  if host:
    return

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Program Administrator '}

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsClubAdminForClub(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no Club Admin
     entity for the specified club.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Club Admin exists for the specified club, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  try:
    # if the current user is invited to create a host profile we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  # TODO(srabbelier) implement this

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Club Admin for this Club'}

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsInvited(request, args, kwargs):
  """Returns an alternate HTTP response if Google Account has no Host entity
     for the specified program.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Host exists for the specified program, or a subclass of
    django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  try:
    # if the current user is a developer we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  login_message_fmt = DEF_DEV_LOGOUT_LOGIN_MSG_FMT % {
      'role': 'a Program Administrator for this Program'}

  splitpath = request.path.split('/')
  splitpath = splitpath[1:] # cut off leading ''

  if len(splitpath) < 4:
    # TODO: perhaps this needs a better explanation?
    deny(request, args, kwargs)

  role = splitpath[0]
  group_id = splitpath[2]
  user_id = splitpath[3]

  user = user_logic.getForFields({'account': users.get_current_user()},
                                 unique=True)

  if user_id != user.link_id:
    # TODO: perhaps this needs a better explanation?
    deny(request, args, kwargs)

  properties = {
      'link_id': user_id,
      'role': role,
      'scope_path': group_id,
      'group_accepted': True,
      }

  request = request_logic.getForFields(properties, unique=True)

  if request:
    return

  raise out_of_band.LoginRequest(message_fmt=login_message_fmt)

def checkIsApplicationAccepted(app_logic):
  """Returns an alternate HTTP response if Google Account has no Club App
     entity for the specified Club.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if Club App  exists for the specified program, or a subclass
    of django.http.HttpResponse which contains the alternate response
    should be returned by the calling view.
  """

  def wrapper(request, args, kwargs):
    try:
      # if the current user is a developer we allow access
      checkIsDeveloper(request, args, kwargs)
      return
    except out_of_band.Error:
      pass

    checkIsUser(request, args, kwargs)

    user = user_logic.getForCurrentAccount()

    properties = {
        'applicant': user,
        'reviewed': True,
        'accepted': True,
        'application_completed': False,
        }

    application = app_logic.logic.getForFields(properties, unique=True)

    if application:
      return

    # TODO(srabbelier) Make this give a proper error message
    deny(request, args, kwargs)

  return wrapper

def checkIsMyNotification(request, args, kwargs):
  """Returns an alternate HTTP response if this request is for a Notification belonging
     to the current user.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the current User is allowed to access this Notification.
  """

  try:
    # if the current user is a developer we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  checkIsUser(request, args, kwargs)

  # Mine the url for params
  try:
    callback, args, kwargs = urlresolvers.resolve(request.path)
  except Exception:
    deny(request, args, kwargs)

  properties = dicts.filter(kwargs, ['link_id', 'scope_path'])

  notification = notification_logic.getForFields(properties, unique=True)
  user = user_logic.getForCurrentAccount()

  # We need to check to see if the key's are equal since the User
  # objects are different and the default __eq__ method does not check
  # if the keys are equal (which is what we want).
  if user.key() == notification.scope.key():
    return None

  # TODO(ljvderijk) Make this give a proper error message
  deny(request, args, kwargs)

def checkIsMyApplication(app_logic):
  """Returns an alternate HTTP response if this request is for a Application belonging
     to the current user.

  Args:
    request: a Django HTTP request

   Raises:
     AccessViolationResponse: if the required authorization is not met

  Returns:
    None if the current User is allowed to access this Application.
  """

  def wrapper(request, args, kwargs):
    try:
      # if the current user is a developer we allow access
      checkIsDeveloper(request, args, kwargs)
      return
    except out_of_band.Error:
      pass

    checkIsUser(request, args, kwargs)

    properties = dicts.filter(kwargs, ['link_id'])

    application = app_logic.logic.getForFields(properties, unique=True)

    if not application:
      deny(request, args, kwargs)

    user = user_logic.getForCurrentAccount()

    # We need to check to see if the key's are equal since the User
    # objects are different and the default __eq__ method does not check
    # if the keys are equal (which is what we want).
    if user.key() == application.applicant.key():
      return None

    # TODO(srabbelier) Make this give a proper error message
    deny(request, args, kwargs)

  return wrapper

def checkCanInvite(request, args, kwargs):
  """Checks to see if the current user can create an invite.

  Note that if the current url is not in the default 'request' form
  this method either deny()s or performs the wrong access check.

  Args:
    request: a Django HTTP request
  """

  try:
    # if the current user is a developer we allow access
    checkIsDeveloper(request, args, kwargs)
    return
  except out_of_band.Error:
    pass

  # Mine the url for params
  try:
    callback, args, kwargs = urlresolvers.resolve(request.path)
  except Exception:
    deny(request, args, kwargs)

  # Construct a new url by reshufling the kwargs
  order = ['role', 'access_type', 'scope_path', 'link_id']
  url_params = dicts.unzip(kwargs, order)
  url = '/'.join([''] + list(url_params))

  # Mine the reshufled url
  try:
    callback, args, kwargs = urlresolvers.resolve(url)
  except Exception:
    deny(request, args, kwargs)

  # Get the everything we need for the access check
  params = callback.im_self.getParams()
  access_type = kwargs['access_type']

  # Perform the access check
  checkAccess(access_type, request, rights=params['rights'])

def checkIsDocumentPublic(request, args, kwargs):
  """Checks whether a document is public.

  Args:
    request: a Django HTTP request
  """

  # TODO(srabbelier): A proper check needs to be done to see if the document
  # is public or not, probably involving analysing it's scope or such.
  allow(request, args, kwargs)
<|endoftext|>


<commit_before>import hashlib
import hmac
import importlib
import pkgutil
import random
import sys
import threading
from asyncio import coroutine, iscoroutinefunction, current_task, get_event_loop

SALT_CHARS = 'bcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'

class MuffinException(Exception):

    """ Implement a Muffin's exception. """

    pass

def to_coroutine(func):
    """ Ensure that the function is coroutine.

    If not convert to coroutine.

    """
    if not iscoroutinefunction(func):
        func = coroutine(func)
    return func

def create_signature(secret, value, digestmod='sha256', encoding='utf-8'):
    """ Create HMAC Signature from secret for value. """
    if isinstance(secret, str):
        secret = secret.encode(encoding)

    if isinstance(value, str):
        value = value.encode(encoding)

    if isinstance(digestmod, str):
        digestmod = getattr(hashlib, digestmod, hashlib.sha1)

    hm = hmac.new(secret, digestmod=digestmod)
    hm.update(value)
    return hm.hexdigest()

def check_signature(signature, *args, **kwargs):
    """ Check for the signature is correct. """
    return hmac.compare_digest(signature, create_signature(*args, **kwargs))

def generate_password_hash(password, digestmod='sha256', salt_length=8):
    """ Hash a password with given method and salt length. """

    salt = ''.join(random.sample(SALT_CHARS, salt_length))
    signature = create_signature(salt, password, digestmod=digestmod)
    return '$'.join((digestmod, salt, signature))

def check_password_hash(password, pwhash):
    if pwhash.count('$') < 2:
        return False
    digestmod, salt, signature = pwhash.split('$', 2)
    return check_signature(signature, salt, password, digestmod=digestmod)

class Struct(dict):

    """ `Attribute` dictionary. Use attributes as keys. """

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError("Attribute '%s' doesn't exists. " % name)

    def __setattr__(self, name, value):
        self[name] = value

class LStruct(Struct):

    """ Frosen structure. Used as application/plugins settings.

    Going to be immutable after application is started.

    """

    def __init__(self, *args, **kwargs):
        object.__setattr__(self, 'frozen', False)
        super().__init__(*args, **kwargs)

    def freeze(self):
        object.__setattr__(self, 'frozen', True)

    def __setitem__(self, name, value):
        if self.frozen:
            raise RuntimeError('`%s` is frozen.' % type(self))
        super().__setitem__(name, value)

class local_storage:
    pass

class local:

    """ coroutine.local storage is simular to python's threading.local.

    Usage: ::

        local = muffin.local(loop)
        local.value = 42

    """

    __slots__ = '_loop',
    __loops__ = {}

    def __new__(cls, loop=None):
        """ The local is singleton per loop. """
        key = "%s%s" % (cls, id(loop))
        if key not in cls.__loops__:
            cls.__loops__[key] = object.__new__(cls)
        return cls.__loops__[key]

    def __init__(self, loop=None):
        """ Bind loop to self. """
        object.__setattr__(self, '_loop', loop or get_event_loop())

    def __getattribute__(self, name):
        """ Get attribute from current task's space. """
        if name in ('__setattr__', '__getattr__', '__delattr__', '_loop', '__curtask__'):
            return object.__getattribute__(self, name)
        return getattr(self.__curtask__, name)

    def __setattr__(self, name, value):
        """ Set attribute to current task's space. """
        self.__curtask__.__dict__[name] = value

    def __delattr__(self, name):
        delattr(self.__curtask__, name)

    @property
    def __curtask__(self):
        """ Create namespace in current task. """
        task = current_task(loop=self._loop)
        if not task:
            raise RuntimeError('No task is currently running')

        if not hasattr(task, '_locals'):
            task._locals = local_storage()
        return task._locals

tlocals = threading.local()

class slocal(local):

    """ Fail silently to threading.local if curent loop is not running. """

    __loops__ = {}

    def __getattribute__(self, name):
        """ Get attribute from current task's space. """
        if name in ('__setattr__', '__getattr__', '__delattr__', '_loop', '__curtask__'):
            return object.__getattribute__(self, name)

        if self._loop.is_running():
            return getattr(self.__curtask__, name)
        return getattr(tlocals, name)

    def __setattr__(self, name, value):
        """ Set attribute to current task's space. """
        if self._loop.is_running():
            super().__setattr__(name, value)
        else:
            setattr(tlocals, name, value)

def import_submodules(package_name, *submodules):
    """Import all submodules by package name."""
    package = sys.modules[package_name]
    return {
        name: importlib.import_module(package_name + '.' + name)
        for _, name, _ in pkgutil.walk_packages(package.__path__)
        if not submodules or name in submodules
    }

#  pylama:ignore=W0212
<commit_msg>fix: compatibility with python<3.7<commit_after>import hashlib
import hmac
import importlib
import pkgutil
import random
import sys
import threading

from asyncio import coroutine, iscoroutinefunction, get_event_loop

try:
    # Python 3.7
    from asyncio import current_task
except ImportError:
    current_task = Task.current_task

SALT_CHARS = 'bcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'

class MuffinException(Exception):

    """ Implement a Muffin's exception. """

    pass

def to_coroutine(func):
    """ Ensure that the function is coroutine.

    If not convert to coroutine.

    """
    if not iscoroutinefunction(func):
        func = coroutine(func)
    return func

def create_signature(secret, value, digestmod='sha256', encoding='utf-8'):
    """ Create HMAC Signature from secret for value. """
    if isinstance(secret, str):
        secret = secret.encode(encoding)

    if isinstance(value, str):
        value = value.encode(encoding)

    if isinstance(digestmod, str):
        digestmod = getattr(hashlib, digestmod, hashlib.sha1)

    hm = hmac.new(secret, digestmod=digestmod)
    hm.update(value)
    return hm.hexdigest()

def check_signature(signature, *args, **kwargs):
    """ Check for the signature is correct. """
    return hmac.compare_digest(signature, create_signature(*args, **kwargs))

def generate_password_hash(password, digestmod='sha256', salt_length=8):
    """ Hash a password with given method and salt length. """

    salt = ''.join(random.sample(SALT_CHARS, salt_length))
    signature = create_signature(salt, password, digestmod=digestmod)
    return '$'.join((digestmod, salt, signature))

def check_password_hash(password, pwhash):
    if pwhash.count('$') < 2:
        return False
    digestmod, salt, signature = pwhash.split('$', 2)
    return check_signature(signature, salt, password, digestmod=digestmod)

class Struct(dict):

    """ `Attribute` dictionary. Use attributes as keys. """

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError("Attribute '%s' doesn't exists. " % name)

    def __setattr__(self, name, value):
        self[name] = value

class LStruct(Struct):

    """ Frosen structure. Used as application/plugins settings.

    Going to be immutable after application is started.

    """

    def __init__(self, *args, **kwargs):
        object.__setattr__(self, 'frozen', False)
        super().__init__(*args, **kwargs)

    def freeze(self):
        object.__setattr__(self, 'frozen', True)

    def __setitem__(self, name, value):
        if self.frozen:
            raise RuntimeError('`%s` is frozen.' % type(self))
        super().__setitem__(name, value)

class local_storage:
    pass

class local:

    """ coroutine.local storage is simular to python's threading.local.

    Usage: ::

        local = muffin.local(loop)
        local.value = 42

    """

    __slots__ = '_loop',
    __loops__ = {}

    def __new__(cls, loop=None):
        """ The local is singleton per loop. """
        key = "%s%s" % (cls, id(loop))
        if key not in cls.__loops__:
            cls.__loops__[key] = object.__new__(cls)
        return cls.__loops__[key]

    def __init__(self, loop=None):
        """ Bind loop to self. """
        object.__setattr__(self, '_loop', loop or get_event_loop())

    def __getattribute__(self, name):
        """ Get attribute from current task's space. """
        if name in ('__setattr__', '__getattr__', '__delattr__', '_loop', '__curtask__'):
            return object.__getattribute__(self, name)
        return getattr(self.__curtask__, name)

    def __setattr__(self, name, value):
        """ Set attribute to current task's space. """
        self.__curtask__.__dict__[name] = value

    def __delattr__(self, name):
        delattr(self.__curtask__, name)

    @property
    def __curtask__(self):
        """ Create namespace in current task. """
        task = current_task(loop=self._loop)
        if not task:
            raise RuntimeError('No task is currently running')

        if not hasattr(task, '_locals'):
            task._locals = local_storage()
        return task._locals

tlocals = threading.local()

class slocal(local):

    """ Fail silently to threading.local if curent loop is not running. """

    __loops__ = {}

    def __getattribute__(self, name):
        """ Get attribute from current task's space. """
        if name in ('__setattr__', '__getattr__', '__delattr__', '_loop', '__curtask__'):
            return object.__getattribute__(self, name)

        if self._loop.is_running():
            return getattr(self.__curtask__, name)
        return getattr(tlocals, name)

    def __setattr__(self, name, value):
        """ Set attribute to current task's space. """
        if self._loop.is_running():
            super().__setattr__(name, value)
        else:
            setattr(tlocals, name, value)

def import_submodules(package_name, *submodules):
    """Import all submodules by package name."""
    package = sys.modules[package_name]
    return {
        name: importlib.import_module(package_name + '.' + name)
        for _, name, _ in pkgutil.walk_packages(package.__path__)
        if not submodules or name in submodules
    }

#  pylama:ignore=W0212
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# author: Feiyi Wang
#
# This program provides a MPI-based parallization for checksumming
#
#
from __future__ import print_function
import os
import hashlib
import argparse
import stat
import sys
import signal
import cPickle as pickle
import shutil
from mpi4py import MPI
from cStringIO import StringIO

from circle import Circle
from _version import get_versions
from task import BaseTask
from utils import bytes_fmt, timestamp2, conv_unit
from fwalk import FWalk
from cio import readn
from fdef import ChunkSum
from globals import G
import utils
from pcircle.mpihelper import tally_hosts, parse_and_bcast, ThrowingArgumentParser
from bfsignature import BFsignature

__version__ = get_versions()['version']
args = None
comm = MPI.COMM_WORLD

def sig_handler(signum, sigstack):
    print("\tUser cancelled ... cleaning up")
    sys.exit(1)

def err_and_exit(msg, code=0):
    if comm.rank == 0:
        print("\n%s" % msg)
    MPI.Finalize()
    sys.exit(0)

def gen_parser():
    parser = ThrowingArgumentParser(description="fsum")
    parser.add_argument("-v", "--version", action="version", version="{version}".format(version=__version__))
    parser.add_argument("--loglevel", default="error", help="log level, default: ERROR")
    parser.add_argument("path", nargs='+', default=".", help="path")
    parser.add_argument("-i", "--interval", type=int, default=10, help="interval")
    parser.add_argument("-o", "--output", default="sha1-%s.sig" % timestamp2(), help="sha1 output file")
    parser.add_argument("--chunksize", help="chunk size (K, M, G, T)")
    #parser.add_argument("--use-store", action="store_true", help="Use persistent store")
    #parser.add_argument("--export-block-signatures", action="store_true", help="export block-level signatures")

    return parser

class Checksum(BaseTask):
    def __init__(self, circle, treewalk, chunksize, totalsize=0, totalfiles=0):
        BaseTask.__init__(self, circle)
        self.circle = circle
        self.treewalk = treewalk
        self.totalsize = totalsize
        self.totalfiles = totalfiles
        self.total_chunks = 0
        self.workcnt = 0
        #self.chunkq = []
        self.chunksize = chunksize

        # debug
        self.d = {"rank": "rank %s" % circle.rank}
        self.wtime_started = MPI.Wtime()
        self.wtime_ended = None

        # reduce
        self.vsize = 0

        self.logger = utils.getLogger(__name__)

        if self.circle.rank == 0:
            print("Start parallel checksumming ...")

    def create(self):

        for fi in self.treewalk.flist:
            if not os.path.islink(fi.path) and stat.S_ISREG(fi.st_mode):
                self.enq_file(fi)

        if len(self.treewalk.flist_buf) > 0:
           for fi in self.treewalk.flist_buf:
               if not os.path.islink(fi.path) and stat.S_ISREG(fi.st_mode):
                   self.enq_file(fi)

                    # right after this, we do first checkpoint

                    # if self.checkpoint_file:
                    #    self.do_no_interrupt_checkpoint()
                    #    self.checkpoint_last = MPI.Wtime()

        if self.treewalk.use_store:
            while self.treewalk.flist_db.qsize > 0:
                fitems, _ = self.treewalk.flist_db.mget(G.DB_BUFSIZE)
                for fi in fitems:
                    if stat.S_ISREG(fi.st_mode):
                        self.enq_file(fi)  # where chunking takes place
                self.treewalk.flist_db.mdel(G.DB_BUFSIZE)

        # gather total chunks
        self.circle.comm.barrier()
        self.total_chunks = self.circle.comm.reduce(self.workcnt, op=MPI.SUM)
        self.total_chunks = self.circle.comm.bcast(self.total_chunks)
        #print("total chunks = ", self.total_chunks)
        self.bfsign = BFsignature(self.total_chunks)

    def enq_file(self, f):
        """
        f[0] path f[1] mode f[2] size - we enq all in one shot
        CMD = copy src  dest  off_start  last_chunk
        """
        chunks = f.st_size / self.chunksize
        remaining = f.st_size % self.chunksize

        workcnt = 0

        if f.st_size == 0:  # empty file
            ck = ChunkSum(f.path)
            self.enq(ck)
            self.logger.debug("%s" % ck, extra=self.d)
            workcnt += 1
        else:
            for i in range(chunks):
                ck = ChunkSum(f.path)
                ck.offset = i * self.chunksize
                ck.length = self.chunksize
                self.enq(ck)
                self.logger.debug("%s" % ck, extra=self.d)
            workcnt += chunks

        if remaining > 0:
            # send remainder
            ck = ChunkSum(f.path)
            ck.offset = chunks * self.chunksize
            ck.length = remaining
            self.enq(ck)
            self.logger.debug("%s" % ck, extra=self.d)
            workcnt += 1

        # tally work cnt
        self.workcnt += workcnt

    # def process(self):
    #     ck = self.deq()
    #     self.logger.debug("process: %s" % ck, extra=self.d)
    #     blocks = ck.length / self.chunksize
    #     remaining = ck.length % self.chunksize
    #
    #     chunk_digests = StringIO()
    #     try:
    #         fd = os.open(ck.filename, os.O_RDONLY)
    #     except OSError as e:
    #         self.logger.warn("%s, Skipping ... " % e, extra=self.d)
    #         return
    #
    #     os.lseek(fd, ck.offset, os.SEEK_SET)
    #
    #     for i in range(blocks):
    #         chunk_digests.write(hashlib.sha1(readn(fd, self.chunksize)).hexdigest())
    #
    #     if remaining > 0:
    #         chunk_digests.write(hashlib.sha1(readn(fd, remaining)).hexdigest())
    #
    #     ck.digest = chunk_digests.getvalue()
    #
    #     self.chunkq.append(ck)
    #
    #     self.vsize += ck.length
    #     try:
    #         os.close(fd)
    #     except Exception as e:
    #         self.logger.warn(e, extra=self.d)

    def process(self):
        ck = self.deq()
        try:
            fd = os.open(ck.filename, os.O_RDONLY)
        except OSError as e:
            self.logger.warn("%s, Skipping ... " % e, extra=self.d)
            return

        os.lseek(fd, ck.offset, os.SEEK_SET)
        digest = hashlib.sha1()
        blocksize = 4*1024*1024 # 4MiB block
        blockcount = ck.length / blocksize
        remaining = ck.length % blocksize
        for _ in xrange(blockcount):
            digest.update(readn(fd, blocksize))
        if remaining > 0:
            digest.update(readn(fd, remaining))
        try:
            os.close(fd)
        except Exception as e:
            self.logger.warn(e, extra=self.d)
        ck.digest = digest.hexdigest()
        #self.chunkq.append(ck)
        self.vsize += ck.length

        self.bfsign.insert_item(ck.digest)

    def reduce_init(self, buf):
        buf['vsize'] = self.vsize

    def reduce_report(self, buf):
        out = ""
        if self.totalsize != 0:
            out += "%.2f %% block checksummed, " % (100 * float(buf['vsize']) / self.totalsize)

        out += "%s bytes done" % bytes_fmt(buf['vsize'])
        print(out)

    def reduce_finish(self, buf):
        # self.reduce_report(buf)
        pass

    def reduce(self, buf1, buf2):
        buf1['vsize'] += buf2['vsize']
        return buf1

    def epilogue(self):
        self.wtime_ended = MPI.Wtime()
        if self.circle.rank == 0:
            print("")
            if self.totalsize == 0:
                return
            time = self.wtime_ended - self.wtime_started
            rate = float(self.totalsize) / time
            print("Checksumming Completed In: %.2f seconds" % time)
            print("Average Rate: %s/s\n" % bytes_fmt(rate))

def _read_in_blocks(chunks, chunksize=26214):
    """ deprecated, do_checksum() return signature directly"""
    idx = 0
    total = len(chunks)
    buf = StringIO()

    while True:
        if idx == total:
            break
        buf.write(chunks[idx].digest)
        idx += 1
        if idx % chunksize == 0 or idx == total:
            yield hashlib.sha1(buf.getvalue()).hexdigest()
            buf = StringIO()

def do_checksum(chunks):
    buf = StringIO()
    h = hashlib.sha1()
    for chunk in chunks:
        buf.write(chunk.digest)
        # idx += 1
        # if idx % blocks == 0 or idx == len(chunks):
        #     h.update(buf.getvalue())
        #     buf = StringIO()
    h.update(buf.getvalue())
    return h.hexdigest()

def export_checksum(chunks):
    fullpath = os.path.abspath(args.output)
    ex_base = os.path.basename(fullpath).split(".")[0] + ".checksums"
    ex_dir = os.path.dirname(fullpath)
    ex_path = os.path.join(ex_dir, ex_base)

    for c in chunks:
        c.filename = os.path.relpath(c.filename, start=args.path)

    with open(ex_path, "wb") as f:
        pickle.dump(chunks, f, pickle.HIGHEST_PROTOCOL)

    return os.path.basename(ex_path)

def export_checksum2(chunks, ofile):
    # fullpath = os.path.abspath(ofile)
    # ex_base = os.path.basename(fullpath).split(".")[0] + ".checksums"
    # ex_dir = os.path.dirname(fullpath)
    # ex_path = os.path.join(ex_dir, ex_base)

    with open(ofile, "a") as f:
        f.write("----block checksums----\n")
        for c in chunks:
            f.write("%s!@%s\n" % (c, c.digest))

    return ofile

def main():
    global args, comm
    signal.signal(signal.SIGINT, sig_handler)
    args = parse_and_bcast(comm, gen_parser)

    try:
        G.src = utils.check_src(args.path)
    except ValueError as e:
        err_and_exit("Error: %s not accessible" % e)

    G.loglevel = args.loglevel
    #G.use_store = args.use_store
    G.reduce_interval = args.interval

    hosts_cnt = tally_hosts()
    circle = Circle()

    if circle.rank == 0:
        print("Running Parameters:\n")
        print("\t{:<20}{:<20}".format("FSUM version:", __version__))
        print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
        print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
        print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))

    fwalk = FWalk(circle, G.src)
    circle.begin(fwalk)
    if G.use_store:
        fwalk.flushdb()
    totalsize = fwalk.epilogue()
    circle.finalize()

    # by default, we use adaptive chunksize
    chunksize = utils.calc_chunksize(totalsize)
    if args.chunksize:
        chunksize = conv_unit(args.chunksize)

    if circle.rank == 0:
        print("Chunksize = ", chunksize)

    circle = Circle()
    fcheck = Checksum(circle, fwalk, chunksize, totalsize, G.total_files)

    circle.begin(fcheck)
    circle.finalize()

    if circle.rank == 0:
        sys.stdout.write("\nAggregating ... ")

    """
    chunkl = circle.comm.gather(fcheck.chunkq)

    if circle.rank == 0:
        chunks = [item for sublist in chunkl for item in sublist]
        chunks.sort()
        sys.stdout.write("%s chunks\n" % len(chunks))
        sha1val = do_checksum(chunks)
        with open(args.output, "w") as f:
            f.write("sha1: %s\n" % sha1val)
            f.write("chunksize: %s\n" % chunksize)
            f.write("fwalk version: %s\n" % __version__)
            f.write("src: %s\n" % utils.choplist(G.src))
            f.write("date: %s\n" % utils.current_time())
            f.write("totalsize: %s\n" % totalsize)

        print("\nSHA1: %s" % sha1val)
        print("Signature file: [%s]" % args.output)
        if args.export_block_signatures:
            export_checksum2(chunks, args.output)
            print("Exporting block signatures ... \n")
    """
    if circle.rank > 0:
        circle.comm.send(fcheck.bfsign.bitarray, dest=0)
    else:
        for p in xrange(1, circle.comm.size):
            other_bitarray = circle.comm.recv(source=p)
            fcheck.bfsign.or_bf(other_bitarray)
    circle.comm.Barrier()

    if circle.comm.rank == 0:
        sha1val = fcheck.bfsign.gen_signature()
        with open(args.output, "w") as f:
            f.write("sha1: %s\n" % sha1val)
            f.write("chunksize: %s\n" % chunksize)
            f.write("fwalk version: %s\n" % __version__)
            f.write("src: %s\n" % utils.choplist(G.src))
            f.write("date: %s\n" % utils.current_time())
            f.write("totalsize: %s\n" % totalsize)

        print("\nSHA1: %s" % sha1val)
        print("Signature file: [%s]" % args.output)

    fcheck.epilogue()

    if circle.comm.rank == 0:
        if os.path.exists(G.tempdir):
            shutil.rmtree(G.tempdir, ignore_errors=True)

if __name__ == "__main__":
    main()
<commit_msg>add --item argument, rate report in fsum<commit_after>#!/usr/bin/env python
#
# author: Feiyi Wang
#
# This program provides a MPI-based parallization for checksumming
#
#
from __future__ import print_function
import os
import hashlib
import argparse
import stat
import sys
import signal
import cPickle as pickle
import shutil
from mpi4py import MPI
from cStringIO import StringIO

from circle import Circle
from _version import get_versions
from task import BaseTask
from utils import bytes_fmt, timestamp2, conv_unit
from fwalk import FWalk
from cio import readn
from fdef import ChunkSum
from globals import G
import utils
from pcircle.mpihelper import tally_hosts, parse_and_bcast, ThrowingArgumentParser
from bfsignature import BFsignature

__version__ = get_versions()['version']
args = None
comm = MPI.COMM_WORLD

def sig_handler(signum, sigstack):
    print("\tUser cancelled ... cleaning up")
    sys.exit(1)

def err_and_exit(msg, code=0):
    if comm.rank == 0:
        print("\n%s" % msg)
    MPI.Finalize()
    sys.exit(0)

def gen_parser():
    parser = ThrowingArgumentParser(description="fsum")
    parser.add_argument("-v", "--version", action="version", version="{version}".format(version=__version__))
    parser.add_argument("--loglevel", default="error", help="log level, default: ERROR")
    parser.add_argument("path", nargs='+', default=".", help="path")
    parser.add_argument("-i", "--interval", type=int, default=10, help="interval")
    parser.add_argument("-o", "--output", default="sha1-%s.sig" % timestamp2(), help="sha1 output file")
    parser.add_argument("--chunksize", help="chunk size (K, M, G, T)")
    parser.add_argument("--item", type=int, default="4000000", help="number of items stored in memory, default: 3000000")
    #parser.add_argument("--use-store", action="store_true", help="Use persistent store")
    #parser.add_argument("--export-block-signatures", action="store_true", help="export block-level signatures")

    return parser

class Checksum(BaseTask):
    def __init__(self, circle, treewalk, chunksize, totalsize=0, totalfiles=0):
        BaseTask.__init__(self, circle)
        self.circle = circle
        self.treewalk = treewalk
        self.totalsize = totalsize
        self.totalfiles = totalfiles
        self.total_chunks = 0
        self.workcnt = 0
        #self.chunkq = []
        self.chunksize = chunksize

        # debug
        self.d = {"rank": "rank %s" % circle.rank}
        self.wtime_started = MPI.Wtime()
        self.wtime_ended = None

        # reduce
        self.vsize = 0
        self.vsize_prior = 0

        self.logger = utils.getLogger(__name__)

        if self.circle.rank == 0:
            print("Start parallel checksumming ...")

    def create(self):

        for fi in self.treewalk.flist:
            if not os.path.islink(fi.path) and stat.S_ISREG(fi.st_mode):
                self.enq_file(fi)

        if len(self.treewalk.flist_buf) > 0:
           for fi in self.treewalk.flist_buf:
               if not os.path.islink(fi.path) and stat.S_ISREG(fi.st_mode):
                   self.enq_file(fi)

                    # right after this, we do first checkpoint

                    # if self.checkpoint_file:
                    #    self.do_no_interrupt_checkpoint()
                    #    self.checkpoint_last = MPI.Wtime()

        if self.treewalk.use_store:
            while self.treewalk.flist_db.qsize > 0:
                fitems, _ = self.treewalk.flist_db.mget(G.DB_BUFSIZE)
                for fi in fitems:
                    if stat.S_ISREG(fi.st_mode):
                        self.enq_file(fi)  # where chunking takes place
                self.treewalk.flist_db.mdel(G.DB_BUFSIZE)

        # gather total chunks
        self.circle.comm.barrier()
        self.total_chunks = self.circle.comm.reduce(self.workcnt, op=MPI.SUM)
        self.total_chunks = self.circle.comm.bcast(self.total_chunks)
        if self.circle.rank == 0:
            print("total chunks = ", self.total_chunks)
        self.bfsign = BFsignature(self.total_chunks)

    def enq_file(self, f):
        """
        f[0] path f[1] mode f[2] size - we enq all in one shot
        CMD = copy src  dest  off_start  last_chunk
        """
        chunks = f.st_size / self.chunksize
        remaining = f.st_size % self.chunksize

        workcnt = 0

        if f.st_size == 0:  # empty file
            ck = ChunkSum(f.path)
            self.enq(ck)
            self.logger.debug("%s" % ck, extra=self.d)
            workcnt += 1
        else:
            for i in range(chunks):
                ck = ChunkSum(f.path)
                ck.offset = i * self.chunksize
                ck.length = self.chunksize
                self.enq(ck)
                self.logger.debug("%s" % ck, extra=self.d)
            workcnt += chunks

        if remaining > 0:
            # send remainder
            ck = ChunkSum(f.path)
            ck.offset = chunks * self.chunksize
            ck.length = remaining
            self.enq(ck)
            self.logger.debug("%s" % ck, extra=self.d)
            workcnt += 1

        # tally work cnt
        self.workcnt += workcnt

    # def process(self):
    #     ck = self.deq()
    #     self.logger.debug("process: %s" % ck, extra=self.d)
    #     blocks = ck.length / self.chunksize
    #     remaining = ck.length % self.chunksize
    #
    #     chunk_digests = StringIO()
    #     try:
    #         fd = os.open(ck.filename, os.O_RDONLY)
    #     except OSError as e:
    #         self.logger.warn("%s, Skipping ... " % e, extra=self.d)
    #         return
    #
    #     os.lseek(fd, ck.offset, os.SEEK_SET)
    #
    #     for i in range(blocks):
    #         chunk_digests.write(hashlib.sha1(readn(fd, self.chunksize)).hexdigest())
    #
    #     if remaining > 0:
    #         chunk_digests.write(hashlib.sha1(readn(fd, remaining)).hexdigest())
    #
    #     ck.digest = chunk_digests.getvalue()
    #
    #     self.chunkq.append(ck)
    #
    #     self.vsize += ck.length
    #     try:
    #         os.close(fd)
    #     except Exception as e:
    #         self.logger.warn(e, extra=self.d)

    def process(self):
        ck = self.deq()
        try:
            fd = os.open(ck.filename, os.O_RDONLY)
        except OSError as e:
            self.logger.warn("%s, Skipping ... " % e, extra=self.d)
            return

        os.lseek(fd, ck.offset, os.SEEK_SET)
        digest = hashlib.sha1()
        blocksize = 4*1024*1024 # 4MiB block
        blockcount = ck.length / blocksize
        remaining = ck.length % blocksize
        for _ in xrange(blockcount):
            digest.update(readn(fd, blocksize))
        if remaining > 0:
            digest.update(readn(fd, remaining))
        try:
            os.close(fd)
        except Exception as e:
            self.logger.warn(e, extra=self.d)
        ck.digest = digest.hexdigest()
        #self.chunkq.append(ck)
        self.vsize += ck.length

        self.bfsign.insert_item(ck.digest)

    def reduce_init(self, buf):
        buf['vsize'] = self.vsize

    def reduce_report(self, buf):
        out = ""
        if self.totalsize != 0:
            out += "%.2f %% block checksummed, " % (100 * float(buf['vsize']) / self.totalsize)

        out += "%s bytes done" % bytes_fmt(buf['vsize'])
        if self.circle.reduce_time_interval != 0:
            rate = float(buf['vsize'] - self.vsize_prior) / self.circle.reduce_time_interval
            self.vsize_prior = buf['vsize']
            out += ", estimated checksum rate: %s/s" % bytes_fmt(rate)
        print(out)

    def reduce_finish(self, buf):
        # self.reduce_report(buf)
        pass

    def reduce(self, buf1, buf2):
        buf1['vsize'] += buf2['vsize']
        return buf1

    def epilogue(self):
        self.wtime_ended = MPI.Wtime()
        if self.circle.rank == 0:
            print("")
            if self.totalsize == 0:
                return
            time = self.wtime_ended - self.wtime_started
            rate = float(self.totalsize) / time
            print("Checksumming Completed In: %.2f seconds" % time)
            print("Average Rate: %s/s\n" % bytes_fmt(rate))

def _read_in_blocks(chunks, chunksize=26214):
    """ deprecated, do_checksum() return signature directly"""
    idx = 0
    total = len(chunks)
    buf = StringIO()

    while True:
        if idx == total:
            break
        buf.write(chunks[idx].digest)
        idx += 1
        if idx % chunksize == 0 or idx == total:
            yield hashlib.sha1(buf.getvalue()).hexdigest()
            buf = StringIO()

def do_checksum(chunks):
    buf = StringIO()
    h = hashlib.sha1()
    for chunk in chunks:
        buf.write(chunk.digest)
        # idx += 1
        # if idx % blocks == 0 or idx == len(chunks):
        #     h.update(buf.getvalue())
        #     buf = StringIO()
    h.update(buf.getvalue())
    return h.hexdigest()

def export_checksum(chunks):
    fullpath = os.path.abspath(args.output)
    ex_base = os.path.basename(fullpath).split(".")[0] + ".checksums"
    ex_dir = os.path.dirname(fullpath)
    ex_path = os.path.join(ex_dir, ex_base)

    for c in chunks:
        c.filename = os.path.relpath(c.filename, start=args.path)

    with open(ex_path, "wb") as f:
        pickle.dump(chunks, f, pickle.HIGHEST_PROTOCOL)

    return os.path.basename(ex_path)

def export_checksum2(chunks, ofile):
    # fullpath = os.path.abspath(ofile)
    # ex_base = os.path.basename(fullpath).split(".")[0] + ".checksums"
    # ex_dir = os.path.dirname(fullpath)
    # ex_path = os.path.join(ex_dir, ex_base)

    with open(ofile, "a") as f:
        f.write("----block checksums----\n")
        for c in chunks:
            f.write("%s!@%s\n" % (c, c.digest))

    return ofile

def main():
    global args, comm
    signal.signal(signal.SIGINT, sig_handler)
    args = parse_and_bcast(comm, gen_parser)

    try:
        G.src = utils.check_src(args.path)
    except ValueError as e:
        err_and_exit("Error: %s not accessible" % e)

    G.loglevel = args.loglevel
    #G.use_store = args.use_store
    G.reduce_interval = args.interval
    G.memitem_threshold = args.item

    hosts_cnt = tally_hosts()
    circle = Circle()

    if circle.rank == 0:
        print("Running Parameters:\n")
        print("\t{:<20}{:<20}".format("FSUM version:", __version__))
        print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
        print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
        print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
        print("\t{:<20}{:<20}".format("Items in memory:", G.memitem_threshold))

    fwalk = FWalk(circle, G.src)
    circle.begin(fwalk)
    if G.use_store:
        fwalk.flushdb()
    totalsize = fwalk.epilogue()
    circle.finalize()

    # by default, we use adaptive chunksize
    chunksize = utils.calc_chunksize(totalsize)
    if args.chunksize:
        chunksize = conv_unit(args.chunksize)

    if circle.rank == 0:
        print("Chunksize = ", chunksize)

    circle = Circle()
    fcheck = Checksum(circle, fwalk, chunksize, totalsize, G.total_files)

    circle.begin(fcheck)
    circle.finalize()

    if circle.rank == 0:
        sys.stdout.write("\nAggregating ... ")

    """
    chunkl = circle.comm.gather(fcheck.chunkq)

    if circle.rank == 0:
        chunks = [item for sublist in chunkl for item in sublist]
        chunks.sort()
        sys.stdout.write("%s chunks\n" % len(chunks))
        sha1val = do_checksum(chunks)
        with open(args.output, "w") as f:
            f.write("sha1: %s\n" % sha1val)
            f.write("chunksize: %s\n" % chunksize)
            f.write("fwalk version: %s\n" % __version__)
            f.write("src: %s\n" % utils.choplist(G.src))
            f.write("date: %s\n" % utils.current_time())
            f.write("totalsize: %s\n" % totalsize)

        print("\nSHA1: %s" % sha1val)
        print("Signature file: [%s]" % args.output)
        if args.export_block_signatures:
            export_checksum2(chunks, args.output)
            print("Exporting block signatures ... \n")
    """
    if circle.rank > 0:
        circle.comm.send(fcheck.bfsign.bitarray, dest=0)
    else:
        for p in xrange(1, circle.comm.size):
            other_bitarray = circle.comm.recv(source=p)
            fcheck.bfsign.or_bf(other_bitarray)
    circle.comm.Barrier()

    if circle.comm.rank == 0:
        sha1val = fcheck.bfsign.gen_signature()
        with open(args.output, "w") as f:
            f.write("sha1: %s\n" % sha1val)
            f.write("chunksize: %s\n" % chunksize)
            f.write("fwalk version: %s\n" % __version__)
            f.write("src: %s\n" % utils.choplist(G.src))
            f.write("date: %s\n" % utils.current_time())
            f.write("totalsize: %s\n" % totalsize)

        print("\nSHA1: %s" % sha1val)
        print("Signature file: [%s]" % args.output)

    fcheck.epilogue()

    if circle.comm.rank == 0:
        if os.path.exists(G.tempdir):
            shutil.rmtree(G.tempdir, ignore_errors=True)

if __name__ == "__main__":
    main()
<|endoftext|>