<commit_before>
from __future__ import print_function

import re
import select
import threading
import traceback
import codecs

from six.moves import queue

def _handle_output_packet_string(packet_contents):
    if (not packet_contents) or (len(packet_contents) < 1):
        return None
    elif packet_contents[0] != "O":
        return None
    elif packet_contents == "OK":
        return None
    else:
        return packet_contents[1:].decode("hex")

def _dump_queue(the_queue):
    while not the_queue.empty():
        print(codecs.encode(the_queue.get(True), "string_escape"))
        print("\n")

class PumpQueues(object):

    def __init__(self):
        self._output_queue = queue.Queue()
        self._packet_queue = queue.Queue()

    def output_queue(self):
        return self._output_queue

    def packet_queue(self):
        return self._packet_queue

    def verify_queues_empty(self):
        # Warn if there is any content left in any of the queues.
        # That would represent unmatched packets.
        if not self.output_queue().empty():
            print("warning: output queue entries still exist:")
            _dump_queue(self.output_queue())
            print("from here:")
            traceback.print_stack()

        if not self.packet_queue().empty():
            print("warning: packet queue entries still exist:")
            _dump_queue(self.packet_queue())
            print("from here:")
            traceback.print_stack()

class SocketPacketPump(object):
    """A threaded packet reader that partitions packets into two streams.

    All incoming $O packet content is accumulated with the current accumulation
    state put into the OutputQueue.

    All other incoming packets are placed in the packet queue.

    A select thread can be started and stopped, and runs to place packet
    content into the two queues.
    """

    _GDB_REMOTE_PACKET_REGEX = re.compile(r'^\$([^\#]*)#[0-9a-fA-F]{2}')

    def __init__(self, pump_socket, pump_queues, logger=None):
        if not pump_socket:
            raise Exception("pump_socket cannot be None")

        self._thread = None
        self._stop_thread = False
        self._socket = pump_socket
        self._logger = logger
        self._receive_buffer = ""
        self._accumulated_output = ""
        self._pump_queues = pump_queues

    def __enter__(self):
        """Support the python 'with' statement.

        Start the pump thread."""
        self.start_pump_thread()
        return self

    def __exit__(self, exit_type, value, the_traceback):
        """Support the python 'with' statement.

        Shut down the pump thread."""
        self.stop_pump_thread()

    def start_pump_thread(self):
        if self._thread:
            raise Exception("pump thread is already running")
        self._stop_thread = False
        self._thread = threading.Thread(target=self._run_method)
        self._thread.start()

    def stop_pump_thread(self):
        self._stop_thread = True
        if self._thread:
            self._thread.join()

    def _process_new_bytes(self, new_bytes):
        if not new_bytes:
            return
        if len(new_bytes) < 1:
            return

        # Add new bytes to our accumulated unprocessed packet bytes.
        self._receive_buffer += new_bytes

        # Parse fully-formed packets into individual packets.
        has_more = len(self._receive_buffer) > 0
        while has_more:
            if len(self._receive_buffer) <= 0:
                has_more = False
            # handle '+' ack
            elif self._receive_buffer[0] == "+":
                self._pump_queues.packet_queue().put("+")
                self._receive_buffer = self._receive_buffer[1:]
                if self._logger:
                    self._logger.debug(
                        "parsed packet from stub: +\n" +
                        "new receive_buffer: {}".format(
                            self._receive_buffer))
            else:
                packet_match = self._GDB_REMOTE_PACKET_REGEX.match(
                    self._receive_buffer)
                if packet_match:
                    # Our receive buffer matches a packet at the
                    # start of the receive buffer.
                    new_output_content = _handle_output_packet_string(
                        packet_match.group(1))
                    if new_output_content:
                        # This was an $O packet with new content.
                        self._accumulated_output += new_output_content
                        self._pump_queues.output_queue().put(self._accumulated_output)
                    else:
                        # Any packet other than $O.
                        self._pump_queues.packet_queue().put(packet_match.group(0))

                    # Remove the parsed packet from the receive
                    # buffer.
                    self._receive_buffer = self._receive_buffer[
                        len(packet_match.group(0)):]
                    if self._logger:
                        self._logger.debug(
                            "parsed packet from stub: " +
                            packet_match.group(0))
                        self._logger.debug(
                            "new receive_buffer: " +
                            self._receive_buffer)
                else:
                    # We don't have enough in the receive bufferto make a full
                    # packet. Stop trying until we read more.
                    has_more = False

    def _run_method(self):
        self._receive_buffer = ""
        self._accumulated_output = ""

        if self._logger:
            self._logger.info("socket pump starting")

        # Keep looping around until we're asked to stop the thread.
        while not self._stop_thread:
            can_read, _, _ = select.select([self._socket], [], [], 0)
            if can_read and self._socket in can_read:
                try:
                    new_bytes = self._socket.recv(4096).decode("utf-8")
                    if self._logger and new_bytes and len(new_bytes) > 0:
                        self._logger.debug(
                            "pump received bytes: {}".format(new_bytes))
                except:
                    # Likely a closed socket.  Done with the pump thread.
                    if self._logger:
                        self._logger.debug(
                            "socket read failed, stopping pump read thread\n" +
                            traceback.format_exc(3))
                    break
                self._process_new_bytes(new_bytes)

        if self._logger:
            self._logger.info("socket pump exiting")

    def get_accumulated_output(self):
        return self._accumulated_output

    def get_receive_buffer(self):
        return self._receive_buffer
<commit_msg>Revert "Fix one more string/bytes issue in lldb-server tests"<commit_after>
from __future__ import print_function

import re
import select
import threading
import traceback
import codecs

from six.moves import queue

def _handle_output_packet_string(packet_contents):
    if (not packet_contents) or (len(packet_contents) < 1):
        return None
    elif packet_contents[0] != "O":
        return None
    elif packet_contents == "OK":
        return None
    else:
        return packet_contents[1:].decode("hex")

def _dump_queue(the_queue):
    while not the_queue.empty():
        print(codecs.encode(the_queue.get(True), "string_escape"))
        print("\n")

class PumpQueues(object):

    def __init__(self):
        self._output_queue = queue.Queue()
        self._packet_queue = queue.Queue()

    def output_queue(self):
        return self._output_queue

    def packet_queue(self):
        return self._packet_queue

    def verify_queues_empty(self):
        # Warn if there is any content left in any of the queues.
        # That would represent unmatched packets.
        if not self.output_queue().empty():
            print("warning: output queue entries still exist:")
            _dump_queue(self.output_queue())
            print("from here:")
            traceback.print_stack()

        if not self.packet_queue().empty():
            print("warning: packet queue entries still exist:")
            _dump_queue(self.packet_queue())
            print("from here:")
            traceback.print_stack()

class SocketPacketPump(object):
    """A threaded packet reader that partitions packets into two streams.

    All incoming $O packet content is accumulated with the current accumulation
    state put into the OutputQueue.

    All other incoming packets are placed in the packet queue.

    A select thread can be started and stopped, and runs to place packet
    content into the two queues.
    """

    _GDB_REMOTE_PACKET_REGEX = re.compile(r'^\$([^\#]*)#[0-9a-fA-F]{2}')

    def __init__(self, pump_socket, pump_queues, logger=None):
        if not pump_socket:
            raise Exception("pump_socket cannot be None")

        self._thread = None
        self._stop_thread = False
        self._socket = pump_socket
        self._logger = logger
        self._receive_buffer = ""
        self._accumulated_output = ""
        self._pump_queues = pump_queues

    def __enter__(self):
        """Support the python 'with' statement.

        Start the pump thread."""
        self.start_pump_thread()
        return self

    def __exit__(self, exit_type, value, the_traceback):
        """Support the python 'with' statement.

        Shut down the pump thread."""
        self.stop_pump_thread()

    def start_pump_thread(self):
        if self._thread:
            raise Exception("pump thread is already running")
        self._stop_thread = False
        self._thread = threading.Thread(target=self._run_method)
        self._thread.start()

    def stop_pump_thread(self):
        self._stop_thread = True
        if self._thread:
            self._thread.join()

    def _process_new_bytes(self, new_bytes):
        if not new_bytes:
            return
        if len(new_bytes) < 1:
            return

        # Add new bytes to our accumulated unprocessed packet bytes.
        self._receive_buffer += new_bytes

        # Parse fully-formed packets into individual packets.
        has_more = len(self._receive_buffer) > 0
        while has_more:
            if len(self._receive_buffer) <= 0:
                has_more = False
            # handle '+' ack
            elif self._receive_buffer[0] == "+":
                self._pump_queues.packet_queue().put("+")
                self._receive_buffer = self._receive_buffer[1:]
                if self._logger:
                    self._logger.debug(
                        "parsed packet from stub: +\n" +
                        "new receive_buffer: {}".format(
                            self._receive_buffer))
            else:
                packet_match = self._GDB_REMOTE_PACKET_REGEX.match(
                    self._receive_buffer)
                if packet_match:
                    # Our receive buffer matches a packet at the
                    # start of the receive buffer.
                    new_output_content = _handle_output_packet_string(
                        packet_match.group(1))
                    if new_output_content:
                        # This was an $O packet with new content.
                        self._accumulated_output += new_output_content
                        self._pump_queues.output_queue().put(self._accumulated_output)
                    else:
                        # Any packet other than $O.
                        self._pump_queues.packet_queue().put(packet_match.group(0))

                    # Remove the parsed packet from the receive
                    # buffer.
                    self._receive_buffer = self._receive_buffer[
                        len(packet_match.group(0)):]
                    if self._logger:
                        self._logger.debug(
                            "parsed packet from stub: " +
                            packet_match.group(0))
                        self._logger.debug(
                            "new receive_buffer: " +
                            self._receive_buffer)
                else:
                    # We don't have enough in the receive bufferto make a full
                    # packet. Stop trying until we read more.
                    has_more = False

    def _run_method(self):
        self._receive_buffer = ""
        self._accumulated_output = ""

        if self._logger:
            self._logger.info("socket pump starting")

        # Keep looping around until we're asked to stop the thread.
        while not self._stop_thread:
            can_read, _, _ = select.select([self._socket], [], [], 0)
            if can_read and self._socket in can_read:
                try:
                    new_bytes = self._socket.recv(4096)
                    if self._logger and new_bytes and len(new_bytes) > 0:
                        self._logger.debug(
                            "pump received bytes: {}".format(new_bytes))
                except:
                    # Likely a closed socket.  Done with the pump thread.
                    if self._logger:
                        self._logger.debug(
                            "socket read failed, stopping pump read thread\n" +
                            traceback.format_exc(3))
                    break
                self._process_new_bytes(new_bytes)

        if self._logger:
            self._logger.info("socket pump exiting")

    def get_accumulated_output(self):
        return self._accumulated_output

    def get_receive_buffer(self):
        return self._receive_buffer
<|endoftext|>


<commit_before>
<commit_msg>Implement unittests for models<commit_after>from twisted.internet.defer import inlineCallbacks

from globaleaks.models import Context, Receiver, Folder, InternalFile
from globaleaks.db import tables
from globaleaks.settings import transact
from globaleaks.tests import helpers

class TestModels(helpers.TestHandler):
    def setUp(self):
        return self._setup_database()

    @transact
    def _setup_database(self):
        self.store.execute(tables.generateCreateQuery(Context))
        self.store.execute(tables.generateCreateQuery(Receiver))
        self.store.execute(tables.generateCreateQuery(Folder))
        self.store.execute(tables.generateCreateQuery(InternalFile))

    @transact
    def create_context_with_receivers(self):
        context = Context()
        receiver1 = Receiver()
        receiver2 = Receiver()
        context.receivers.add(receiver1)
        context.receivers.add(receiver2)
        self.store.add(context)
        return context.id

    @transact
    def create_folder_with_files(self):
        file1 = InternalFile()
        file2 = InternalFile()
        self.store.add(file1)
        self.store.add(file2)

        folder = Folder()
        folder.files.add(file1)
        folder.files.add(file2)
        self.store.add(folder)
        return folder.id

    @transact
    def list_receivers_of_context(self, context_id):
        context = self.store.find(Context, Context.id == context_id).one()
        receivers = []
        for receiver in context.receivers:
            receivers.append(receiver.id)
        return receivers

    @transact
    def list_files_of_folder(self, folder_id):
        folder = self.store.find(Folder, Folder.id == folder_id).one()
        files = []
        for file in folder.files:
            files.append(file.id)
        return files

    @inlineCallbacks
    def test_context_receiver_reference(self):
        context_id = yield self.create_context_with_receivers()
        receivers = yield self.list_receivers_of_context(context_id)
        self.assertEqual(2, len(receivers))

    @inlineCallbacks
    def test_folder_receiver_reference_set(self):
        context_id = yield self.create_folder_with_files()
        files = yield self.list_files_of_folder(context_id)
        self.assertEqual(2, len(files))

<|endoftext|>


<commit_before>"""
Authors: Tim Bedin, Tim Erwin

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the ArgumentCreator class.

"""

import itertools
import logging

from cwsl.core.constraint import Constraint

module_logger = logging.getLogger('cwsl.core.argument_creator')

class ArgumentCreator(object):
    '''
    The ArgumentCreator class compares (one or more) input DataSets with
    (one or more) output FileCreators.

    There are different cases to consider -

    1. The ArgumentCreator adds/removes no attributes.
        In this case, there is a one-to-one mapping between input and output.

        Here the ArgumentCreator should return the a tuple (inputfile, outputfile),
        one output file (from the file creator) for each file
        in the input DataSet.

    2. The output has more attributes than the input.

        Here each input file will create many output files.

        In this case, it returns a mapping like
        (inputfile1, outputfile1), (inputfile1, outputfile2) ...
        up to (inputfile1, outputfileN), where N is the number of
        allowed values of the added attribute.

        It will do this once for every file in the input Dataset.

    3. The ArgumentCreator removes one or more attributes.

        Here many input files will be combined to create a single outputfile.

        For this case, the looper will return something like
        ([inputfile1, inputfile2, ..., inputfileN], outputfile)

    There are also mixed cases in which these operations are combined.

    '''

    def __init__(self, input_datasets, output_file_creators,
                 map_dict={}):

        ''' The class takes in a list of DataSet objects for its input and
        a list of FileCreator objects for output.

        Optionally a map_dict can be passed in which maps a constraint in the
        input with one from the output.

        The FileCreator is associated with a DataSet to check
        if files already exist.

        '''

        self.inputs = input_datasets
        self.output = output_file_creators

        self.map_dict = map_dict

        # Put the inputs/outputs into lists if they are not iterable.
        if not hasattr(self.inputs, '__iter__'):
            self.input = [self.inputs]
        if not hasattr(self.output, '__iter__'):
            self.output = [self.output]

        in_constraints = set.union(*[in_set.constraints
                                     for in_set in self.inputs])
        out_constraints = set.union(*[out_set.constraints
                                      for out_set in self.output])
        module_logger.debug('All input constraints are: ' + str(in_constraints))
        module_logger.debug('All output constraints are: ' + str(out_constraints))

        # Now apply the check constraints function to map
        # empty output constraints to the corresponding constraint in the input
        # and ensure that constraints are not repeated.
        self.input_cons, self.output_cons = self.check_constraints(in_constraints,
                                                                   out_constraints)

        # Find the shared attributes between the input and output.
        self.shared_constraints = self.input_cons.intersection(self.output_cons)

        # Find constraints that exist in the input but not the output - these
        # will be added to the returned combination dictionary for use in positional
        # or keyword arguments.
        self.input_only = self.input_cons.difference(self.output_cons)
        # The cons.values set is turned into a list so we can just get one value.
        self.input_only_dict = {cons.key: list(cons.values)[0] for cons in self.input_only}

        # Output only constraints are effectively shared constraints and must be added to
        # the valid combinations and the shared_constraints set.
        in_keys = [cons.key for cons in self.input_cons]
        self.output_only = [cons for cons in self.output_cons
                            if cons.key not in in_keys]

        self.shared_constraints = self.shared_constraints.union(self.output_only)

        module_logger.debug("Shared input and output constraints are: {0}".
                            format(self.shared_constraints))

        self.shared_keys = set([cons.key for cons in self.shared_constraints])

    def check_constraints(self, in_cons, out_cons):
        """ This method checks and cleans up any problems with input and
            output constraints in a logical manner. """

        # Check for repeated constraints.
        # If a constraint appears multiple times, add it to
        # the 'repeated_keys' list.
        new_ins = set()
        key_list = [cons.key for cons in in_cons]

        key_counter = {}
        for key in key_list:
            try:
                key_counter[key] += 1
            except KeyError:
                key_counter[key] = 1

        repeated_keys = []
        for cons in in_cons:
            if key_counter[cons.key] == 1:
                new_ins.add(cons)
            else:
                repeated_keys.append(cons.key)

        # Check that there are no empty constraints in the input - just
        # to make sure!
        for in_constraint in in_cons:
            if in_constraint.values == set():
                module_logger.error("Constraint {0} has no values!".format(in_constraint))
            assert(in_constraint.values != set())

        if repeated_keys:
            module_logger.debug("Repeated keys are: {0}".format(repeated_keys))

        # For repeated output keys, just get the shortest list.
        for key in set(repeated_keys):
            multi_cons = [cons for cons in in_cons
                          if cons.key == key]
            new_vals = set.intersection(*[set(con.values
                                              for con in multi_cons)])
            new_constraint = Constraint(key, new_vals)
            new_ins.add(new_constraint)

        # Now fix up empty output constraints.
        new_outs = set()
        for out_constraint in out_cons:
            if out_constraint.values:
                new_outs.add(out_constraint)
            else:
                # If it is empty, get the matching constraint
                # from the input.
                new_cons = [in_con for in_con in new_ins
                            if in_con.key == out_constraint.key]
                assert(len(new_cons) == 1)
                if not new_cons:
                    module_logger.error("Output constraint {0} has no values!"
                                        .format(out_constraints))
                new_outs.add(*new_cons)

        return new_ins, new_outs

    def __iter__(self):
        # Sets up the combinations in the argument_creator for looping.

        valids = [ds.valid_combinations for ds in self.inputs]
        set_of_valids = set.union(*valids)

        if self.output_only:
            module_logger.debug("Output only constraints are: {}"
                                .format(self.output_only))
            self.valid_iter = self.custom_iterator(set_of_valids, self.output_only)
        else:
            self.valid_iter = iter(set_of_valids)

        # Store which combinations of constraints
        # have been processed this iteration.
        self.done_combinations = []

        return self

    def custom_iterator(self, valid_set, output_cons):
        """ This iterator adds any output_only constraints to the valid combinations from the input."""

        all_possibles = []
        for combination in itertools.product(*output_cons):
            this_com_list = []
            for cons in combination:
                this_com_list.append(Constraint(cons[0], [cons[1]]))
            all_possibles.append(this_com_list)

        module_logger.debug("All possible new constraints to be added are: {}".format(all_possibles))
        for items in itertools.product(iter(valid_set), all_possibles):
            input_list = list(items[0])
            input_list += items[1]
            module_logger.debug("Input list is: {}".format(input_list))
            yield(set(input_list))

    def next(self):
        """ Return the next group of input and output file/metafile objects.

        """

        next_in = None
        next_out = None

        while not next_in and not next_out:

            # Get a particular set of valid constraints from the input.
            this_combination_vals = self.valid_iter.next()
            module_logger.debug("This combination values are: {}"
                                .format(this_combination_vals))

            # Get the subset of this combination which is in the shared
            # constraints.
            this_shared = [cons for cons in this_combination_vals
                           if cons.key in self.shared_keys]

            this_combination_dict = {}
            for cons in this_shared:
                this_combination_dict[cons.key] = iter(cons.values).next()

            module_logger.debug("This combination is: {0}"
                                .format(this_combination_dict))

            # If this combination has been done, continue.
            this_hash = hash(frozenset(this_combination_dict.items()))
            if this_hash in self.done_combinations:
                module_logger.debug("The combination with hash {0} has already been processed".format(this_hash))
                continue

            # Get the matching files for this combination of constraints.
            in_metas = []
            out_metas = []

            for input_ds in self.inputs:
                in_metas.append(input_ds.get_files(this_combination_dict,
                                check=True))

            module_logger.debug("Found matching inputs: {0}".format(in_metas))
            # We only need to look for outputs if there is an input file present
            # for every input dataset.
            module_logger.debug("Length of in_metas: {}, length of self.inputs: {}"
                                .format(len(in_metas), len(self.inputs)))
            module_logger.debug("self.inputs: {}".format(self.inputs))
            if len(in_metas) != len(self.inputs):
                in_metas = []

            if in_metas[0]:
                module_logger.debug("Searching for output metafiles.")

                for output_ds in self.output:
                    these_files = output_ds.get_files(this_combination_dict,
                                                      check=False, update=True)
                    if these_files is not None:
                        out_metas.append(these_files)
            else:
                module_logger.debug("No files found for this combination.")

            next_in = in_metas
            next_out = out_metas

            # This combination is done, at its hash to the list.
            self.done_combinations.append(this_hash)

            # Returns a tuple of lists if a file exists, otherwise loop again.
            if next_in and next_out:
                full_return = dict(this_combination_dict.items() + self.input_only_dict.items())
                print(full_return)
                return (next_in, next_out, full_return)
            else:
                # If there is no input metafiles,
                # call for the next combination.
                continue
<commit_msg>Removed check for repeated constraints<commit_after>"""
Authors: Tim Bedin, Tim Erwin

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the ArgumentCreator class.

"""

import itertools
import logging

from cwsl.core.constraint import Constraint

module_logger = logging.getLogger('cwsl.core.argument_creator')

class ArgumentCreator(object):
    '''
    The ArgumentCreator class compares (one or more) input DataSets with
    (one or more) output FileCreators.

    There are different cases to consider -

    1. The ArgumentCreator adds/removes no attributes.
        In this case, there is a one-to-one mapping between input and output.

        Here the ArgumentCreator should return the a tuple (inputfile, outputfile),
        one output file (from the file creator) for each file
        in the input DataSet.

    2. The output has more attributes than the input.

        Here each input file will create many output files.

        In this case, it returns a mapping like
        (inputfile1, outputfile1), (inputfile1, outputfile2) ...
        up to (inputfile1, outputfileN), where N is the number of
        allowed values of the added attribute.

        It will do this once for every file in the input Dataset.

    3. The ArgumentCreator removes one or more attributes.

        Here many input files will be combined to create a single outputfile.

        For this case, the looper will return something like
        ([inputfile1, inputfile2, ..., inputfileN], outputfile)

    There are also mixed cases in which these operations are combined.

    '''

    def __init__(self, input_datasets, output_file_creators,
                 map_dict={}):

        ''' The class takes in a list of DataSet objects for its input and
        a list of FileCreator objects for output.

        Optionally a map_dict can be passed in which maps a constraint in the
        input with one from the output.

        The FileCreator is associated with a DataSet to check
        if files already exist.

        '''

        self.inputs = input_datasets
        self.output = output_file_creators

        self.map_dict = map_dict

        # Put the inputs/outputs into lists if they are not iterable.
        if not hasattr(self.inputs, '__iter__'):
            self.input = [self.inputs]
        if not hasattr(self.output, '__iter__'):
            self.output = [self.output]

        in_constraints = set.union(*[in_set.constraints
                                     for in_set in self.inputs])
        out_constraints = set.union(*[out_set.constraints
                                      for out_set in self.output])
        module_logger.debug('All input constraints are: ' + str(in_constraints))
        module_logger.debug('All output constraints are: ' + str(out_constraints))

        # Now apply the check constraints function to map
        # empty output constraints to the corresponding constraint in the input
        # and ensure that constraints are not repeated.
        self.input_cons, self.output_cons = self.check_constraints(in_constraints,
                                                                   out_constraints)

        # Find the shared attributes between the input and output.
        self.shared_constraints = self.input_cons.intersection(self.output_cons)

        # Find constraints that exist in the input but not the output - these
        # will be added to the returned combination dictionary for use in positional
        # or keyword arguments.
        self.input_only = self.input_cons.difference(self.output_cons)

        # The cons.values set is turned into a list so we can just get one value.
        module_logger.debug("self.input_only: {}".format(self.input_only))
        self.input_only_dict = {cons.key: list(cons.values)[0] for cons in self.input_only}

        # Output only constraints are effectively shared constraints and must be added to
        # the valid combinations and the shared_constraints set.
        in_keys = [cons.key for cons in self.input_cons]
        self.output_only = [cons for cons in self.output_cons
                            if cons.key not in in_keys]

        self.shared_constraints = self.shared_constraints.union(self.output_only)

        module_logger.debug("Shared input and output constraints are: {0}".
                            format(self.shared_constraints))

        self.shared_keys = set([cons.key for cons in self.shared_constraints])

    def check_constraints(self, in_cons, out_cons):
        """ This method checks and cleans up any problems with input and
            output constraints in a logical manner. """

        # Check that there are no empty constraints in the input - just
        # to make sure!
        for in_constraint in in_cons:
            if in_constraint.values == set():
                module_logger.error("Constraint {0} has no values!".format(in_constraint))
            assert(in_constraint.values != set())

        # Now fix up empty output constraints.
        new_outs = set()
        for out_constraint in out_cons:
            if out_constraint.values:
                new_outs.add(out_constraint)
            else:
                # If it is empty, get the matching constraint
                # from the input.
                new_cons = [in_con for in_con in new_ins
                            if in_con.key == out_constraint.key]
                assert(len(new_cons) == 1)
                if not new_cons:
                    module_logger.error("Output constraint {0} has no values!"
                                        .format(out_constraints))
                new_outs.add(*new_cons)

        return in_cons, new_outs

    def __iter__(self):
        # Sets up the combinations in the argument_creator for looping.

        valids = [ds.valid_combinations for ds in self.inputs]
        set_of_valids = set.union(*valids)

        if self.output_only:
            module_logger.debug("Output only constraints are: {}"
                                .format(self.output_only))
            self.valid_iter = self.custom_iterator(set_of_valids, self.output_only)
        else:
            self.valid_iter = iter(set_of_valids)

        # Store which combinations of constraints
        # have been processed this iteration.
        self.done_combinations = []

        return self

    def custom_iterator(self, valid_set, output_cons):
        """ This iterator adds any output_only constraints to the valid combinations from the input."""

        all_possibles = []
        for combination in itertools.product(*output_cons):
            this_com_list = []
            for cons in combination:
                this_com_list.append(Constraint(cons[0], [cons[1]]))
            all_possibles.append(this_com_list)

        module_logger.debug("All possible new constraints to be added are: {}".format(all_possibles))
        for items in itertools.product(iter(valid_set), all_possibles):
            input_list = list(items[0])
            input_list += items[1]
            module_logger.debug("Input list is: {}".format(input_list))
            yield(set(input_list))

    def next(self):
        """ Return the next group of input and output file/metafile objects.

        """

        next_in = None
        next_out = None

        while not next_in and not next_out:

            # Get a particular set of valid constraints from the input.
            this_combination_vals = self.valid_iter.next()
            module_logger.debug("This combination values are: {}"
                                .format(this_combination_vals))

            # Get the subset of this combination which is in the shared
            # constraints.
            this_shared = [cons for cons in this_combination_vals
                           if cons.key in self.shared_keys]

            this_combination_dict = {}
            for cons in this_shared:
                this_combination_dict[cons.key] = iter(cons.values).next()

            module_logger.debug("This combination is: {0}"
                                .format(this_combination_dict))

            # If this combination has been done, continue.
            this_hash = hash(frozenset(this_combination_dict.items()))
            if this_hash in self.done_combinations:
                module_logger.debug("The combination with hash {0} has already been processed".format(this_hash))
                continue

            # Get the matching files for this combination of constraints.
            in_metas = []
            out_metas = []

            for input_ds in self.inputs:
                in_metas.append(input_ds.get_files(this_combination_dict,
                                check=True))

            module_logger.debug("Found matching inputs: {0}".format(in_metas))
            # We only need to look for outputs if there is an input file present
            # for every input dataset.
            module_logger.debug("Length of in_metas: {}, length of self.inputs: {}"
                                .format(len(in_metas), len(self.inputs)))
            module_logger.debug("self.inputs: {}".format(self.inputs))
            if len(in_metas) != len(self.inputs):
                in_metas = []

            if in_metas[0]:
                module_logger.debug("Searching for output metafiles.")

                for output_ds in self.output:
                    these_files = output_ds.get_files(this_combination_dict,
                                                      check=False, update=True)
                    if these_files is not None:
                        out_metas.append(these_files)
            else:
                module_logger.debug("No files found for this combination.")

            next_in = in_metas
            next_out = out_metas

            # This combination is done, at its hash to the list.
            self.done_combinations.append(this_hash)

            # Returns a tuple of lists if a file exists, otherwise loop again.
            if next_in and next_out:
                full_return = dict(this_combination_dict.items() + self.input_only_dict.items())
                print(full_return)
                return (next_in, next_out, full_return)
            else:
                # If there is no input metafiles,
                # call for the next combination.
                continue
<|endoftext|>


<commit_before>#!env /usr/bin/python3

#Copyright (C) 2013 Baptiste COVOLATO <b.covolato@gmail.com>
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of
#this software and associated documentation files (the "Software"), to deal in
#the Software without restriction, including without limitation the rights to
#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
#of the Software, and to permit persons to whom the Software is furnished to do
#so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#SOFTWARE.

import sys
import subprocess
import os
import re
import configparser
import filecmp
import argparse

testme_version = "0.2"
testme_config_name = "testme.conf"
testme_args = {}

class TestPrinter:
    def __init__(self):
        self.verbose = False
        self.summary_on = True

    def print_summary(self, category, good, total):
        if self.summary_on:
            print("[TESTME] Category : '" + category + "' :", good, "/", total)

    def print_verbose(self, message):
        if self.verbose:
            print(message)

    def print_error(self, message):
        print("\033[91m" + message + "\033[0m")

    def print_result(self, test_result, file_name):
        if (test_result):
            print("\033[32m[TESTME] Test :", file_name, "passed\033[0m")
        elif (not test_result):
            print("\033[91m[TESTME] Test :", file_name, "failed\033[0m")

class TestSuit:
    def __init__(self, printer):
        self.environement = {}
        self.test_list = {}
        self.running_cat = ""
        self.running_dir = ""
        self.ext = ""
        self.total_test = 0
        self.total_good = 0
        self.cat_test = 0
        self.cat_good = 0
        self.printer = printer

    def expand(self, string):
        while 1:
            m = re.search('\$\{(.*?)\}', string)

            if m == None or self.environement.get(m.group(1), None) == None:
                break

            string = re.sub('\$\{' + m.group(1) + '\}', self.environement[m.group(1)], string)

        return string

    def cat_field_get(self, field):
        if isinstance(self.test_list[self.running_cat][field], str):
            return self.expand(self.test_list[self.running_cat][field])
        else:
            return self.test_list[self.running_cat][field]

    def is_runnable(self):
        if testme_args.get('category', None) != None:
            return testme_args['category'].count(self.running_cat) > 0

        return True

    def change_extension(self, change_file, ext):
        path = os.path.splitext(change_file)
        return path[0] + "." + ext

    def stdin_input(self, test_name):
        stdin_val = ""

        if self.cat_field_get('stdin'):
            stdin_file = os.path.join(self.cat_field_get('stdin_dir'), test_name)
            stdin_file = self.change_extension(stdin_file, self.cat_field_get('stdin_ext'))
            stdin_fd = open(stdin_file, "r")
            stdin_val = stdin_fd.read()
            stdin_fd.close()

        return stdin_val

    def compare_out(self, output, value, ret, test_file):

        if self.cat_field_get(output):
            filename = self.change_extension(test_file, self.cat_field_get(output + '_ext'))
            filename = os.path.join(self.cat_field_get(output + '_dir'), filename)

            if os.path.exists(filename):
                fd = open(filename, "r")
                file_content = fd.read()
                fd.close()

                ret = ret and (file_content == value)
            else:
                print_verbose("\033[33m[TESTME] " + filename + " not present " + output + " ignored\033[0m")

        return ret

    def run_test(self, test_file):
        ret_value = True

        if self.cat_field_get('input'):
            input_file = os.path.join(self.cat_field_get('input_dir'), test_file)
            self.environement['TESTME_RUNNING_INPUT'] = input_file

        stdinput = self.stdin_input(test_file)

        command = self.cat_field_get('cmd_line')

        process = subprocess.Popen(command, stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   shell=True)

        stdout, stderr = process.communicate(stdinput.encode('utf-8'))
        stdout, stderr = stdout.decode('utf-8'), stderr.decode('utf-8')

        ret_value = self.compare_out('stdout', stdout, ret_value, test_file)
        ret_value = self.compare_out('stderr', stderr, ret_value, test_file)

        if self.cat_field_get('check_code'):
            ret_value &= str(process.returncode) in self.cat_field_get('error_code')

        return ret_value

    def run_directory(self):
        try:
            for dir_file in os.listdir(self.cat_field_get(self.running_dir)):
                if dir_file.endswith(self.cat_field_get(self.ext)):
                    self.total_test += 1
                    self.cat_test += 1
                    test_result = self.run_test(dir_file)
                    if test_result:
                        self.cat_good += 1
                        self.total_good += 1
                    self.printer.print_result(test_result, dir_file)

            self.printer.print_summary(self.running_cat, self.cat_good,
                                       self.cat_test)

        except OSError:
           self.printer.print_error("[TESTME] Fatal error with " +
                                    "directory : " +
                                    self.cat_field_get(self.running_dir))

    def run(self):
        self.total_test = 0
        self.total_good = 0

        # Run every categories
        for cat in self.test_list:
            self.cat_test = 0
            self.cat_good = 0
            self.running_cat = cat
            self.environement['TESTME_CATEGORY'] = cat
            self.environement['TESTME_CATEGORY_DIR'] = os.path.join(self.environement['TESTME_TESTDIR'], cat)

            if not self.is_runnable():
                continue

            print("\n[TESTME] Running tests in", self.running_cat)
            self.running_dir = 'input_dir' if self.cat_field_get('input') else 'stdin_dir'
            self.ext = 'input_ext' if self.cat_field_get('input') else 'stdin_ext'

            self.run_directory()

        print("\n[TESTME] Final result :", self.total_good, "/", self.total_test)

class ConfigBuilder:
    def __init__(self, printer, filename):
        self.filename = filename
        self.configuration = {}
        self.printer = printer

    def default(self, cat_name):
        return {'stdout' : 0, # Stdout test
                'stdin' : 0, # Stdin test
                'stderr' : 0, # Stderr test
                'input' : 0, # Input test
                'input_dir' : os.path.join(testme_args['dir'], cat_name, "input"),
                'stdout_dir' : os.path.join(testme_args['dir'], cat_name, "stdout"),
                'stdin_dir' : os.path.join(testme_args['dir'], cat_name, "stdin"),
                'stderr_dir' : os.path.join(testme_args['dir'], cat_name, "stderr"),
                'input_ext' : 'inp',
                'stdout_ext' : 'out',
                'stdin_ext' : 'in',
                'stderr_ext' : 'err',
                'cmd_line' : '',
                'check_code' : 0,
                'error_code' : ["0"],
                'display_ok_tests' : 1,
                'display_ko_tests' : 1,
                'display_summary' : 1}

    def get_bool(self, section, value):
        try:
            self.configuration[section][value] = self.parser.getboolean(section,
                                                                        value)
        except configparser.NoOptionError:
            return 0
        except:
            die_on_error(value + " in section '" + section + "' is not a boolean")

    def get_string(self, section, value):
        try:
            self.configuration[section][value] = self.parser.get(section, value)
            return
        except configparser.NoOptionError:
            return

    def get_int(self, section, value):
        try:
            self.configuration[section][value] = self.parser.getint(section, value)
        except configparser.NoOptionError:
            return 0
        except:
            die_on_error(value + " in section '" + section + "' is not an integer")

    def parse_config(self):
        self.parser = configparser.RawConfigParser()
        self.parser.read(self.filename)

        for section in self.parser.sections():
            self.configuration[section] = self.default(section)

            self.get_bool(section, 'stdout')
            self.get_bool(section, 'stdin')
            self.get_bool(section, 'stderr')
            self.get_bool(section, 'input')
            self.get_string(section, 'stdout_dir')
            self.get_string(section, 'stdin_dir')
            self.get_string(section, 'stderr_dir')
            self.get_string(section, 'input_dir')
            self.get_string(section, 'cmd_line')
            self.get_string(section, 'input_ext')
            self.get_string(section, 'stdin_ext')
            self.get_string(section, 'stdout_ext')
            self.get_string(section, 'stderr_ext')
            self.get_string(section, 'cmd_line')
            self.get_bool(section, 'check_code')
            self.get_string(section, 'error_code')
            self.get_bool(section, 'display_ok_tests')
            self.get_bool(section, 'diplay_ko_tests')
            self.get_bool(section, 'display_summary')

            if self.configuration[section]['error_code'] is str:
                self.configuration[section]['error_code'] = self.configuration[section]['error_code'].split("|")

def parse_argv():
    global testme_args
    argc = 1

    if len (sys.argv) == 1:
        return # No option to parse

    parser = argparse.ArgumentParser(description='TestMe ' + testme_version
                                     + ' by Baptiste COVOLATO')

    parser.add_argument('--dir', action="store",
                        help='The directory where you want to run testme')
    parser.add_argument('-v', '--verbose', action="store_true",
                        help='TestMe will display extra informations')
    parser.add_argument('-c', '--category', nargs='+', action="store",
                        help='Select specific categories to run')
    parser.add_argument('-x', '--extra-light-display', action="store_true",
                        help='Only print summary of each run category')
    parser.add_argument('-f', '--full-display', action="store_true",
                        help='Print all tests result')
    parser.add_argument('-l', '--light-display', action="store_true",
                        help='Print only failed tests')

    testme_args = vars(parser.parse_args())

def main():
    printer = TestPrinter()
    parse_argv()

    printer.verbose = testme_args.get('verbose', None)

    if testme_args.get('dir', None) == None:
        testme_args['dir'] = os.getcwd()

    printer.print_verbose("[TESTME] Running directory : " + testme_args['dir'])
    config_full_path = os.path.join(testme_args['dir'], testme_config_name)

    config = ConfigBuilder(printer, config_full_path)
    config.parse_config()

    test_suit = TestSuit(printer)
    test_suit.environement['TESTME_TESTDIR'] = testme_args['dir']
    test_suit.test_list = config.configuration
    test_suit.run()

if __name__ == "__main__":
    main()
<commit_msg>[TESTSUITE] Add threads<commit_after>#!env /usr/bin/python3

#Copyright (C) 2013 Baptiste COVOLATO <b.covolato@gmail.com>
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of
#this software and associated documentation files (the "Software"), to deal in
#the Software without restriction, including without limitation the rights to
#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
#of the Software, and to permit persons to whom the Software is furnished to do
#so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#SOFTWARE.

import sys
import subprocess
import os
import re
import configparser
import filecmp
import argparse
import logging
from threading import Thread
from queue import Queue

testme_version = "0.2"
testme_config_name = "testme.conf"
testme_args = {'threads' : 1}

class ThreadPool:

    class _ThreadQueue(Thread):
        def __init__(self, pool, *args, **kwargs):
            super(ThreadPool._ThreadQueue, self).__init__(*args, **kwargs)
            self.tasks = pool.tasks
            self.daemon = True
            self.start()

        def run(self):
            while True:
                task,args = self.tasks.get(True)
                try:
                    task(*args)
                except Exception:
                    pass
                finally:
                    self.tasks.task_done()

    def __init__(self, num=10):
        self.tasks = Queue(num)
        for _ in range(num):
            self._ThreadQueue(self)

    def add_task(self, target, args):
        self.tasks.put((target, args))

    def wait_completion(self):
        self.tasks.join()

class TestPrinter:
    def __init__(self):
        self.verbose = False
        self.summary_on = True

    def print_summary(self, category, good, total):
        if self.summary_on:
            print("[TESTME] Category : '" + category + "' :", good, "/", total)

    def print_verbose(self, message):
        if self.verbose:
            print(message)

    def print_error(self, message):
        print("\033[91m" + message + "\033[0m")

    def print_result(self, test_result, file_name):
        if (test_result):
            print("\033[32m[TESTME] Test :", file_name, "passed\033[0m")
        elif (not test_result):
            print("\033[91m[TESTME] Test :", file_name, "failed\033[0m")

class TestSuit:
    def __init__(self, printer):
        self.environement = {}
        self.test_list = {}
        self.running_cat = ""
        self.running_dir = ""
        self.ext = ""
        self.total_test = 0
        self.total_good = 0
        self.cat_test = 0
        self.cat_good = 0
        self.printer = printer

    def expand(self, string):
        while 1:
            m = re.search('\$\{(.*?)\}', string)

            if m == None or self.environement.get(m.group(1), None) == None:
                break

            string = re.sub('\$\{' + m.group(1) + '\}', self.environement[m.group(1)], string)

        return string

    def cat_field_get(self, field):
        if isinstance(self.test_list[self.running_cat][field], str):
            return self.expand(self.test_list[self.running_cat][field])
        else:
            return self.test_list[self.running_cat][field]

    def is_runnable(self):
        if testme_args.get('category', None) != None:
            return testme_args['category'].count(self.running_cat) > 0

        return True

    def change_extension(self, change_file, ext):
        path = os.path.splitext(change_file)
        return path[0] + "." + ext

    def stdin_input(self, test_name):
        stdin_val = ""

        if self.cat_field_get('stdin'):
            stdin_file = os.path.join(self.cat_field_get('stdin_dir'), test_name)
            stdin_file = self.change_extension(stdin_file, self.cat_field_get('stdin_ext'))
            stdin_fd = open(stdin_file, "r")
            stdin_val = stdin_fd.read()
            stdin_fd.close()

        return stdin_val

    def compare_out(self, output, value, ret, test_file):

        if self.cat_field_get(output):
            filename = self.change_extension(test_file, self.cat_field_get(output + '_ext'))
            filename = os.path.join(self.cat_field_get(output + '_dir'), filename)

            if os.path.exists(filename):
                fd = open(filename, "r")
                file_content = fd.read()
                fd.close()

                ret = ret and (file_content == value)
            else:
                print_verbose("\033[33m[TESTME] " + filename + " not present " + output + " ignored\033[0m")

        return ret

    def run_test(self, test_file):
        self.total_test += 1
        self.cat_test += 1

        ret_value = True

        if self.cat_field_get('input'):
            input_file = os.path.join(self.cat_field_get('input_dir'), test_file)
            self.environement['TESTME_RUNNING_INPUT'] = input_file

        stdinput = self.stdin_input(test_file)

        command = self.cat_field_get('cmd_line')

        process = subprocess.Popen(command, stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   shell=True)

        stdout, stderr = process.communicate(stdinput.encode('utf-8'))
        stdout, stderr = stdout.decode('utf-8'), stderr.decode('utf-8')

        ret_value = self.compare_out('stdout', stdout, ret_value, test_file)
        ret_value = self.compare_out('stderr', stderr, ret_value, test_file)

        if self.cat_field_get('check_code'):
            ret_value &= str(process.returncode) in self.cat_field_get('error_code')

        if ret_value:
            self.cat_good += 1
            self.total_good += 1
        self.printer.print_result(ret_value, test_file)

    def thread_run(self, dir_file):
        if dir_file.endswith(self.cat_field_get(self.ext)):
            self.run_test(dir_file)

    def run_directory(self):
        global testme_args
        try:
            pool = ThreadPool(testme_args['threads'])
            for dir_file in os.listdir(self.cat_field_get(self.running_dir)):
                pool.add_task(target=self.run_test, args=(dir_file,))

            pool.wait_completion()
            self.printer.print_summary(self.running_cat, self.cat_good,
                                       self.cat_test)

        except OSError:
           self.printer.print_error("[TESTME] Fatal error with " +
                                    "directory : " +
                                    self.cat_field_get(self.running_dir))

    def run(self):
        self.total_test = 0
        self.total_good = 0

        # Run every categories
        for cat in self.test_list:
            self.cat_test = 0
            self.cat_good = 0
            self.running_cat = cat
            self.environement['TESTME_CATEGORY'] = cat
            self.environement['TESTME_CATEGORY_DIR'] = os.path.join(self.environement['TESTME_TESTDIR'], cat)

            if not self.is_runnable():
                continue

            print("\n[TESTME] Running tests in", self.running_cat)
            self.running_dir = 'input_dir' if self.cat_field_get('input') else 'stdin_dir'
            self.ext = 'input_ext' if self.cat_field_get('input') else 'stdin_ext'

            self.run_directory()

        print("\n[TESTME] Final result :", self.total_good, "/", self.total_test)

class ConfigBuilder:
    def __init__(self, printer, filename):
        self.filename = filename
        self.configuration = {}
        self.printer = printer

    def default(self, cat_name):
        return {'stdout' : 0, # Stdout test
                'stdin' : 0, # Stdin test
                'stderr' : 0, # Stderr test
                'input' : 0, # Input test
                'input_dir' : os.path.join(testme_args['dir'], cat_name, "input"),
                'stdout_dir' : os.path.join(testme_args['dir'], cat_name, "stdout"),
                'stdin_dir' : os.path.join(testme_args['dir'], cat_name, "stdin"),
                'stderr_dir' : os.path.join(testme_args['dir'], cat_name, "stderr"),
                'input_ext' : 'inp',
                'stdout_ext' : 'out',
                'stdin_ext' : 'in',
                'stderr_ext' : 'err',
                'cmd_line' : '',
                'check_code' : 0,
                'error_code' : ["0"],
                'display_ok_tests' : 1,
                'display_ko_tests' : 1,
                'display_summary' : 1}

    def get_bool(self, section, value):
        try:
            self.configuration[section][value] = self.parser.getboolean(section,
                                                                        value)
        except configparser.NoOptionError:
            return 0
        except:
            die_on_error(value + " in section '" + section + "' is not a boolean")

    def get_string(self, section, value):
        try:
            self.configuration[section][value] = self.parser.get(section, value)
            return
        except configparser.NoOptionError:
            return

    def get_int(self, section, value):
        try:
            self.configuration[section][value] = self.parser.getint(section, value)
        except configparser.NoOptionError:
            return 0
        except:
            die_on_error(value + " in section '" + section + "' is not an integer")

    def parse_config(self):
        self.parser = configparser.RawConfigParser()
        self.parser.read(self.filename)

        for section in self.parser.sections():
            self.configuration[section] = self.default(section)

            self.get_bool(section, 'stdout')
            self.get_bool(section, 'stdin')
            self.get_bool(section, 'stderr')
            self.get_bool(section, 'input')
            self.get_string(section, 'stdout_dir')
            self.get_string(section, 'stdin_dir')
            self.get_string(section, 'stderr_dir')
            self.get_string(section, 'input_dir')
            self.get_string(section, 'cmd_line')
            self.get_string(section, 'input_ext')
            self.get_string(section, 'stdin_ext')
            self.get_string(section, 'stdout_ext')
            self.get_string(section, 'stderr_ext')
            self.get_string(section, 'cmd_line')
            self.get_bool(section, 'check_code')
            self.get_string(section, 'error_code')
            self.get_bool(section, 'display_ok_tests')
            self.get_bool(section, 'diplay_ko_tests')
            self.get_bool(section, 'display_summary')

            if self.configuration[section]['error_code'] is str:
                self.configuration[section]['error_code'] = self.configuration[section]['error_code'].split("|")

def parse_argv():
    global testme_args
    argc = 1

    if len (sys.argv) == 1:
        return # No option to parse

    parser = argparse.ArgumentParser(description='TestMe ' + testme_version
                                     + ' by Baptiste COVOLATO')

    parser.add_argument('--dir', action="store",
                        help='The directory where you want to run testme')
    parser.add_argument('--threads', action="store", type=int,
                        help='Indicate the number of thread you want to use')
    parser.add_argument('-v', '--verbose', action="store_true",
                        help='TestMe will display extra informations')
    parser.add_argument('-c', '--category', nargs='+', action="store",
                        help='Select specific categories to run')
    parser.add_argument('-x', '--extra-light-display', action="store_true",
                        help='Only print summary of each run category')
    parser.add_argument('-f', '--full-display', action="store_true",
                        help='Print all tests result')
    parser.add_argument('-l', '--light-display', action="store_true",
                        help='Print only failed tests')

    testme_args = vars(parser.parse_args())

def main():
    printer = TestPrinter()
    parse_argv()

    printer.verbose = testme_args.get('verbose', None)

    if testme_args.get('dir', None) == None:
        testme_args['dir'] = os.getcwd()

    printer.print_verbose("[TESTME] Running directory : " + testme_args['dir'])
    config_full_path = os.path.join(testme_args['dir'], testme_config_name)

    config = ConfigBuilder(printer, config_full_path)
    config.parse_config()

    test_suit = TestSuit(printer)
    test_suit.environement['TESTME_TESTDIR'] = testme_args['dir']
    test_suit.test_list = config.configuration
    test_suit.run()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from .models import NeuropowerModel
import requests
import os
os.environ['http_proxy']=''
import urllib
from django.conf import settings
from django.http import HttpResponseRedirect

def get_neuropower_steps(template_page,sid,pi1=None):

    '''get_neuropower_steps returns a complete dictionary object with button colors, active status, and  currently active based on a template page and session data object.
    '''

    # template name, step class, and color
    pages = {"neuropower/neuropowerstart.html": {"class":"overview","color":"#F7F7F7","enabled":"yes"},
             "neuropower/neuropowerinput.html": {"class":"start","color":"#F7941E","enabled":"yes"},
             "neuropower/neuropowerviewer.html": {"class":"viewer","color":"#FFDE16","enabled":"yes"},
             "neuropower/neuropowertable.html": {"class":"peak-table","color":"#D7DF21","enabled":"yes"},
             "neuropower/neuropowermodel.html": {"class":"fit","color":"#0BA14B","enabled":"yes"},
             "neuropower/neuropowersamplesize.html": {"class":"power-calculation","color":"#25AAE1","enabled":"yes"},
             "neuropower/neuropowercrosstab.html": {"class":"power-table","color":"#25AAE1","enabled":"yes"}}

    # Set enabled or disabled page status depending on session data
    if not NeuropowerModel.objects.filter(SID=sid):
        pages["neuropower/neuropowerviewer.html"]["enabled"] = "no"
        pages["neuropower/neuropowertable.html"]["enabled"] = "no"
        pages["neuropower/neuropowermodel.html"]["enabled"] = "no"
        pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
        pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

    else:
        neuropowerdata = NeuropowerModel.objects.get(SID=sid)

        if neuropowerdata.step == 1:
            pages["neuropower/neuropowermodel.html"]["enabled"] = "no"
            pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

        if neuropowerdata.step == 2:
            pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

        if neuropowerdata.step == 3:
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

        if pi1==0:
            pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

    # Set the active page
    pages["active"] = pages[template_page]
    return pages

def get_db_entries(template_page,sid):

    '''get_db_entries checks which db models have entries, and can as such redirect to right page if a certain model is missing.
    '''
    # template name, step class, and color
    relink = {"pm":"neuropowerinput",
            "ptm":"neuropowertable",
            "mm":"neuropowermodel",
            "powerm":"neuropowersamplesize"}
    message = {"pm":"Your session has expired and we can't find the description of the data.  Please fill out this form again to access your power analysis.",
            "ptm":"Something went wrong and we couldn't find the table of extracted peaks, so we've sent you back to this page.  Don't worry, we re-calculated the table here,so you can continue your power analysis now.",
            "mm":"Something went wrong and we couldn't find the estimated mixture distribution for your power analysis.  Don't worry, we re-estimated the model here, so you can continue your power analysis now.",
            "powerm":"Something went wrong and we couldn't find the power estimates for your power analysis.  Don't worry, we re-estimated the power curves here, so you can continue your power analysis now."
            }

    err = ""

    if template_page == "neuropower/neuropowerviewer.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"

    elif template_page == "neuropower/neuropowertable.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"

    elif template_page == "neuropower/neuropowermodel.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"
        else:
            neuropowerdata = NeuropowerModel.objects.get(SID=sid)
            if neuropowerdata.step == 1:
                err = "ptm"

    elif template_page == "neuropower/neuropowersamplesize.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"
        else:
            neuropowerdata = NeuropowerModel.objects.get(SID=sid)
            if neuropowerdata.step == 1:
                err = "ptm"
            elif neuropowerdata.step == 2:
                err = "mm"

    elif template_page == "neuropower/neuropowercrosstab.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"
        else:
            neuropowerdata = NeuropowerModel.objects.get(SID=sid)
            if neuropowerdata.step == 1:
                err = "ptm"
            elif neuropowerdata.step == 2:
                err = "mm"
            elif neuropowerdata.step == 3:
                err = "powerm"

    if not err == "":
        link = "http://www.neuropowertools.org/"+relink[err]+"/?message="+message[err]
    else:
        link = ""

    return link

def get_url(url,return_json=True):
    '''get_url uses requests to return a url. Default return format is json. If return_json is set to false, will return raw data
    :param url: the url to retrieve
    :param return_json: boolean, default True, to return json
    '''
    response = requests.get(url)
    if return_json == True:
        return response.json()
    else:
        return response.text

def get_session_id(request):
    '''get_session_id gets the user session id, and creates one if it doesn't exist'''
    if not request.session.exists(request.session.session_key):
        request.session.create()
    sid = request.session.session_key
    return(sid)

def create_local_copy(map_url,map_local):
    if map_url.split(".")[-1] == "gz":
        map_local = map_local+".nii.gz"
    else:
        map_local = map_local+".nii"
    urllib.urlretrieve(map_url,map_local)
    # response = requests.get(map_url,stream=True)
    # if response.status_code == 200:
    #     with open(map_local,"wb") as f:
    #         f.write(response.raw.read())

    return map_local

def get_neurovault_form(neurovault_id):
    neurovault_image = get_url("http://neurovault.org/api/images/%s/?format=json" %(neurovault_id))
    collection_id = str(neurovault_image['collection_id'])

    message = None
    if not (neurovault_image['map_type'] == 'Z map' or neurovault_image['map_type'] == 'T map' or neurovault_image['analysis_level']==None):
        message = "Power analyses can only be performed on Z or T maps."
    if not (neurovault_image['analysis_level'] == 'group' or neurovault_image['analysis_level']==None):
        message = "Power analyses can only be performed on group statistical maps."

    parsform = ParameterForm(request.POST or None,
                             request.FILES or None,
                             default_url = "",
                             err = '',
                             initial = {"url":neurovault_image["file"],
                                        "ZorT":"T" if neurovault_image["map_type"] =="T map" else "Z",
                                        "Subj":neurovault_image["number_of_subjects"]})

    out = {
        "parsform": parsform,
        "message": message
    }

    return out
<commit_msg>remove bug when importing from neurovault<commit_after>from .models import NeuropowerModel
from .forms import ParameterForm, PeakTableForm, MixtureForm, PowerTableForm, PowerForm
import requests
import os
os.environ['http_proxy']=''
import urllib
from django.conf import settings
from django.http import HttpResponseRedirect

def get_neuropower_steps(template_page,sid,pi1=None):

    '''get_neuropower_steps returns a complete dictionary object with button colors, active status, and  currently active based on a template page and session data object.
    '''

    # template name, step class, and color
    pages = {"neuropower/neuropowerstart.html": {"class":"overview","color":"#F7F7F7","enabled":"yes"},
             "neuropower/neuropowerinput.html": {"class":"start","color":"#F7941E","enabled":"yes"},
             "neuropower/neuropowerviewer.html": {"class":"viewer","color":"#FFDE16","enabled":"yes"},
             "neuropower/neuropowertable.html": {"class":"peak-table","color":"#D7DF21","enabled":"yes"},
             "neuropower/neuropowermodel.html": {"class":"fit","color":"#0BA14B","enabled":"yes"},
             "neuropower/neuropowersamplesize.html": {"class":"power-calculation","color":"#25AAE1","enabled":"yes"},
             "neuropower/neuropowercrosstab.html": {"class":"power-table","color":"#25AAE1","enabled":"yes"}}

    # Set enabled or disabled page status depending on session data
    if not NeuropowerModel.objects.filter(SID=sid):
        pages["neuropower/neuropowerviewer.html"]["enabled"] = "no"
        pages["neuropower/neuropowertable.html"]["enabled"] = "no"
        pages["neuropower/neuropowermodel.html"]["enabled"] = "no"
        pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
        pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

    else:
        neuropowerdata = NeuropowerModel.objects.get(SID=sid)

        if neuropowerdata.step == 1:
            pages["neuropower/neuropowermodel.html"]["enabled"] = "no"
            pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

        if neuropowerdata.step == 2:
            pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

        if neuropowerdata.step == 3:
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

        if pi1==0:
            pages["neuropower/neuropowersamplesize.html"]["enabled"] = "no"
            pages["neuropower/neuropowercrosstab.html"]["enabled"] = "no"

    # Set the active page
    pages["active"] = pages[template_page]
    return pages

def get_db_entries(template_page,sid):

    '''get_db_entries checks which db models have entries, and can as such redirect to right page if a certain model is missing.
    '''
    # template name, step class, and color
    relink = {"pm":"neuropowerinput",
            "ptm":"neuropowertable",
            "mm":"neuropowermodel",
            "powerm":"neuropowersamplesize"}
    message = {"pm":"Your session has expired and we can't find the description of the data.  Please fill out this form again to access your power analysis.",
            "ptm":"Something went wrong and we couldn't find the table of extracted peaks, so we've sent you back to this page.  Don't worry, we re-calculated the table here,so you can continue your power analysis now.",
            "mm":"Something went wrong and we couldn't find the estimated mixture distribution for your power analysis.  Don't worry, we re-estimated the model here, so you can continue your power analysis now.",
            "powerm":"Something went wrong and we couldn't find the power estimates for your power analysis.  Don't worry, we re-estimated the power curves here, so you can continue your power analysis now."
            }

    err = ""

    if template_page == "neuropower/neuropowerviewer.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"

    elif template_page == "neuropower/neuropowertable.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"

    elif template_page == "neuropower/neuropowermodel.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"
        else:
            neuropowerdata = NeuropowerModel.objects.get(SID=sid)
            if neuropowerdata.step == 1:
                err = "ptm"

    elif template_page == "neuropower/neuropowersamplesize.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"
        else:
            neuropowerdata = NeuropowerModel.objects.get(SID=sid)
            if neuropowerdata.step == 1:
                err = "ptm"
            elif neuropowerdata.step == 2:
                err = "mm"

    elif template_page == "neuropower/neuropowercrosstab.html":
        if not NeuropowerModel.objects.filter(SID=sid):
            err = "pm"
        else:
            neuropowerdata = NeuropowerModel.objects.get(SID=sid)
            if neuropowerdata.step == 1:
                err = "ptm"
            elif neuropowerdata.step == 2:
                err = "mm"
            elif neuropowerdata.step == 3:
                err = "powerm"

    if not err == "":
        link = "http://www.neuropowertools.org/"+relink[err]+"/?message="+message[err]
    else:
        link = ""

    return link

def get_url(url,return_json=True):
    '''get_url uses requests to return a url. Default return format is json. If return_json is set to false, will return raw data
    :param url: the url to retrieve
    :param return_json: boolean, default True, to return json
    '''
    response = requests.get(url)
    if return_json == True:
        return response.json()
    else:
        return response.text

def get_session_id(request):
    '''get_session_id gets the user session id, and creates one if it doesn't exist'''
    if not request.session.exists(request.session.session_key):
        request.session.create()
    sid = request.session.session_key
    return(sid)

def create_local_copy(map_url,map_local):
    if map_url.split(".")[-1] == "gz":
        map_local = map_local+".nii.gz"
    else:
        map_local = map_local+".nii"
    urllib.urlretrieve(map_url,map_local)
    # response = requests.get(map_url,stream=True)
    # if response.status_code == 200:
    #     with open(map_local,"wb") as f:
    #         f.write(response.raw.read())

    return map_local

def get_neurovault_form(neurovault_id):
    neurovault_image = get_url("http://neurovault.org/api/images/%s/?format=json" %(neurovault_id))
    collection_id = str(neurovault_image['collection_id'])

    message = None
    if not (neurovault_image['map_type'] == 'Z map' or neurovault_image['map_type'] == 'T map' or neurovault_image['analysis_level']==None):
        message = "Power analyses can only be performed on Z or T maps."
    if not (neurovault_image['analysis_level'] == 'group' or neurovault_image['analysis_level']==None):
        message = "Power analyses can only be performed on group statistical maps."

    parsform = ParameterForm(request.POST or None,
                             request.FILES or None,
                             default_url = "",
                             err = '',
                             initial = {"url":neurovault_image["file"],
                                        "ZorT":"T" if neurovault_image["map_type"] =="T map" else "Z",
                                        "Subj":neurovault_image["number_of_subjects"]})

    out = {
        "parsform": parsform,
        "message": message
    }

    return out
<|endoftext|>


<commit_before>"""
Dropbox API Storage object.
"""
import tempfile, copy
from .base import BaseStorage, StorageError
from ConfigParser import ConfigParser
from django.conf import settings
from dropbox.auth import Authenticator
from dropbox.client import DropboxClient
from oauth.oauth import OAuthToken

DEFAULT_CONFIG = {
    'consumer_key': None,
    'consumer_secret': None,
    'verifier': '',
    'server': 'api.dropbox.com',
    'content_server': 'api-content.dropbox.com',
    'port': 80,
    'request_token_url': 'https://api.dropbox.com/0/oauth/request_token',
    'access_token_url': 'https://api.dropbox.com/0/oauth/access_token',
    'authorization_url': 'https://www.dropbox.com/0/oauth/authorize',
    'root': 'dropbox',
}

################################
#  Dropbox Storage Object
################################

class Storage(BaseStorage):
    """ Dropbox API Storage. """
    TOKENS_FILEPATH = getattr(settings, 'DBBACKUP_TOKENS_FILEPATH', None)
    DROPBOX_DIRECTORY = getattr(settings, 'DBBACKUP_DROPBOX_DIRECTORY', "/django-dbbackups/")
    DROPBOX_DIRECTORY = '/%s/' % DROPBOX_DIRECTORY.strip('/')
    DROPBOX_CONFIG = copy.copy(DEFAULT_CONFIG)
    DROPBOX_CONFIG.update(getattr(settings, 'DBBACKUP_DROPBOX_CONFIG', {}))
    REQUEST_KEY = 'request_key'
    ACCESS_KEY = 'access_key'
    _request_token = None
    _access_token = None

    def __init__(self, server_name=None):
        self._check_dropbox_errors()
        self.name = 'Dropbox'
        self.dropbox = self.get_dropbox_client()
        BaseStorage.__init__(self)

    def _check_dropbox_errors(self):
        """ Check we have all the required settings defined. """
        if not self.TOKENS_FILEPATH:
            raise StorageError('Dropbox storage requires DBBACKUP_TOKENS_FILEPATH to be defined in settings.')
        if not self.DROPBOX_CONFIG:
            raise StorageError('%s storage requires DBBACKUP_DROPBOX_CONFIG to be defined in settings.' % self.name)
        if not self.DROPBOX_CONFIG.get('consumer_key'):
            raise StorageError('%s storage requires DBBACKUP_DROPBOX_CONFIG["consumer_key"] to be specified.' % self.name)
        if not self.DROPBOX_CONFIG.get('consumer_secret'):
            raise StorageError('%s storage requires DBBACKUP_DROPBOX_CONFIG["consumer_secret"] to be specified.' % self.name)

    ###################################
    #  DBBackup Storage Methods
    ###################################

    def backup_dir(self):
        return self.DROPBOX_DIRECTORY

    def delete_file(self, filepath):
        """ Delete the specified filepath. """
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.file_delete(root, filepath)
        self._check_response(response)

    def list_directory(self):
        """ List all stored backups for the specified. """
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.metadata(root, self.DROPBOX_DIRECTORY)
        self._check_response(response)
        filepaths = [x['path'] for x in response.data['contents'] if not x['is_dir']]
        return sorted(filepaths)

    def write_file(self, filehandle):
        """ Write the specified file. """
        filehandle.seek(0)
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.put_file(root, self.DROPBOX_DIRECTORY, filehandle)
        self._check_response(response)

    def read_file(self, filepath):
        """ Read the specified file and return it's handle. """
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.get_file(root, filepath)
        self._check_response(response)
        filehandle = tempfile.SpooledTemporaryFile()
        filehandle.write(response.read())
        return filehandle

    def _check_response(self, response):
        """ Check we have a valid 200 response from Dropbox. """
        http_response = getattr(response, 'http_response', response)
        if http_response.status != 200:
            errmsg = "ERROR [%s]: %s" % (response.http_response.status, response.reason)
            raise StorageError(errmsg)

    ###################################
    #  oAuth Client Methods
    ###################################

    def get_dropbox_client(self):
        """ Connect and return a Dropbox client object. """
        self.read_token_file()
        auth = Authenticator(self.DROPBOX_CONFIG)
        access_token = self.get_access_token(auth)
        oauth = OAuthToken.from_string(access_token)
        server = self.DROPBOX_CONFIG['server']
        content_server = self.DROPBOX_CONFIG['content_server']
        port = self.DROPBOX_CONFIG['port']
        dropbox = DropboxClient(server, content_server, port, auth, oauth)
        # Test the connection
        if dropbox.account_info().http_response.status != 200:
            self.create_request_token(auth)
        return dropbox

    def read_token_file(self):
        """ Reload the config from disk. """
        tokenfile = ConfigParser()
        tokenfile.read(self.TOKENS_FILEPATH)
        if tokenfile.has_section(self.REQUEST_KEY) and tokenfile.has_option(self.REQUEST_KEY, 'token'):
            self._request_token = tokenfile.get(self.REQUEST_KEY, 'token')
        if tokenfile.has_section(self.ACCESS_KEY) and tokenfile.has_option(self.ACCESS_KEY, 'token'):
            self._access_token = tokenfile.get(self.ACCESS_KEY, 'token')

    def get_request_token(self, auth):
        """ Return Request oAuthToken. If not available, a new one will be created, saved
            and a RequestUrl object will be returned.
        """
        if not self._request_token:
            return self.create_request_token(auth)
        return self._request_token

    def get_access_token(self, auth):
        """ Return Access oAuthToken. If not available, a new one will be created and saved. """
        if not self._access_token:
            request_token = self.get_request_token(auth)
            return self.create_access_token(auth, request_token)
        return self._access_token

    def create_request_token(self, auth):
        """ Create and save a new request token. """
        self._request_token = auth.obtain_request_token()
        self._access_token = None
        self.save_token_file()
        # If we hit this code, we know were not authorized
        message = "Dropbox not authorized, visit the following URL to authorize:\n"
        message += auth.build_authorize_url(self._request_token)
        raise StorageError(message)

    def create_access_token(self, auth, request_token):
        """ Create and save a new access token to self.filepath. """
        try:
            oauth = OAuthToken.from_string(request_token)
            verifier = self.DROPBOX_CONFIG['verifier']
            self._access_token = auth.obtain_access_token(oauth, verifier).to_string()
            self.save_token_file()
            return self._access_token
        except AssertionError, e:
            # Check we have a bad request token
            if '403' in e.args[0]:
                self.create_request_token(Authenticator(self.config))

    def save_token_file(self):
        """ Save the config to disk. """
        tokenfile = ConfigParser()
        if self._request_token:
            tokenfile.add_section(self.REQUEST_KEY)
            tokenfile.set(self.REQUEST_KEY, 'token', self._request_token)
        if self._access_token:
            tokenfile.add_section(self.ACCESS_KEY)
            tokenfile.set(self.ACCESS_KEY, 'token', self._access_token)
        with open(self.TOKENS_FILEPATH, 'wb') as tokenhandle:
            tokenfile.write(tokenhandle)
<commit_msg>Fix name error<commit_after>"""
Dropbox API Storage object.
"""
import tempfile, copy
from .base import BaseStorage, StorageError
from ConfigParser import ConfigParser
from django.conf import settings
from dropbox.auth import Authenticator
from dropbox.client import DropboxClient
from oauth.oauth import OAuthToken

DEFAULT_CONFIG = {
    'consumer_key': None,
    'consumer_secret': None,
    'verifier': '',
    'server': 'api.dropbox.com',
    'content_server': 'api-content.dropbox.com',
    'port': 80,
    'request_token_url': 'https://api.dropbox.com/0/oauth/request_token',
    'access_token_url': 'https://api.dropbox.com/0/oauth/access_token',
    'authorization_url': 'https://www.dropbox.com/0/oauth/authorize',
    'root': 'dropbox',
}

################################
#  Dropbox Storage Object
################################

class Storage(BaseStorage):
    """ Dropbox API Storage. """
    TOKENS_FILEPATH = getattr(settings, 'DBBACKUP_TOKENS_FILEPATH', None)
    DROPBOX_DIRECTORY = getattr(settings, 'DBBACKUP_DROPBOX_DIRECTORY', "/django-dbbackups/")
    DROPBOX_DIRECTORY = '/%s/' % DROPBOX_DIRECTORY.strip('/')
    DROPBOX_CONFIG = copy.copy(DEFAULT_CONFIG)
    DROPBOX_CONFIG.update(getattr(settings, 'DBBACKUP_DROPBOX_CONFIG', {}))
    REQUEST_KEY = 'request_key'
    ACCESS_KEY = 'access_key'
    _request_token = None
    _access_token = None

    def __init__(self, server_name=None):
        self._check_dropbox_errors()
        self.name = 'Dropbox'
        self.dropbox = self.get_dropbox_client()
        BaseStorage.__init__(self)

    def _check_dropbox_errors(self):
        """ Check we have all the required settings defined. """
        if not self.TOKENS_FILEPATH:
            raise StorageError('Dropbox storage requires DBBACKUP_TOKENS_FILEPATH to be defined in settings.')
        if not self.DROPBOX_CONFIG:
            raise StorageError('%s storage requires DBBACKUP_DROPBOX_CONFIG to be defined in settings.' % self.name)
        if not self.DROPBOX_CONFIG.get('consumer_key'):
            raise StorageError('%s storage requires DBBACKUP_DROPBOX_CONFIG["consumer_key"] to be specified.' % self.name)
        if not self.DROPBOX_CONFIG.get('consumer_secret'):
            raise StorageError('%s storage requires DBBACKUP_DROPBOX_CONFIG["consumer_secret"] to be specified.' % self.name)

    ###################################
    #  DBBackup Storage Methods
    ###################################

    def backup_dir(self):
        return self.DROPBOX_DIRECTORY

    def delete_file(self, filepath):
        """ Delete the specified filepath. """
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.file_delete(root, filepath)
        self._check_response(response)

    def list_directory(self):
        """ List all stored backups for the specified. """
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.metadata(root, self.DROPBOX_DIRECTORY)
        self._check_response(response)
        filepaths = [x['path'] for x in response.data['contents'] if not x['is_dir']]
        return sorted(filepaths)

    def write_file(self, filehandle):
        """ Write the specified file. """
        filehandle.seek(0)
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.put_file(root, self.DROPBOX_DIRECTORY, filehandle)
        self._check_response(response)

    def read_file(self, filepath):
        """ Read the specified file and return it's handle. """
        root = self.DROPBOX_CONFIG['root']
        response = self.dropbox.get_file(root, filepath)
        self._check_response(response)
        filehandle = tempfile.SpooledTemporaryFile()
        filehandle.write(response.read())
        return filehandle

    def _check_response(self, response):
        """ Check we have a valid 200 response from Dropbox. """
        http_response = getattr(response, 'http_response', response)
        if http_response.status != 200:
            errmsg = "ERROR [%s]: %s" % (response.http_response.status, response.reason)
            raise StorageError(errmsg)

    ###################################
    #  oAuth Client Methods
    ###################################

    def get_dropbox_client(self):
        """ Connect and return a Dropbox client object. """
        self.read_token_file()
        auth = Authenticator(self.DROPBOX_CONFIG)
        access_token = self.get_access_token(auth)
        oauth = OAuthToken.from_string(access_token)
        server = self.DROPBOX_CONFIG['server']
        content_server = self.DROPBOX_CONFIG['content_server']
        port = self.DROPBOX_CONFIG['port']
        dropbox = DropboxClient(server, content_server, port, auth, oauth)
        # Test the connection
        if dropbox.account_info().http_response.status != 200:
            self.create_request_token(auth)
        return dropbox

    def read_token_file(self):
        """ Reload the config from disk. """
        tokenfile = ConfigParser()
        tokenfile.read(self.TOKENS_FILEPATH)
        if tokenfile.has_section(self.REQUEST_KEY) and tokenfile.has_option(self.REQUEST_KEY, 'token'):
            self._request_token = tokenfile.get(self.REQUEST_KEY, 'token')
        if tokenfile.has_section(self.ACCESS_KEY) and tokenfile.has_option(self.ACCESS_KEY, 'token'):
            self._access_token = tokenfile.get(self.ACCESS_KEY, 'token')

    def get_request_token(self, auth):
        """ Return Request oAuthToken. If not available, a new one will be created, saved
            and a RequestUrl object will be returned.
        """
        if not self._request_token:
            return self.create_request_token(auth)
        return self._request_token

    def get_access_token(self, auth):
        """ Return Access oAuthToken. If not available, a new one will be created and saved. """
        if not self._access_token:
            request_token = self.get_request_token(auth)
            return self.create_access_token(auth, request_token)
        return self._access_token

    def create_request_token(self, auth):
        """ Create and save a new request token. """
        self._request_token = auth.obtain_request_token()
        self._access_token = None
        self.save_token_file()
        # If we hit this code, we know were not authorized
        message = "Dropbox not authorized, visit the following URL to authorize:\n"
        message += auth.build_authorize_url(self._request_token)
        raise StorageError(message)

    def create_access_token(self, auth, request_token):
        """ Create and save a new access token to self.filepath. """
        try:
            oauth = OAuthToken.from_string(request_token)
            verifier = self.DROPBOX_CONFIG['verifier']
            self._access_token = auth.obtain_access_token(oauth, verifier).to_string()
            self.save_token_file()
            return self._access_token
        except AssertionError, e:
            # Check we have a bad request token
            if '403' in e.args[0]:
                self.create_request_token(Authenticator(self.DROPBOX_CONFIG))

    def save_token_file(self):
        """ Save the config to disk. """
        tokenfile = ConfigParser()
        if self._request_token:
            tokenfile.add_section(self.REQUEST_KEY)
            tokenfile.set(self.REQUEST_KEY, 'token', self._request_token)
        if self._access_token:
            tokenfile.add_section(self.ACCESS_KEY)
            tokenfile.set(self.ACCESS_KEY, 'token', self._access_token)
        with open(self.TOKENS_FILEPATH, 'wb') as tokenhandle:
            tokenfile.write(tokenhandle)
<|endoftext|>