<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-07-2017 11:39
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Download manager and its helper agents
"""

import os
import time
import random
import threading
import subprocess

class Agent(threading.Thread):
    def __init__(self, url, dst_folder, download_attempts=32, timeout_attempts=3, download_timeout=600):
        super(Agent, self).__init__()
        self.__download_url = url
        self.__dst_folder = dst_folder
        self.__download_attempts = download_attempts
        self.__timeout_attempts = timeout_attempts
        self.__download_timeout = download_timeout
        # Compute destination file name, using the same file name as in the given URL
        self.__dst_filename = url[url.rfind("/") + 1:]
        # Prepare standard output and error output
        self.__stdout = b' '
        self.__stderr = b' '
        # Result object
        self.__result = {'msg': '', 'success': True, 'url': str(self.__download_url)}
        # Seed random module
        random.seed(time.time())
        # We have everything we need, auto-start the thread
        self.start()

    def _build_result(self, msg, success=True):
        """
        Result object builder.

        The result object will contain anything that happened during the process of downloading a file from the given
        URL, and whether it was successful or not.
        :param msg: message to add to the final result object
        :param success: whether this extra informatoin makes the process successful or not
        :return: no value is returned
        """
        self.__result['msg'] = self.__result['msg'] + "\n" + msg
        self.__result['success'] = self.__result['success'] and success

    def __download_with_timeout(self):
        """
        This is a helper method that will download the given URL setting a timeout limit.
        :return: True if success
        :except: a subprocess.TimeoutExpired exception is raised if the download can't be completed within the given
        temporal constraints
        """
        download_command = "cd " + str(self.get_dst_folder()) + "; curl -L -O -C - " + str(self.get_download_url())
        download_subprocess = subprocess.Popen(download_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                               shell=True)
        stdout = b' '
        stderr = b' '
        try:
            self._build_result("Downloading '{}' with timeout set to {} seconds"
                               .format(self.get_download_url(),
                                       self.get_download_timeout()))
            (stdout, stderr) = download_subprocess.communicate(timeout=self.get_download_timeout())
        except subprocess.TimeoutExpired as exception_download_timeout:
            self._build_result("Timeout ({} seconds) ERROR downloading '{}', STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                               .format(self.get_download_timeout(),
                                       self.get_download_url(),
                                       stdout.decode('utf8'),
                                       stderr.decode('utf8')))
            raise
        # Let's check the result from downloading the file
        if download_subprocess.poll():
            if download_subprocess.returncode != 0:
                # ERROR
                self._build_result("ERROR downloading '{}', STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                                   .format(self.get_download_timeout(),
                                           self.get_download_url(),
                                           stdout.decode('utf8'),
                                           stderr.decode('utf8')))
                return False
            # SUCCESS
            self._build_result("SUCCESSFUL download for '{}', STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                               .format(self.get_download_timeout(),
                                       self.get_download_url(),
                                       stdout.decode('utf8'),
                                       stderr.decode('utf8')))
        else:
            # TODO
            # The child process is still running, and this is weird stuff because we are using a timeout parameter, we
            # should never reach this part of the code
            self._build_result("WEIRD DOWNLOAD - THE SUBPROCESS IS STILL RUNNING FOR '{}' DESPITE THE TIME OUT "
                               "PARAMETER, STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                               .format(self.get_download_timeout(),
                                       self.get_download_url(),
                                       stdout.decode('utf8'),
                                       stderr.decode('utf8')))
            return False
        return True

    def __download_with_timeout_attempts(self):
        """
        Download the given URL given a time constraint with a limited number of attempts upon timeout errors.
        :return: True if success, False if we reached the maximum number of attempts
        """
        timeout_attempt_counter = 0
        while timeout_attempt_counter < self.get_timeout_attempts():
            self._build_result("Downloading '{}', timeout attempt #{} out of #{}"
                               .format(self.get_download_url(),
                                       timeout_attempt_counter,
                                       self.get_timeout_attempts()))
            timeout_attempt_counter += 1
            try:
                return self.__download_with_timeout()
            except subprocess.TimeoutExpired as exception_download_timeout:
                self._build_result("Download of '{}' TIMED OUT, timeout attempt #{} out of #{}"
                                   .format(self.get_download_url(),
                                           timeout_attempt_counter,
                                           self.get_timeout_attempts()))
                # wait for a random amount of time before retrying the download
                # WARNING! - MAGIC NUMBER AHEAD!!!
                time.sleep(random.randint(0, 60))
        return False

    def run(self):
        """
        This is the main thread for the download agent
        :return: no value is returned
        """
        # TODO - Validate URL
        attempt_counter = 0
        download_completion = False
        while attempt_counter < self.get_download_attempts():
            attempt_counter += 1
            self._build_result("Downloading '{}', download attempt #{} out of #{}"
                               .format(self.get_download_url(),
                                       attempt_counter,
                                       self.get_download_attempts()))
            try:
                download_completion = self.__download_with_timeout_attempts()
                if download_completion:
                    # Quit as soon as we succeed on downloading the file
                    break
            except Exception as e:
                self._build_result("ERROR downloading '{}', on download attempt #{} out of #{}, ERROR: {}"
                                   .format(self.get_download_url(),
                                           attempt_counter,
                                           self.get_download_attempts(),
                                           str(e)))
            if download_completion:
                self._build_result("Download for '{}' COMPLETED, on download attempt #{} out of #{}"
                                   .format(self.get_download_url(),
                                           attempt_counter,
                                           self.get_download_attempts()))
            else:
                self._build_result("Download for '{}' FAILED, on download attempt #{} out of #{}"
                                   .format(self.get_download_url(),
                                           attempt_counter,
                                           self.get_download_attempts()),
                                   False)

    def cancel(self):
        """
        I think this is the way to stop the thread, but I'm not sure, because the documentation about it is a little bit
        tricky, or I didn't find the right documentation.
        :return: no value is returned
        """
        self.alive = False
        self.join()

    def wait(self):
        """
        Wait for this thread to finish its job (download a file), and get the result object.
        :return: result object with information on the finished download process
        """
        self.join()
        return self.get_result()

    def get_result(self):
        """
        Get the result object built by this Agent.

        This method should be called when the agent finishes its job, but not in the middle of it.
        :return: result object with information on the finished download process
        """
        return self.__result

    def get_dst_folder(self):
        return self.__dst_folder

    def get_download_timeout(self):
        return self.__download_timeout

    def get_download_url(self):
        return self.__download_url

    def get_timeout_attempts(self):
        return self.__timeout_attempts

    def get_download_attempts(self):
        return self.__download_attempts

class Manager:
    def __init__(self, urls, download_destination_folder, logger, download_attempts=32, timeout_attempts=3, download_timeout=120):
        self.__urls = urls
        self.__download_destination_folder = download_destination_folder
        self.__logger = logger
        self.__download_attempts = download_attempts
        self.__timeout_attempts = timeout_attempts
        self.__download_timeout = download_timeout
        self.__agents = {}
        self.__success = True

    def start_downloads(self):
        # TODO
        pass

    def wait_all(self):
        # TODO
        pass

    def is_success(self):
        return self.__success

    def get_urls_to_download(self):
        return self.__urls

    def get_download_destination_folder(self):
        return self.__download_destination_folder

    def _get_logger(self):
        return self.__logger

    def get_download_attempts(self):
        return self.__download_attempts

    def get_timeout_attempts(self):
        return self.__timeout_attempts<commit_msg>download timeout getter<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-07-2017 11:39
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Download manager and its helper agents
"""

import os
import time
import random
import threading
import subprocess

class Agent(threading.Thread):
    def __init__(self, url, dst_folder, download_attempts=32, timeout_attempts=3, download_timeout=600):
        super(Agent, self).__init__()
        self.__download_url = url
        self.__dst_folder = dst_folder
        self.__download_attempts = download_attempts
        self.__timeout_attempts = timeout_attempts
        self.__download_timeout = download_timeout
        # Compute destination file name, using the same file name as in the given URL
        self.__dst_filename = url[url.rfind("/") + 1:]
        # Prepare standard output and error output
        self.__stdout = b' '
        self.__stderr = b' '
        # Result object
        self.__result = {'msg': '', 'success': True, 'url': str(self.__download_url)}
        # Seed random module
        random.seed(time.time())
        # We have everything we need, auto-start the thread
        self.start()

    def _build_result(self, msg, success=True):
        """
        Result object builder.

        The result object will contain anything that happened during the process of downloading a file from the given
        URL, and whether it was successful or not.
        :param msg: message to add to the final result object
        :param success: whether this extra informatoin makes the process successful or not
        :return: no value is returned
        """
        self.__result['msg'] = self.__result['msg'] + "\n" + msg
        self.__result['success'] = self.__result['success'] and success

    def __download_with_timeout(self):
        """
        This is a helper method that will download the given URL setting a timeout limit.
        :return: True if success
        :except: a subprocess.TimeoutExpired exception is raised if the download can't be completed within the given
        temporal constraints
        """
        download_command = "cd " + str(self.get_dst_folder()) + "; curl -L -O -C - " + str(self.get_download_url())
        download_subprocess = subprocess.Popen(download_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                               shell=True)
        stdout = b' '
        stderr = b' '
        try:
            self._build_result("Downloading '{}' with timeout set to {} seconds"
                               .format(self.get_download_url(),
                                       self.get_download_timeout()))
            (stdout, stderr) = download_subprocess.communicate(timeout=self.get_download_timeout())
        except subprocess.TimeoutExpired as exception_download_timeout:
            self._build_result("Timeout ({} seconds) ERROR downloading '{}', STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                               .format(self.get_download_timeout(),
                                       self.get_download_url(),
                                       stdout.decode('utf8'),
                                       stderr.decode('utf8')))
            raise
        # Let's check the result from downloading the file
        if download_subprocess.poll():
            if download_subprocess.returncode != 0:
                # ERROR
                self._build_result("ERROR downloading '{}', STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                                   .format(self.get_download_timeout(),
                                           self.get_download_url(),
                                           stdout.decode('utf8'),
                                           stderr.decode('utf8')))
                return False
            # SUCCESS
            self._build_result("SUCCESSFUL download for '{}', STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                               .format(self.get_download_timeout(),
                                       self.get_download_url(),
                                       stdout.decode('utf8'),
                                       stderr.decode('utf8')))
        else:
            # TODO
            # The child process is still running, and this is weird stuff because we are using a timeout parameter, we
            # should never reach this part of the code
            self._build_result("WEIRD DOWNLOAD - THE SUBPROCESS IS STILL RUNNING FOR '{}' DESPITE THE TIME OUT "
                               "PARAMETER, STDOUT: |||> {} <|||, STDERR XXX> {} <XXX"
                               .format(self.get_download_timeout(),
                                       self.get_download_url(),
                                       stdout.decode('utf8'),
                                       stderr.decode('utf8')))
            return False
        return True

    def __download_with_timeout_attempts(self):
        """
        Download the given URL given a time constraint with a limited number of attempts upon timeout errors.
        :return: True if success, False if we reached the maximum number of attempts
        """
        timeout_attempt_counter = 0
        while timeout_attempt_counter < self.get_timeout_attempts():
            self._build_result("Downloading '{}', timeout attempt #{} out of #{}"
                               .format(self.get_download_url(),
                                       timeout_attempt_counter,
                                       self.get_timeout_attempts()))
            timeout_attempt_counter += 1
            try:
                return self.__download_with_timeout()
            except subprocess.TimeoutExpired as exception_download_timeout:
                self._build_result("Download of '{}' TIMED OUT, timeout attempt #{} out of #{}"
                                   .format(self.get_download_url(),
                                           timeout_attempt_counter,
                                           self.get_timeout_attempts()))
                # wait for a random amount of time before retrying the download
                # WARNING! - MAGIC NUMBER AHEAD!!!
                time.sleep(random.randint(0, 60))
        return False

    def run(self):
        """
        This is the main thread for the download agent
        :return: no value is returned
        """
        # TODO - Validate URL
        attempt_counter = 0
        download_completion = False
        while attempt_counter < self.get_download_attempts():
            attempt_counter += 1
            self._build_result("Downloading '{}', download attempt #{} out of #{}"
                               .format(self.get_download_url(),
                                       attempt_counter,
                                       self.get_download_attempts()))
            try:
                download_completion = self.__download_with_timeout_attempts()
                if download_completion:
                    # Quit as soon as we succeed on downloading the file
                    break
            except Exception as e:
                self._build_result("ERROR downloading '{}', on download attempt #{} out of #{}, ERROR: {}"
                                   .format(self.get_download_url(),
                                           attempt_counter,
                                           self.get_download_attempts(),
                                           str(e)))
            if download_completion:
                self._build_result("Download for '{}' COMPLETED, on download attempt #{} out of #{}"
                                   .format(self.get_download_url(),
                                           attempt_counter,
                                           self.get_download_attempts()))
            else:
                self._build_result("Download for '{}' FAILED, on download attempt #{} out of #{}"
                                   .format(self.get_download_url(),
                                           attempt_counter,
                                           self.get_download_attempts()),
                                   False)

    def cancel(self):
        """
        I think this is the way to stop the thread, but I'm not sure, because the documentation about it is a little bit
        tricky, or I didn't find the right documentation.
        :return: no value is returned
        """
        self.alive = False
        self.join()

    def wait(self):
        """
        Wait for this thread to finish its job (download a file), and get the result object.
        :return: result object with information on the finished download process
        """
        self.join()
        return self.get_result()

    def get_result(self):
        """
        Get the result object built by this Agent.

        This method should be called when the agent finishes its job, but not in the middle of it.
        :return: result object with information on the finished download process
        """
        return self.__result

    def get_dst_folder(self):
        return self.__dst_folder

    def get_download_timeout(self):
        return self.__download_timeout

    def get_download_url(self):
        return self.__download_url

    def get_timeout_attempts(self):
        return self.__timeout_attempts

    def get_download_attempts(self):
        return self.__download_attempts

class Manager:
    def __init__(self, urls, download_destination_folder, logger, download_attempts=32, timeout_attempts=3, download_timeout=120):
        self.__urls = urls
        self.__download_destination_folder = download_destination_folder
        self.__logger = logger
        self.__download_attempts = download_attempts
        self.__timeout_attempts = timeout_attempts
        self.__download_timeout = download_timeout
        self.__agents = {}
        self.__success = True

    def start_downloads(self):
        # TODO
        pass

    def wait_all(self):
        # TODO
        pass

    def is_success(self):
        return self.__success

    def get_urls_to_download(self):
        return self.__urls

    def get_download_destination_folder(self):
        return self.__download_destination_folder

    def _get_logger(self):
        return self.__logger

    def get_download_attempts(self):
        return self.__download_attempts

    def get_timeout_attempts(self):
        return self.__timeout_attempts

    def get_download_timeouts(self):
        return self.__download_timeout<|endoftext|>


<commit_before>import inspect
import itertools

import logging
l = logging.getLogger("simuvex.s_procedure")

symbolic_count = itertools.count()

class SimProcedure(object):
    def __init__(
        self, addr, arch,
        symbolic_return=None,
        returns=None, is_syscall=None,
        num_args=None, display_name=None,
        convention=None, sim_kwargs=None,
        is_function=None, is_continuation=False,
        continuation_addr=None
    ):
        """
        :param arch:            The architecture to use for this procedure

        The following parameters are optional:

        :param symbolic_return: Whether the procedure's return value should be stubbed into a
                                single symbolic variable constratined to the real return value
        :param returns:         Whether the procedure should return to its caller afterwards
        :param is_syscall:      Whether this procedure is a syscall
        :param num_args:        The number of arguments this procedure should extract
        :param display_name:    The name to use when displaying this procedure
        :param convention:      The SimCC to use for this procedure
        :param sim_kwargs:      Additional keyword arguments to be passed to run()
        :param is_function:     Whether this procedure emulates a function
        """
        self.addr = addr
        self.arch = arch

        self.kwargs = { } if sim_kwargs is None else sim_kwargs
        self.display_name = type(self).__name__ if display_name is None else display_name
        self.symbolic_return = symbolic_return

        # types
        self.argument_types = { } # a dictionary of index-to-type (i.e., type of arg 0: SimTypeString())
        self.return_type = None

        # calling convention
        if convention is None:
            # default conventions
            if self.arch.name in DefaultCC:
                self.cc = DefaultCC[self.arch.name](self.arch)
            else:
                raise SimProcedureError('There is no default calling convention for architecture %s.' +
                                        ' You must specify a calling convention.', arch.name)

        else:
            self.cc = convention

        # set some properties about the type of procedure this is
        self.returns = returns if returns is not None else not self.NO_RET
        self.is_syscall = is_syscall if is_syscall is not None else self.IS_SYSCALL
        self.is_function = is_function if is_function is not None else self.IS_FUNCTION
        self.is_continuation = is_continuation
        self.continuation_addr = continuation_addr

        if self.continuation_addr is None and self.is_function:
            raise ValueError("This procedure is a function but no continuation address is provided!")

        # Get the concrete number of arguments that should be passed to this procedure
        if num_args is None:
            run_spec = inspect.getargspec(self.run)
            self.num_args = len(run_spec.args) - (len(run_spec.defaults) if run_spec.defaults is not None else 0) - 1
        else:
            self.num_args = num_args

        # runtime values
        self.state = None
        self.successors = None
        self.arguments = None
        self.ret_to = None
        self.ret_expr = None

    def __repr__(self):
        syscall = ' (syscall)' if self.IS_SYSCALL else ''
        return "<SimProcedure %s%s>" % (self.display_name, syscall)

    def execute(self, state, successors=None, arguments=None, ret_to=None):
        """
        Call this method with a SimState and a SimSuccessors to execute the procedure.

        Alternately, successors may be none if this is an inline call. In that case, you should
        provide arguments to the function.
        """
        # set runtime variables
        self.state = state
        self.successors = successors
        self.arguments = arguments
        self.ret_to = ret_to
        self.ret_expr = None

        # check to see if this is a syscall and if we should override its return value
        override = None
        if self.is_syscall:
            state.scratch.executed_syscall_count = 1
            if len(state.posix.queued_syscall_returns):
                override = state.posix.queued_syscall_returns.pop(0)

        if callable(override):
            try:
                r = override(state, run=self)
            except TypeError:
                r = override(state)

        elif override is not None:
            r = override

        else:
            # get the arguments
            if arguments is None:
                sim_args = [ self.arg(_) for _ in xrange(self.num_args) ]
            else:
                sim_args = self.arguments[:self.num_args]

            # handle if this is a continuation from a return
            if self.is_continuation:
                if len(state.procedure_data.callstack) == 0:
                    raise SimProcedureError("Tried to run simproc continuation with empty stack")

                continue_at, stack_space, saved_local_vars = state.procedure_data.callstack.pop()
                run_func = getattr(self, continue_at)
                state.regs.sp += stack_space
                for name, val in saved_local_vars:
                    setattr(self, name, val)
            else:
                run_func = self.run

            # run it
            r = run_func(*sim_args, **self.kwargs)

        if self.returns:
            self.ret(r)

        # TODO: remove this once we're done plastering over the metaclass embarassment
        return self

    #
    # Implement these in a subclass of SimProcedure!
    #

    NO_RET = False          # set this to true if control flow will never return from this function
    ADDS_EXITS = False      # set this to true if you do any control flow other than returning
    IS_SYSCALL = False      # self-explanitory.
    IS_FUNCTION = False     # set this to true if you use the self.call() control flow

    local_vars = ()         # if you use self.call(), set this to a list of all the local variable
                            # names in your class. They will be restored on return.

    def run(self, *args, **kwargs): #pylint:disable=unused-argument
        """
        Implement the actual procedure here!
        """
        raise SimProcedureError("%s does not implement a run() method" % self.__class__.__name__)

    def static_exits(self, blocks):  # pylint: disable=unused-argument
        """
        Get new exits by performing static analysis and heuristics. This is a fast and best-effort approach to get new
        exits for scenarios where states are not available (e.g. when building a fast CFG).

        :param list blocks: Blocks that are executed before reaching this SimProcedure.
        :return: A list of tuples. Each tuple is (address, jumpkind).
        :rtype: list
        """

        if self.ADDS_EXITS:
            raise SimProcedureError("static_exits() is not implemented for %s" % self)
        else:
            # This SimProcedure does not add any new exit
            return [ ]

    #
    # misc properties
    #

    @property
    def should_add_successors(self):
        return self.successors is not None

    @property
    def use_state_arguments(self):
        return self.arguments is None

    #
    # Working with calling conventions
    #

    def set_args(self, args):
        arg_session = self.cc.arg_session
        for arg in args:
            if self.cc.is_fp_value(args):
                arg_session.next_arg(True).set_value(self.state, arg)
            else:
                arg_session.next_arg(False).set_value(self.state, arg)

    def arg(self, i):
        """
        Returns the ith argument. Raise a SimProcedureArgumentError if we don't have such an argument available.

        :param int i: The index of the argument to get
        :return: The argument
        :rtype: object
        """
        if self.use_state_arguments:
            r = self.cc.arg(self.state, i)
        else:
            if i >= len(self.arguments):
                raise SimProcedureArgumentError("Argument %d does not exist." % i)
            r = self.arguments[i]           # pylint: disable=unsubscriptable-object

        l.debug("returning argument")
        return r

    def set_return_expr(self, expr):
        """
        Set this expression as the return value for the function.
        If this is not an inline call, this will write the expression to the state via the
        calling convention.
        """
        if isinstance(expr, (int, long)):
            expr = self.state.se.BVV(expr, self.state.arch.bits)

        if o.SIMPLIFY_RETS in self.state.options:
            l.debug("... simplifying")
            l.debug("... before: %s", expr)
            expr = self.state.se.simplify(expr)
            l.debug("... after: %s", expr)

        if self.symbolic_return:
            size = len(expr)
            new_expr = self.state.se.Unconstrained("symbolic_return_" + self.__class__.__name__, size) #pylint:disable=maybe-no-member
            self.state.add_constraints(new_expr == expr)
            expr = new_expr

        self.ret_expr = expr
        if self.use_state_arguments:
            self.cc.return_val.set_value(self.state, expr)

    #
    # Control Flow
    #

    def inline_call(self, procedure, *arguments, **sim_kwargs):
        """
        Call another SimProcedure in-line to retrieve its return value.
        Returns an instance of the procedure with the ret_expr property set.

        :param procedure:       The class of the procedure to execute
        :param arguments:       Any additional positional args will be used as arguments to the
                                procedure call
        :param sim_kwargs:      Any additional keyword args will be passed as sim_kwargs to the
                                procedure construtor
        """
        e_args = [ self.state.se.BVV(a, self.state.arch.bits) if isinstance(a, (int, long)) else a for a in arguments ]
        p = procedure(self.addr, self.arch, sim_kwargs=sim_kwargs)
        p.execute(self.state, None, arguments=e_args)
        return p

    def ret(self, expr=None):
        """
        Add an exit representing a return from this function.
        If this is not an inline call, grab a return address from the state and jump to it.
        If this is not an inline call, set a return expression with the calling convention.
        """
        if expr is not None:
            self.set_return_expr(expr)

        if not self.should_add_successors:
            l.debug("Returning without setting exits due to 'internal' call.")
            return

        if self.ret_to is not None:
            # TODO: If set ret_to, do we also want to pop an unused ret addr from the stack?
            ret_addr = self.ret_to
        else:
            if self.state.arch.call_pushes_ret:
                ret_addr = self.state.stack_pop()
            else:
                ret_addr = self.state.registers.load(self.state.arch.lr_offset, self.state.arch.bytes)

        self._exit_action(self.state, ret_addr)
        self.successors.add_successor(self.state, ret_addr, self.state.se.true, 'Ijk_Ret')

    def call(self, addr, args, continue_at, cc=None):
        """
        Add an exit representing calling another function via pointer.

        :param addr:        The address of the function to call
        :param args:        The list of arguments to call the function with
        :param continue_at: Later, when the called function returns, execution of the current
                            procedure will continue in the named method.
        :param cc:          Optional: use this calling convention for calling the new function.
                            Default is to use the current convention.
        """
        if not self.is_function:
            raise ValueError("%s called self.call() without IS_FUNCTION = True")
        if cc is None:
            cc = self.cc

        call_state = self.state.copy()
        ret_addr = self.continuation_addr
        saved_local_vars = zip(self.local_vars, map(lambda name: getattr(self, name), self.local_vars))
        simcallstack_entry = (continue_at, cc.stack_space(args), saved_local_vars)
        cc.setup_callsite(call_state, ret_addr, args)
        call_state.procedure_data.callstack.append(simcallstack_entry)

        if call_state.libc.ppc64_abiv == 'ppc64_1':
            call_state.regs.r2 = self.state.mem[addr + 8:].long.resolved
            addr = call_state.mem[addr:].long.resolved
        elif call_state.arch.name in ('MIPS32', 'MIPS64'):
            call_state.regs.t9 = addr

        self._exit_action(call_state, addr)
        self.successors.add_successor(call_state, addr, call_state.se.true, 'Ijk_Call')

        if o.DO_RET_EMULATION in self.state.options:
            ret_state = self.state.copy()
            cc.setup_callsite(ret_state, ret_addr, args)
            ret_state.procedure_data.callstack.append(simcallstack_entry)
            guard = ret_state.se.true if o.TRUE_RET_EMULATION_GUARD in ret_state.options else ret_state.se.false
            self.successors.add_successor(ret_state, ret_addr, guard, 'Ijk_FakeRet')

    def jump(self, addr):
        """
        Add an exit representing jumping to an address.
        """
        self._exit_action(self.state, addr)
        self.successors.add_successor(self.state, addr, self.state.se.true, 'Ijk_Boring')

    def exit(self, exit_code):
        """
        Add an exit representing terminating the program.
        """
        self.state.options.discard(o.AST_DEPS)
        self.state.options.discard(o.AUTO_REFS)

        if isinstance(exit_code, (int, long)):
            exit_code = self.state.se.BVV(exit_code, self.state.arch.bits)
        self.state.log.add_event('terminate', exit_code=exit_code)
        self.successors.add_successor(self.state, self.state.regs.ip, self.state.se.true, 'Ijk_Exit')

    @staticmethod
    def _exit_action(state, addr):
        if o.TRACK_JMP_ACTIONS in state.options:
            state.log.add_action(SimActionExit(state, addr))

    #
    # misc
    #

    def ty_ptr(self, ty):
        return SimTypePointer(self.arch, ty)

from . import s_options as o
from .s_errors import SimProcedureError, SimProcedureArgumentError
from .s_type import SimTypePointer
from .s_action import SimActionExit
from .s_cc import DefaultCC
<commit_msg>add forgotten arg<commit_after>import inspect
import itertools

import logging
l = logging.getLogger("simuvex.s_procedure")

symbolic_count = itertools.count()

class SimProcedure(object):
    def __init__(
        self, addr, arch,
        symbolic_return=None,
        returns=None, is_syscall=None,
        num_args=None, display_name=None,
        convention=None, sim_kwargs=None,
        is_function=None, is_continuation=False,
        continuation_addr=None
    ):
        """
        :param arch:            The architecture to use for this procedure

        The following parameters are optional:

        :param symbolic_return: Whether the procedure's return value should be stubbed into a
                                single symbolic variable constratined to the real return value
        :param returns:         Whether the procedure should return to its caller afterwards
        :param is_syscall:      Whether this procedure is a syscall
        :param num_args:        The number of arguments this procedure should extract
        :param display_name:    The name to use when displaying this procedure
        :param convention:      The SimCC to use for this procedure
        :param sim_kwargs:      Additional keyword arguments to be passed to run()
        :param is_function:     Whether this procedure emulates a function
        """
        self.addr = addr
        self.arch = arch

        self.kwargs = { } if sim_kwargs is None else sim_kwargs
        self.display_name = type(self).__name__ if display_name is None else display_name
        self.symbolic_return = symbolic_return

        # types
        self.argument_types = { } # a dictionary of index-to-type (i.e., type of arg 0: SimTypeString())
        self.return_type = None

        # calling convention
        if convention is None:
            # default conventions
            if self.arch.name in DefaultCC:
                self.cc = DefaultCC[self.arch.name](self.arch)
            else:
                raise SimProcedureError('There is no default calling convention for architecture %s.' +
                                        ' You must specify a calling convention.', arch.name)

        else:
            self.cc = convention

        # set some properties about the type of procedure this is
        self.returns = returns if returns is not None else not self.NO_RET
        self.is_syscall = is_syscall if is_syscall is not None else self.IS_SYSCALL
        self.is_function = is_function if is_function is not None else self.IS_FUNCTION
        self.is_continuation = is_continuation
        self.continuation_addr = continuation_addr

        if self.continuation_addr is None and self.is_function:
            raise ValueError("This procedure is a function but no continuation address is provided!")

        # Get the concrete number of arguments that should be passed to this procedure
        if num_args is None:
            run_spec = inspect.getargspec(self.run)
            self.num_args = len(run_spec.args) - (len(run_spec.defaults) if run_spec.defaults is not None else 0) - 1
        else:
            self.num_args = num_args

        # runtime values
        self.state = None
        self.successors = None
        self.arguments = None
        self.ret_to = None
        self.ret_expr = None

    def __repr__(self):
        syscall = ' (syscall)' if self.IS_SYSCALL else ''
        return "<SimProcedure %s%s>" % (self.display_name, syscall)

    def execute(self, state, successors=None, arguments=None, ret_to=None):
        """
        Call this method with a SimState and a SimSuccessors to execute the procedure.

        Alternately, successors may be none if this is an inline call. In that case, you should
        provide arguments to the function.
        """
        # set runtime variables
        self.state = state
        self.successors = successors
        self.arguments = arguments
        self.ret_to = ret_to
        self.ret_expr = None

        # check to see if this is a syscall and if we should override its return value
        override = None
        if self.is_syscall:
            state.scratch.executed_syscall_count = 1
            if len(state.posix.queued_syscall_returns):
                override = state.posix.queued_syscall_returns.pop(0)

        if callable(override):
            try:
                r = override(state, run=self)
            except TypeError:
                r = override(state)

        elif override is not None:
            r = override

        else:
            # get the arguments
            if arguments is None:
                sim_args = [ self.arg(_) for _ in xrange(self.num_args) ]
            else:
                sim_args = self.arguments[:self.num_args]

            # handle if this is a continuation from a return
            if self.is_continuation:
                if len(state.procedure_data.callstack) == 0:
                    raise SimProcedureError("Tried to run simproc continuation with empty stack")

                continue_at, stack_space, saved_local_vars = state.procedure_data.callstack.pop()
                run_func = getattr(self, continue_at)
                state.regs.sp += stack_space
                for name, val in saved_local_vars:
                    setattr(self, name, val)
            else:
                run_func = self.run

            # run it
            r = run_func(*sim_args, **self.kwargs)

        if self.returns:
            self.ret(r)

        # TODO: remove this once we're done plastering over the metaclass embarassment
        return self

    #
    # Implement these in a subclass of SimProcedure!
    #

    NO_RET = False          # set this to true if control flow will never return from this function
    ADDS_EXITS = False      # set this to true if you do any control flow other than returning
    IS_SYSCALL = False      # self-explanitory.
    IS_FUNCTION = False     # set this to true if you use the self.call() control flow

    local_vars = ()         # if you use self.call(), set this to a list of all the local variable
                            # names in your class. They will be restored on return.

    def run(self, *args, **kwargs): #pylint:disable=unused-argument
        """
        Implement the actual procedure here!
        """
        raise SimProcedureError("%s does not implement a run() method" % self.__class__.__name__)

    def static_exits(self, blocks):  # pylint: disable=unused-argument
        """
        Get new exits by performing static analysis and heuristics. This is a fast and best-effort approach to get new
        exits for scenarios where states are not available (e.g. when building a fast CFG).

        :param list blocks: Blocks that are executed before reaching this SimProcedure.
        :return: A list of tuples. Each tuple is (address, jumpkind).
        :rtype: list
        """

        if self.ADDS_EXITS:
            raise SimProcedureError("static_exits() is not implemented for %s" % self)
        else:
            # This SimProcedure does not add any new exit
            return [ ]

    #
    # misc properties
    #

    @property
    def should_add_successors(self):
        return self.successors is not None

    @property
    def use_state_arguments(self):
        return self.arguments is None

    #
    # Working with calling conventions
    #

    def set_args(self, args):
        arg_session = self.cc.arg_session
        for arg in args:
            if self.cc.is_fp_value(args):
                arg_session.next_arg(True).set_value(self.state, arg)
            else:
                arg_session.next_arg(False).set_value(self.state, arg)

    def arg(self, i):
        """
        Returns the ith argument. Raise a SimProcedureArgumentError if we don't have such an argument available.

        :param int i: The index of the argument to get
        :return: The argument
        :rtype: object
        """
        if self.use_state_arguments:
            r = self.cc.arg(self.state, i)
        else:
            if i >= len(self.arguments):
                raise SimProcedureArgumentError("Argument %d does not exist." % i)
            r = self.arguments[i]           # pylint: disable=unsubscriptable-object

        l.debug("returning argument")
        return r

    def set_return_expr(self, expr):
        """
        Set this expression as the return value for the function.
        If this is not an inline call, this will write the expression to the state via the
        calling convention.
        """
        if isinstance(expr, (int, long)):
            expr = self.state.se.BVV(expr, self.state.arch.bits)

        if o.SIMPLIFY_RETS in self.state.options:
            l.debug("... simplifying")
            l.debug("... before: %s", expr)
            expr = self.state.se.simplify(expr)
            l.debug("... after: %s", expr)

        if self.symbolic_return:
            size = len(expr)
            new_expr = self.state.se.Unconstrained("symbolic_return_" + self.__class__.__name__, size) #pylint:disable=maybe-no-member
            self.state.add_constraints(new_expr == expr)
            expr = new_expr

        self.ret_expr = expr
        if self.use_state_arguments:
            self.cc.return_val.set_value(self.state, expr)

    #
    # Control Flow
    #

    def inline_call(self, procedure, *arguments, **sim_kwargs):
        """
        Call another SimProcedure in-line to retrieve its return value.
        Returns an instance of the procedure with the ret_expr property set.

        :param procedure:       The class of the procedure to execute
        :param arguments:       Any additional positional args will be used as arguments to the
                                procedure call
        :param sim_kwargs:      Any additional keyword args will be passed as sim_kwargs to the
                                procedure construtor
        """
        e_args = [ self.state.se.BVV(a, self.state.arch.bits) if isinstance(a, (int, long)) else a for a in arguments ]
        p = procedure(self.addr, self.arch, sim_kwargs=sim_kwargs)
        p.execute(self.state, None, arguments=e_args)
        return p

    def ret(self, expr=None):
        """
        Add an exit representing a return from this function.
        If this is not an inline call, grab a return address from the state and jump to it.
        If this is not an inline call, set a return expression with the calling convention.
        """
        if expr is not None:
            self.set_return_expr(expr)

        if not self.should_add_successors:
            l.debug("Returning without setting exits due to 'internal' call.")
            return

        if self.ret_to is not None:
            # TODO: If set ret_to, do we also want to pop an unused ret addr from the stack?
            ret_addr = self.ret_to
        else:
            if self.state.arch.call_pushes_ret:
                ret_addr = self.state.stack_pop()
            else:
                ret_addr = self.state.registers.load(self.state.arch.lr_offset, self.state.arch.bytes)

        self._exit_action(self.state, ret_addr)
        self.successors.add_successor(self.state, ret_addr, self.state.se.true, 'Ijk_Ret')

    def call(self, addr, args, continue_at, cc=None):
        """
        Add an exit representing calling another function via pointer.

        :param addr:        The address of the function to call
        :param args:        The list of arguments to call the function with
        :param continue_at: Later, when the called function returns, execution of the current
                            procedure will continue in the named method.
        :param cc:          Optional: use this calling convention for calling the new function.
                            Default is to use the current convention.
        """
        if not self.is_function:
            raise ValueError("%s called self.call() without IS_FUNCTION = True" % self)
        if cc is None:
            cc = self.cc

        call_state = self.state.copy()
        ret_addr = self.continuation_addr
        saved_local_vars = zip(self.local_vars, map(lambda name: getattr(self, name), self.local_vars))
        simcallstack_entry = (continue_at, cc.stack_space(args), saved_local_vars)
        cc.setup_callsite(call_state, ret_addr, args)
        call_state.procedure_data.callstack.append(simcallstack_entry)

        if call_state.libc.ppc64_abiv == 'ppc64_1':
            call_state.regs.r2 = self.state.mem[addr + 8:].long.resolved
            addr = call_state.mem[addr:].long.resolved
        elif call_state.arch.name in ('MIPS32', 'MIPS64'):
            call_state.regs.t9 = addr

        self._exit_action(call_state, addr)
        self.successors.add_successor(call_state, addr, call_state.se.true, 'Ijk_Call')

        if o.DO_RET_EMULATION in self.state.options:
            ret_state = self.state.copy()
            cc.setup_callsite(ret_state, ret_addr, args)
            ret_state.procedure_data.callstack.append(simcallstack_entry)
            guard = ret_state.se.true if o.TRUE_RET_EMULATION_GUARD in ret_state.options else ret_state.se.false
            self.successors.add_successor(ret_state, ret_addr, guard, 'Ijk_FakeRet')

    def jump(self, addr):
        """
        Add an exit representing jumping to an address.
        """
        self._exit_action(self.state, addr)
        self.successors.add_successor(self.state, addr, self.state.se.true, 'Ijk_Boring')

    def exit(self, exit_code):
        """
        Add an exit representing terminating the program.
        """
        self.state.options.discard(o.AST_DEPS)
        self.state.options.discard(o.AUTO_REFS)

        if isinstance(exit_code, (int, long)):
            exit_code = self.state.se.BVV(exit_code, self.state.arch.bits)
        self.state.log.add_event('terminate', exit_code=exit_code)
        self.successors.add_successor(self.state, self.state.regs.ip, self.state.se.true, 'Ijk_Exit')

    @staticmethod
    def _exit_action(state, addr):
        if o.TRACK_JMP_ACTIONS in state.options:
            state.log.add_action(SimActionExit(state, addr))

    #
    # misc
    #

    def ty_ptr(self, ty):
        return SimTypePointer(self.arch, ty)

from . import s_options as o
from .s_errors import SimProcedureError, SimProcedureArgumentError
from .s_type import SimTypePointer
from .s_action import SimActionExit
from .s_cc import DefaultCC
<|endoftext|>


<commit_before>from style_grader_functions import check_if_function, check_if_function_prototype
from pyparsing import Literal, Word, Optional, ParseException, Group, SkipTo, alphanums, LineStart, srange
import re

def check_function_def_above_main(self, code):
    prototype = check_if_function_prototype(code)
    function = check_if_function(code)
    inside = Literal("int main")
    if len(inside.searchString(code)):
        return
    elif function and not prototype and self.outside_main:
        function_regex = re.compile("^\s*(\w+)\s+(\w+)")
        match = function_regex.search(code)
        function_name = match.group(2) if match else "NOT_FOUND"
        self.add_error(label="DEFINITION_ABOVE_MAIN", data={'function': function_name})

def check_int_for_bool(self, code):
    if check_if_function(code):
        function_regex = re.compile("^\s*(\w+)\s+(\w+)")
        match = function_regex.search(code)
        if match:
            self.current_function = (match.group(1), match.group(2))
    current_function = getattr(self, "current_function", ("", ""))

    return_regex = re.compile("\s*return\s+(\w+)")
    match = return_regex.search(code)
    if match and match.group(1).isdigit() and current_function[0] == "bool":
        self.add_error(label="INT_FOR_BOOL")

# New operator spacing function
def check_operator_spacing(self, code):
    # Check normal operators
    # account for *=, %=, /=, +=, -=
    indexes = []
    indexes = findOccurences(code, '+') + \
              findOccurences(code, '-') + \
              findOccurences(code, '%') + \
              findOccurences(code, '*') + \
              findOccurences(code, '/') + \
              findOccurences(code, '=') + \
              findOccurences(code, '!') + \
              findMultiOccurences(code, '&&') + \
              findMultiOccurences(code, '||')

    for operator_index in indexes:
        if code[operator_index + 1]:
            if code[operator_index + 1] == '+' or\
                code[operator_index + 1] == '-' or\
                code[operator_index + 1] == '*':
                continue
        if code[operator_index - 1]:
                if code[operator_index - 1] == '+' \
                     or code[operator_index - 1] == '-' \
                     or code[operator_index - 1] == '*':
            # if it's an increment, decrement ignore it
                        continue
        compound = False
        # check to see if it's a compound that's been checked already
        if code[operator_index] == '=':
            if code[operator_index - 1]:
                if code[operator_index - 1] in ['<', '>', '*', '/', '%', '+', '=', '-', '!']:
                    continue
        if code[operator_index + 1] == '=':
            compound = True
        if code[operator_index] == '-' and code[operator_index + 1]:
            if is_number(code[operator_index + 1]) or code[operator_index + 1] == '(':
            #neg number - forget about it
                continue
        if not operator_helper(compound, code, operator_index):
            self.add_error(label="OPERATOR_SPACING", column=operator_index, data={'operator': code[operator_index]})

def operator_helper(compound, code, index):
    correct_spacing = True
    if compound:
        if code[index + 2] and code[index + 2] != '\r' and code[index + 2] != '\n' and code[index + 2] != ' ':
            correct_spacing = False
        if code[index - 1] and code[index - 1] != ' ':
            correct_spacing = False
    else:
        if code[index + 1] and code[index + 1] != '\r' and code[index + 1] != '\n' and code[index + 1] != ' ':
            correct_spacing = False
        if code[index - 1] and code[index - 1] != ' ':
            correct_spacing = False
    return correct_spacing

def findOccurences(s, ch):
    return [i for i, letter in enumerate(s) if letter == ch]
def findMultiOccurences(s, sub):
    occurences = []
    loc = 0
    while s.find(sub, loc) != -1:
        occurences.append(s.find(sub, loc))
        loc = occurences[-1] + 1 # find is inclusive of start, must move forward 1
    return occurences

def check_equals_true(self, code):
    keyword = Literal("true") | Literal("false")
    statement_parser = Group("==" + keyword) | Group(keyword + "==")
    if len(statement_parser.searchString(code)):
        self.add_error(label="EQUALS_TRUE")

def check_goto(self, code):
    # Hacky but gets the job done for now - has holes though
    q_goto = re.compile('\".*goto.*\"')
    r_goto = re.compile('(?:\s+|^|\{)goto\s+')
    if r_goto.search(code) and not q_goto.search(code):
        self.add_error(label="GOTO")

def erase_string(code):
    # remove contents of literal strings
    code = code.replace("\\\"", "") # remove escaped quotes
    results = re.findall(r'"(.*?)"', code)
    for string in results:
        quote_mark = "\""
        code = code.replace(quote_mark + string + quote_mark, "\"\"")

    # remove contents of literal chars
    code = code.replace('\\\\', '') # replace escaped backslash
    code = code.replace("\\'", "") # remove escaped single quote
    results = re.findall(r"'(.*?)'", code)
    for string in results:
        single_quote_mark = "'"
        code = code.replace(single_quote_mark + string + single_quote_mark, "''")
    return code

def check_define_statement(self, code):
    q_define = re.compile('\".*(?:\s+|^)#\s*define\s+.*\"')
    r_define = re.compile('(?:\s+|^)#\s*define\s+')
    if r_define.search(code) and not q_define.search(code):
        words = code.split()
        # They shouldn't be using __MY_HEADER_H__ because __-names are
        # reserved, but we'll allow it anyways.
        legal_endings = ["_H", "_H__"]
        if not any(words[-1].endswith(i) for i in legal_endings):
            self.add_error(label="DEFINE_STATEMENT")

def check_continue(self, code):
    # Hacky but gets the job done for now - has holes though
    q_continue = re.compile('\".*continue.*\"')
    r_continue = re.compile('(?:\s+|^|\{)continue\s*;')
    if r_continue.search(code) and not q_continue.search(code):
        self.add_error(label="CONTINUE_STATEMENT")

def check_ternary_operator(self, code):
    q_ternary = re.compile('\".*\?.*\"')
    r_ternary = re.compile('\?')
    if r_ternary.search(code) and not q_ternary.search(code):
        self.add_error(label="TERNARY_OPERATOR")

def check_while_true(self, code):
    statement_parser = Literal("while") + Literal("(") + Literal("true") + Literal(")")
    if len(statement_parser.searchString(code)):
        self.add_error(label="WHILE_TRUE")

def check_non_const_global(self, code):
    inside = Literal("int main")
    if len(inside.searchString(code)):
        self.outside_main = False

    if self.outside_main:
        function = check_if_function(code)
        variables = variables = re.compile("^(?:\w|_)+\s+(?:\w|_|\[|\])+\s*=\s*.+;")
        keywords = re.compile("^\s*(?:using|class|struct)")
        constants = re.compile("^\s*(?:static\s+)?const")
        if not function and variables.search(code) and \
                not keywords.search(code) and \
                not constants.search(code):
            self.add_error(label="NON_CONST_GLOBAL")

def check_main_syntax(self, code):
    # Return value for main is optional in C++11
    parser = Literal("int") + Literal("main") + Literal("(") + SkipTo(Literal(")")) + Literal(")")
    if len(parser.searchString(code)):
        main_prefix = Literal("int") + Literal("main") + Literal("(")
        full_use = Literal("int") + "argc" + "," + Optional("const") + "char" + "*" + "argv" + "[" + "]" + ")"
        # 3 options for main() syntax
        if not len((main_prefix + Literal(")")).searchString(code)) and \
                not len((main_prefix + Literal("void") + Literal(")")).searchString(code)) and \
                not len((main_prefix + full_use).searchString(code)):
            self.add_error(label="MAIN_SYNTAX")

def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def check_first_char(self, code):
    # check if the first char is lower-case alpha or '_'
    lowercase = re.compile("(?:^|\s+)(?:class|struct)\s+(?:[a-z]|_)\w+")
    bad_naming = lowercase.search(code)
    if bad_naming:
        result = bad_naming.group(0).split()
        self.add_error(label="FIRST_CHAR",
                       data={"keyword": result[0],
                             "expected": str(result[1]).capitalize(),
                             "found": str(result[1])})

def check_unnecessary_include(self, code):
    grammar = Literal('#') + Literal('include') + Literal('<') + Word(alphanums)
    try:
        grammar.parseString(code)
        begin = code.find("<")
        end = code.find(">")
        included_library = code[begin + 1:end]
        if included_library not in self.includes:
            self.add_error(label="UNNECESSARY_INCLUDE")
    except ParseException:
        return

def check_local_include(self, code):
    grammar = Literal('#') + Literal('include') + Literal('"') + Word(alphanums)
    try:
        grammar.parseString(code)
        begin = code.find('"')
        included_file = code[begin + 1:]
        end = included_file.find('"')
        included_file = included_file[:end]
        if included_file not in self.includes:
            self.local_includes[self.current_file].append(included_file)
    except ParseException:
        return

def check_for_loop_semicolon_spacing(self, code):
    # Match the semicolons and any whitespace around them.
    for_loop_regex = re.compile(
        r"""
        \s*for\s*\(
            (?P<code1>[^;]*?)

            (?P<semicolon1>\s*;\s*)

            (?P<code2>[^;]*?)

            (?P<semicolon2>\s*;\s*)

            (?P<code3>[^;]*?)
        \)
        """,
        re.VERBOSE
    )
    match = for_loop_regex.search(code)
    if not match:
        return

    self.for_loop_spacing_before = getattr(self, "for_loop_spacing_before", None)
    self.for_loop_spacing_after = getattr(self, "for_loop_spacing_after", None)

    semicolon1 = match.group("semicolon1")
    semicolon2 = match.group("semicolon2")
    code1 = match.group("code1")
    code2 = match.group("code2")
    code3 = match.group("code3")

    def is_spacing_okay(semicolon, before_code, after_code):
        spacing_before = semicolon.startswith(" ")
        spacing_after = semicolon.endswith(" ")

        def check_spacing(convention, actual):
            if convention is None:
                convention = actual

            if convention != actual:
                return convention, False
            else:
                return convention, True

        if before_code or after_code:
            if before_code:
                self.for_loop_spacing_before, result = check_spacing(
                    self.for_loop_spacing_before,
                    spacing_before
                )
                if not result:
                    return False
            if after_code:
                self.for_loop_spacing_after, result = check_spacing(
                    self.for_loop_spacing_after,
                    spacing_after
                )
                if not result:
                    return False
        else:
            # This is a plain semicolon, so we can't infer anything about the
            # spacing convention.
            pass
        return True

    if not (
                is_spacing_okay(semicolon1, code1, code2)
            and is_spacing_okay(semicolon2, code2, code3)
    ):
        self.add_error(
            label="FOR_LOOP_SEMICOLON_SPACING",
            data={"line": self.current_line_num}
        )
<commit_msg>added functions for refactoring operator spacing<commit_after>from style_grader_functions import check_if_function, check_if_function_prototype
from pyparsing import Literal, Word, Optional, ParseException, Group, SkipTo, alphanums, LineStart, srange
import re

def check_function_def_above_main(self, code):
    prototype = check_if_function_prototype(code)
    function = check_if_function(code)
    inside = Literal("int main")
    if len(inside.searchString(code)):
        return
    elif function and not prototype and self.outside_main:
        function_regex = re.compile("^\s*(\w+)\s+(\w+)")
        match = function_regex.search(code)
        function_name = match.group(2) if match else "NOT_FOUND"
        self.add_error(label="DEFINITION_ABOVE_MAIN", data={'function': function_name})

def check_int_for_bool(self, code):
    if check_if_function(code):
        function_regex = re.compile("^\s*(\w+)\s+(\w+)")
        match = function_regex.search(code)
        if match:
            self.current_function = (match.group(1), match.group(2))
    current_function = getattr(self, "current_function", ("", ""))

    return_regex = re.compile("\s*return\s+(\w+)")
    match = return_regex.search(code)
    if match and match.group(1).isdigit() and current_function[0] == "bool":
        self.add_error(label="INT_FOR_BOOL")

# New operator spacing function
def check_operator_spacing(self, code):
    # Check normal operators
    # account for *=, %=, /=, +=, -=
    indexes = []
    indexes = findOccurences(code, '+') + \
              findOccurences(code, '-') + \
              findOccurences(code, '%') + \
              findOccurences(code, '*') + \
              findOccurences(code, '/') + \
              findOccurences(code, '=') + \
              findOccurences(code, '!') + \
              findMultiOccurences(code, '&') + \
              findMultiOccurences(code, '|')

    for operator_index in indexes:
        if code[operator_index + 1]:
            if code[operator_index + 1] == '+' or\
                code[operator_index + 1] == '-' or\
                code[operator_index + 1] == '*':
                continue
        if code[operator_index - 1]:
                if code[operator_index - 1] == '+' \
                     or code[operator_index - 1] == '-' \
                     or code[operator_index - 1] == '*':
            # if it's an increment, decrement ignore it
                        continue
        compound = False
        # check to see if it's a compound that's been checked already
        if code[operator_index] == '=':
            if code[operator_index - 1]:
                if code[operator_index - 1] in ['<', '>', '*', '/', '%', '+', '=', '-', '!']:
                    continue
        if code[operator_index + 1] == '=':
            compound = True
        if code[operator_index] == '-' and code[operator_index + 1]:
            if is_number(code[operator_index + 1]) or code[operator_index + 1] == '(':
            #neg number - forget about it
                continue
        if not operator_helper(compound, code, operator_index):
            self.add_error(label="OPERATOR_SPACING", column=operator_index, data={'operator': code[operator_index]})

def is_increment_decrement(code, index):
    if code[index + 1]:
        if code[index] in ['+', '-'] and code[index + 1] == code[index]:
            return True
    return False

def is_compound_operator(code, index):
    if code[index + 1]:
        # Check for +=, -=, *=, /=, %=, >=, <=, ==, !=
        if code[index] in ['>', '<', '*', '/', '+', '-', '!', '=', '%']:
            if code[index + 1] == '=':
                return True
        # Check &&, ||
        elif code[index] in ['&', '|']:
            if code[index + 1] == code[index]:
                return True
    return False

def operator_helper(compound, code, index):
    correct_spacing = True
    if compound:
        if code[index + 2] and code[index + 2] != '\r' and code[index + 2] != '\n' and code[index + 2] != ' ':
            correct_spacing = False
        if code[index - 1] and code[index - 1] != ' ':
            correct_spacing = False
    else:
        if code[index + 1] and code[index + 1] != '\r' and code[index + 1] != '\n' and code[index + 1] != ' ':
            correct_spacing = False
        if code[index - 1] and code[index - 1] != ' ':
            correct_spacing = False
    return correct_spacing

def findOccurences(s, ch):
    return [i for i, letter in enumerate(s) if letter == ch]
def findMultiOccurences(s, sub):
    occurences = []
    loc = 0
    while s.find(sub, loc) != -1:
        occurences.append(s.find(sub, loc))
        loc = occurences[-1] + 1 # find is inclusive of start, must move forward 1
    return occurences

def check_equals_true(self, code):
    keyword = Literal("true") | Literal("false")
    statement_parser = Group("==" + keyword) | Group(keyword + "==")
    if len(statement_parser.searchString(code)):
        self.add_error(label="EQUALS_TRUE")

def check_goto(self, code):
    # Hacky but gets the job done for now - has holes though
    q_goto = re.compile('\".*goto.*\"')
    r_goto = re.compile('(?:\s+|^|\{)goto\s+')
    if r_goto.search(code) and not q_goto.search(code):
        self.add_error(label="GOTO")

def erase_string(code):
    # remove contents of literal strings
    code = code.replace("\\\"", "") # remove escaped quotes
    results = re.findall(r'"(.*?)"', code)
    for string in results:
        quote_mark = "\""
        code = code.replace(quote_mark + string + quote_mark, "\"\"")

    # remove contents of literal chars
    code = code.replace('\\\\', '') # replace escaped backslash
    code = code.replace("\\'", "") # remove escaped single quote
    results = re.findall(r"'(.*?)'", code)
    for string in results:
        single_quote_mark = "'"
        code = code.replace(single_quote_mark + string + single_quote_mark, "''")
    return code

def check_define_statement(self, code):
    q_define = re.compile('\".*(?:\s+|^)#\s*define\s+.*\"')
    r_define = re.compile('(?:\s+|^)#\s*define\s+')
    if r_define.search(code) and not q_define.search(code):
        words = code.split()
        # They shouldn't be using __MY_HEADER_H__ because __-names are
        # reserved, but we'll allow it anyways.
        legal_endings = ["_H", "_H__"]
        if not any(words[-1].endswith(i) for i in legal_endings):
            self.add_error(label="DEFINE_STATEMENT")

def check_continue(self, code):
    # Hacky but gets the job done for now - has holes though
    q_continue = re.compile('\".*continue.*\"')
    r_continue = re.compile('(?:\s+|^|\{)continue\s*;')
    if r_continue.search(code) and not q_continue.search(code):
        self.add_error(label="CONTINUE_STATEMENT")

def check_ternary_operator(self, code):
    q_ternary = re.compile('\".*\?.*\"')
    r_ternary = re.compile('\?')
    if r_ternary.search(code) and not q_ternary.search(code):
        self.add_error(label="TERNARY_OPERATOR")

def check_while_true(self, code):
    statement_parser = Literal("while") + Literal("(") + Literal("true") + Literal(")")
    if len(statement_parser.searchString(code)):
        self.add_error(label="WHILE_TRUE")

def check_non_const_global(self, code):
    inside = Literal("int main")
    if len(inside.searchString(code)):
        self.outside_main = False

    if self.outside_main:
        function = check_if_function(code)
        variables = variables = re.compile("^(?:\w|_)+\s+(?:\w|_|\[|\])+\s*=\s*.+;")
        keywords = re.compile("^\s*(?:using|class|struct)")
        constants = re.compile("^\s*(?:static\s+)?const")
        if not function and variables.search(code) and \
                not keywords.search(code) and \
                not constants.search(code):
            self.add_error(label="NON_CONST_GLOBAL")

def check_main_syntax(self, code):
    # Return value for main is optional in C++11
    parser = Literal("int") + Literal("main") + Literal("(") + SkipTo(Literal(")")) + Literal(")")
    if len(parser.searchString(code)):
        main_prefix = Literal("int") + Literal("main") + Literal("(")
        full_use = Literal("int") + "argc" + "," + Optional("const") + "char" + "*" + "argv" + "[" + "]" + ")"
        # 3 options for main() syntax
        if not len((main_prefix + Literal(")")).searchString(code)) and \
                not len((main_prefix + Literal("void") + Literal(")")).searchString(code)) and \
                not len((main_prefix + full_use).searchString(code)):
            self.add_error(label="MAIN_SYNTAX")

def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def check_first_char(self, code):
    # check if the first char is lower-case alpha or '_'
    lowercase = re.compile("(?:^|\s+)(?:class|struct)\s+(?:[a-z]|_)\w+")
    bad_naming = lowercase.search(code)
    if bad_naming:
        result = bad_naming.group(0).split()
        self.add_error(label="FIRST_CHAR",
                       data={"keyword": result[0],
                             "expected": str(result[1]).capitalize(),
                             "found": str(result[1])})

def check_unnecessary_include(self, code):
    grammar = Literal('#') + Literal('include') + Literal('<') + Word(alphanums)
    try:
        grammar.parseString(code)
        begin = code.find("<")
        end = code.find(">")
        included_library = code[begin + 1:end]
        if included_library not in self.includes:
            self.add_error(label="UNNECESSARY_INCLUDE")
    except ParseException:
        return

def check_local_include(self, code):
    grammar = Literal('#') + Literal('include') + Literal('"') + Word(alphanums)
    try:
        grammar.parseString(code)
        begin = code.find('"')
        included_file = code[begin + 1:]
        end = included_file.find('"')
        included_file = included_file[:end]
        if included_file not in self.includes:
            self.local_includes[self.current_file].append(included_file)
    except ParseException:
        return

def check_for_loop_semicolon_spacing(self, code):
    # Match the semicolons and any whitespace around them.
    for_loop_regex = re.compile(
        r"""
        \s*for\s*\(
            (?P<code1>[^;]*?)

            (?P<semicolon1>\s*;\s*)

            (?P<code2>[^;]*?)

            (?P<semicolon2>\s*;\s*)

            (?P<code3>[^;]*?)
        \)
        """,
        re.VERBOSE
    )
    match = for_loop_regex.search(code)
    if not match:
        return

    self.for_loop_spacing_before = getattr(self, "for_loop_spacing_before", None)
    self.for_loop_spacing_after = getattr(self, "for_loop_spacing_after", None)

    semicolon1 = match.group("semicolon1")
    semicolon2 = match.group("semicolon2")
    code1 = match.group("code1")
    code2 = match.group("code2")
    code3 = match.group("code3")

    def is_spacing_okay(semicolon, before_code, after_code):
        spacing_before = semicolon.startswith(" ")
        spacing_after = semicolon.endswith(" ")

        def check_spacing(convention, actual):
            if convention is None:
                convention = actual

            if convention != actual:
                return convention, False
            else:
                return convention, True

        if before_code or after_code:
            if before_code:
                self.for_loop_spacing_before, result = check_spacing(
                    self.for_loop_spacing_before,
                    spacing_before
                )
                if not result:
                    return False
            if after_code:
                self.for_loop_spacing_after, result = check_spacing(
                    self.for_loop_spacing_after,
                    spacing_after
                )
                if not result:
                    return False
        else:
            # This is a plain semicolon, so we can't infer anything about the
            # spacing convention.
            pass
        return True

    if not (
                is_spacing_okay(semicolon1, code1, code2)
            and is_spacing_okay(semicolon2, code2, code3)
    ):
        self.add_error(
            label="FOR_LOOP_SEMICOLON_SPACING",
            data={"line": self.current_line_num}
        )
<|endoftext|>


<commit_before>import math
import numpy as np
from scipy import integrate
import cantera as ct
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import re
import warnings
"""

Setup of System

"""

def get_initial_mole_fractions(stoich_ratio, fuel_mole_ratios, oxygen_per_fuel_at_stoic_list):
    """
    this method obtains your initial mole fractions for fuel in air.
    the product is returned as a list with nitrogen, oxygen, and then
    the fuels are listed.

    stoich_ratio = oxygen to fuel stoichiometric ratio in the system
    fuel_mole_ratios = list of molar ratios of various fuels (must sum to 1)
    oxygen_per_fuel_at_stoic_list = a list containing the number of oxygen
            molecules necessary for full combustion of each molecule. For
            example, Methane would be 2.
    """
    #errror check
    np.testing.assert_allclose(sum(fuel_mole_ratios),1.)#,"mole ratios of fuels needs to add to one")
    assert len(fuel_mole_ratios) ==len(oxygen_per_fuel_at_stoic_list)

    combined_oxygen_per_fuel = np.sum(np.multiply(fuel_mole_ratios,oxygen_per_fuel_at_stoic_list))

    total_fuel = sum(fuel_mole_ratios)
    total_oxygen = total_fuel * combined_oxygen_per_fuel / stoich_ratio
    total_nitrogen = total_oxygen * .79/.21
    total_species = total_fuel + total_oxygen + total_nitrogen
    mole_fractions = np.concatenate(([total_nitrogen, total_oxygen],fuel_mole_ratios),0)
    return mole_fractions/total_species

"""

Running System

"""

def append_data_to_df(simulator,solution, df, basics= ['time','temperature','pressure','density'],
                      add_species = True, species_names='all',
                      add_rxns = False, reaction_names='all'):
    """
    appends the current conditions of a Solution object contianing ReactorNet
    object (simulator) to the pandas.dataframe object (df).

    The optional parameters define what is saved.

    The following are enabled for the basics conditions:
    * time
    * temperature
    * pressure
    * density
    * volume
    * cp (constant pressure heat capacity)
    * cv (constant volume heat capacity)
    """

    conditions = {}
    # add regular conditions
    if 'time' in basics:
        conditions['time (s)'] = simulator.time
    if 'temperature' in basics:
        conditions['temperature (K)'] = solution.T
    if 'pressure' in basics:
        conditions['pressure (Pa)'] = solution.P
    if 'density' in basics:
        conditions['density (kmol/m3)'] = solution.density_mole
    if 'volume' in basics:
        conditions['volume (m3)'] = solution.volume_mole
    if 'cp' in basics:
        conditions['heat capacity, cp (J/kmol/K)'] = solution.cp_mole
    if 'cv' in basics:
        conditions['heat capacity, cp (J/kmol/K)'] = solution.cv_mole
#    if '' in basics:
#        conditions[''] = solution.
    # end adding regular conditions

    if add_species:
        if species_names=='all':
            species_names = solution.species_names
        mole_fractions = solution.mole_fraction_dict()
        for name in species_names:
            try:
                conditions[name] = mole_fractions[name] * solution.density_mole
            except KeyError:
                conditions[name] = 0
                warnings.warn('%s is not listed in the mole fraction dictionary. If this occurs after the first iteration, you may have typed the species incorrectly')

    if add_rxns:
        if reaction_names=='all':
            reaction_names = solution.reaction_equations()

        reaction_rates = get_rxn_rate_dict(solution.reaction_equations(),solution.net_rates_of_progress)
        for name in reaction_names:
            try:
                conditions[name] = reaction_rates[name]
            except KeyError:
                conditions[name] = 0
                warnings.warn('%s is not listed in the reaction names. If this occurs after the first iteration, you may have typed the species incorrectly')

    return df.append(conditions, ignore_index=True)

def get_rxn_rate_dict(reaction_equations, net_rates):
    """
    makes a dictionary out of the two inputs, adding together duplicate
    reactions
    """
    rxn_dict = {}
    for equation, rate in zip(reaction_equations, net_rates):
        try:
            rxn_dict[equation] += rate
        except KeyError:
            rxn_dict[equation] = rate
    return rxn_dict
"""

Mechanism Reduction

"""

def reduce_reactions_in_mechanism(reaction_list, kept_reaction_equations):
    """
    finds reactions that match the form of the reaction equations in
    kept_reaction_equations. It returns just the reactions that are meant
    to be in the mechanism
    """
    reduced_reaction_list = []
    for reaction in reaction_list:
        if reaction.equation in kept_reaction_equations:
            reduced_reaction_list.append(reaction)
    return reduced_reaction_list

def eliminate_species_from_mechanism(species_list, kept_reactions,inert_species):
    """
    finds all the species in kept_reactions, and returns a list of species
    objects of those species. inert_species are automatically kept.
    """

    reacting_species = []
    for reaction in kept_reactions:
        reacting_species += reaction.reactants.keys() + reaction.products.keys()
    # remove duplicates and add inert
    reduced_species_name_list = list(set(reacting_species)) + inert_species

    reduced_species_list = []
    for species in species_list:
        if species.name in reduced_species_name_list:
            reduced_species_list.append(species)

    return reduced_species_list

def create_mechanism(full_model_file_path,kept_reaction_equations='all',non_reactive_species = ['AR','N2','HE']):
    """
    input the full model and a list of reaction equations that you'd like to keep.
    returns a Cantera.Solution object of the mechanism with only the cooresponding
    reactions and their species.
    """
    desired_file = full_model_file_path
    spec = ct.Species.listFromFile(desired_file)
    rxns = ct.Reaction.listFromFile(desired_file)

    if kept_reaction_equations=='all':
        return ct.Solution(thermo='IdealGas', kinetics='GasKinetics',
                      species=spec, reactions=rxns)
    else:
        reduced_reactions = reduce_reactions_in_mechanism(rxns,kept_reaction_equations)
        reduced_species = eliminate_species_from_mechanism(spec,reduced_reactions,non_reactive_species)
        return ct.Solution(thermo='IdealGas', kinetics='GasKinetics',
                          species=reduced_species, reactions=reduced_reactions)
<commit_msg>refactored cantera_tools_get_initial_conditions to optionally output a dictionary of conditions<commit_after>import math
import numpy as np
from scipy import integrate
import cantera as ct
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import re
import warnings
"""

Setup of System

"""

def get_initial_mole_fractions(stoich_ratio, fuel_mole_ratios, oxygen_per_fuel_at_stoic_list, fuels = None):
    """
    this method obtains your initial mole fractions for fuel in air.
    the product is returned as a list with nitrogen, oxygen, and then
    the fuels are listed.

    stoich_ratio = oxygen to fuel stoichiometric ratio in the system
    fuels = list of strings for output dictionary. If ommitted, a list is returned
    fuel_mole_ratios = list of molar ratios of various fuels (must sum to 1)
    oxygen_per_fuel_at_stoic_list = a list containing the number of oxygen
            molecules necessary for full combustion of each molecule. For
            example, Methane would be 2.
    """
    #errror check
    np.testing.assert_allclose(sum(fuel_mole_ratios),1.)#,"mole ratios of fuels needs to add to one")
    assert len(fuel_mole_ratios) ==len(oxygen_per_fuel_at_stoic_list)

    combined_oxygen_per_fuel = np.sum(np.multiply(fuel_mole_ratios,oxygen_per_fuel_at_stoic_list))

    total_fuel = sum(fuel_mole_ratios)
    total_oxygen = total_fuel * combined_oxygen_per_fuel / stoich_ratio
    total_nitrogen = total_oxygen * .79/.21
    total_species = total_fuel + total_oxygen + total_nitrogen
    mole_fractions = np.concatenate(([total_nitrogen, total_oxygen],fuel_mole_ratios),0)
    normalized_mole_fractions = mole_fractions/total_species
    if fuels:
        fuel_zip =  [(fuels[index],normalized_mole_fractions[index+2]) for index in range(len(fuels))]
        air_zip = [('N2',normalized_mole_fractions[0]),('O2',normalized_mole_fractions[1])]
        mole_fraction_dictionary = dict(air_zip+fuel_zip)
        return mole_fraction_dictionary
    else:
        return normalized_mole_fractions
"""

Running System

"""

def append_data_to_df(simulator,solution, df, basics= ['time','temperature','pressure','density'],
                      add_species = True, species_names='all',
                      add_rxns = False, reaction_names='all'):
    """
    appends the current conditions of a Solution object contianing ReactorNet
    object (simulator) to the pandas.dataframe object (df).

    The optional parameters define what is saved.

    The following are enabled for the basics conditions:
    * time
    * temperature
    * pressure
    * density
    * volume
    * cp (constant pressure heat capacity)
    * cv (constant volume heat capacity)
    """

    conditions = {}
    # add regular conditions
    if 'time' in basics:
        conditions['time (s)'] = simulator.time
    if 'temperature' in basics:
        conditions['temperature (K)'] = solution.T
    if 'pressure' in basics:
        conditions['pressure (Pa)'] = solution.P
    if 'density' in basics:
        conditions['density (kmol/m3)'] = solution.density_mole
    if 'volume' in basics:
        conditions['volume (m3)'] = solution.volume_mole
    if 'cp' in basics:
        conditions['heat capacity, cp (J/kmol/K)'] = solution.cp_mole
    if 'cv' in basics:
        conditions['heat capacity, cp (J/kmol/K)'] = solution.cv_mole
#    if '' in basics:
#        conditions[''] = solution.
    # end adding regular conditions

    if add_species:
        if species_names=='all':
            species_names = solution.species_names
        mole_fractions = solution.mole_fraction_dict()
        for name in species_names:
            try:
                conditions[name] = mole_fractions[name] * solution.density_mole
            except KeyError:
                conditions[name] = 0
                warnings.warn('%s is not listed in the mole fraction dictionary. If this occurs after the first iteration, you may have typed the species incorrectly')

    if add_rxns:
        if reaction_names=='all':
            reaction_names = solution.reaction_equations()

        reaction_rates = get_rxn_rate_dict(solution.reaction_equations(),solution.net_rates_of_progress)
        for name in reaction_names:
            try:
                conditions[name] = reaction_rates[name]
            except KeyError:
                conditions[name] = 0
                warnings.warn('%s is not listed in the reaction names. If this occurs after the first iteration, you may have typed the species incorrectly')

    return df.append(conditions, ignore_index=True)

def get_rxn_rate_dict(reaction_equations, net_rates):
    """
    makes a dictionary out of the two inputs, adding together duplicate
    reactions
    """
    rxn_dict = {}
    for equation, rate in zip(reaction_equations, net_rates):
        try:
            rxn_dict[equation] += rate
        except KeyError:
            rxn_dict[equation] = rate
    return rxn_dict
"""

Mechanism Reduction

"""

def reduce_reactions_in_mechanism(reaction_list, kept_reaction_equations):
    """
    finds reactions that match the form of the reaction equations in
    kept_reaction_equations. It returns just the reactions that are meant
    to be in the mechanism
    """
    reduced_reaction_list = []
    for reaction in reaction_list:
        if reaction.equation in kept_reaction_equations:
            reduced_reaction_list.append(reaction)
    return reduced_reaction_list

def eliminate_species_from_mechanism(species_list, kept_reactions,inert_species):
    """
    finds all the species in kept_reactions, and returns a list of species
    objects of those species. inert_species are automatically kept.
    """

    reacting_species = []
    for reaction in kept_reactions:
        reacting_species += reaction.reactants.keys() + reaction.products.keys()
    # remove duplicates and add inert
    reduced_species_name_list = list(set(reacting_species)) + inert_species

    reduced_species_list = []
    for species in species_list:
        if species.name in reduced_species_name_list:
            reduced_species_list.append(species)

    return reduced_species_list

def create_mechanism(full_model_file_path,kept_reaction_equations='all',non_reactive_species = ['AR','N2','HE']):
    """
    input the full model and a list of reaction equations that you'd like to keep.
    returns a Cantera.Solution object of the mechanism with only the cooresponding
    reactions and their species.
    """
    desired_file = full_model_file_path
    spec = ct.Species.listFromFile(desired_file)
    rxns = ct.Reaction.listFromFile(desired_file)

    if kept_reaction_equations=='all':
        return ct.Solution(thermo='IdealGas', kinetics='GasKinetics',
                      species=spec, reactions=rxns)
    else:
        reduced_reactions = reduce_reactions_in_mechanism(rxns,kept_reaction_equations)
        reduced_species = eliminate_species_from_mechanism(spec,reduced_reactions,non_reactive_species)
        return ct.Solution(thermo='IdealGas', kinetics='GasKinetics',
                          species=reduced_species, reactions=reduced_reactions)
<|endoftext|>