<commit_before>import queue
import multiprocessing
import traceback

from coalib.bears.BEAR_KIND import BEAR_KIND
from coalib.bears.GlobalBear import GlobalBear
from coalib.bears.LocalBear import LocalBear
from coalib.misc.StringConstants import StringConstants
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.processes.communication.LogMessage import LogMessage, LOG_LEVEL
from coalib.misc.i18n import _
from coalib.results.Result import Result

def send_msg(message_queue, timeout, log_level, *args, delimiter=' ', end=''):
    """
    Puts message into message queue for a LogPrinter to present to the user.

    :param message_queue: The queue to put the message into and which the
                          LogPrinter reads.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param log_level:     The log_level i.e Error,Debug or Warning.It is sent
                          to the LogPrinter depending on the message.
    :param args:          This includes the elements of the message.
    :param delimiter:     It is the value placed between each arg. By default
                          it is a ' '.
    :param end:           It is the value placed at the end of the message.
    """
    output = str(delimiter).join(str(arg) for arg in args) + str(end)
    message_queue.put(LogMessage(log_level, output),
                      timeout=timeout)

def validate_results(message_queue, timeout, result_list, name, args, kwargs):
    """
    Validates if the result_list passed to it contains valid set of results.
    That is the result_list must itself be a list and contain objects of the
    instance of Result object. If any irregularity is found a message is put in
    the message_queue to present the irregularity to the user. Each result_list
    belongs to an execution of a bear.

    :param message_queue: A queue that contains messages of type
                          errors/warnings/debug statements to be printed in the
                          Log.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param result_list:   The list of results to validate.
    :param name:          The name of the bear executed.
    :param args:          The args with which the bear was executed.
    :param kwargs:        The kwargs with which the bear was executed.
    :return:              Returns None if the result_list is invalid. Else it
                          returns the result_list itself.
    """
    if result_list is None:
        return None

    if not isinstance(result_list, list):
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.ERROR,
                 _("The results from the bear {bear} couldn't be processed "
                   "with arguments {arglist}, {kwarglist}.")
                 .format(bear=name, arglist=args, kwarglist=kwargs))
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.DEBUG,
                 _("The return value of the {bear} is an instance of {ret}"
                   " but should be an instance of list.")
                 .format(bear=name, ret=result_list.__class__))
        return None

    for result in result_list:
        if not isinstance(result, Result):
            send_msg(message_queue,
                     timeout,
                     LOG_LEVEL.ERROR,
                     _("The results from the bear {bear} could only be "
                       "partially processed with arguments {arglist}, "
                       "{kwarglist}")
                     .format(bear=name, arglist=args, kwarglist=kwargs))
            send_msg(message_queue,
                     timeout,
                     LOG_LEVEL.DEBUG,
                     _("One of the results in the list for the bear {bear} is "
                       "an instance of {ret} but it should be an instance of "
                       "Result")
                     .format(bear=name, ret=result.__class__))
            result_list.remove(result)

    return result_list

def run_bear(message_queue, timeout, bear_instance, *args, **kwargs):
    """
    This method is responsible for executing the instance of a bear. It also
    reports or logs errors if any occur during the execution of that bear
    instance.

    :param message_queue: A queue that contains messages of type
                          errors/warnings/debug statements to be printed in the
                          Log.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param bear_instance: The instance of the bear to be executed.
    :param args:          The arguments that are to be passed to the bear.
    :param kwargs:        The keyword arguments that are to be passed to the
                          bear.
    :return:              Returns a valid list of objects of the type Result
                          if the bear executed succesfully. None otherwise.
    """
    if kwargs.get("dependency_results", True) is None:
        del kwargs["dependency_results"]

    name = bear_instance.__class__.__name__

    try:
        result_list = bear_instance.execute(*args,
                                            **kwargs)
    except:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.ERROR,
                 _("The bear {bear} failed to run with the arguments "
                   "{arglist}, {kwarglist}. Skipping bear...")
                 .format(bear=name, arglist=args, kwarglist=kwargs))
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.DEBUG,
                 _("Traceback for error in bear {bear}:")
                 .format(bear=name),
                 traceback.format_exc(),
                 delimiter="\n")

        return None

    return validate_results(message_queue,
                            timeout,
                            result_list,
                            name,
                            args,
                            kwargs)

def get_local_dependency_results(local_result_list, bear_instance):
    """
    This method gets all the results originating from the dependencies of a
    bear_instance. Each bear_instance may or may not have dependencies.

    :param local_result_list: The list of results out of which the dependency
                              results are picked.
    :param bear_instance:     The instance of a local bear to get the
                              dependencies from.
    :return:                  Return none if their are no dependencies for the
                              bear. Else return a dictionary containing
                              dependency results.
    """
    deps = bear_instance.get_dependencies()
    if deps == []:
        return None

    dependency_results = {}
    dep_strings = []
    for dep in deps:
        dep_strings.append(dep.__name__)

    for result in local_result_list:
        if result.origin in dep_strings:
            results = dependency_results.get(result.origin, [])
            results.append(result)
            dependency_results[result.origin] = results

    return dependency_results

def run_local_bear(message_queue,
                   timeout,
                   local_result_list,
                   file_dict,
                   bear_instance,
                   filename):
    """
    Runs an instance of a local bear. Checks if bear_instance is of type
    LocalBear and then passes it to the run_bear to execute.

    :param message_queue:     A queue that contains messages of type
                              errors/warnings/debug statements to be printed in
                              the Log.
    :param timeout:           The queue blocks at most timeout seconds for a
                              free slot to execute the put operation on. After
                              the timeout it returns queue Full exception.
    :param local_result_list: Its a list that stores the results of all local
                              bears.
    :param file_dict:         Dictionary containing contents of file.
    :param bear_instance:     Instance of LocalBear the run.
    :param filename:          Name of the file to run it on.
    :return:                  Returns a list of results generated by the passed
                              bear_instance.
    """
    if not isinstance(bear_instance, LocalBear) or \
            bear_instance.kind() != BEAR_KIND.LOCAL:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.WARNING,
                 _("A given local bear ({}) is not valid. Leaving "
                   "it out...").format(bear_instance.__class__.__name__),
                 StringConstants.THIS_IS_A_BUG)

        return None

    kwargs = {"dependency_results":
                  get_local_dependency_results(local_result_list,
                                               bear_instance)}
    return run_bear(message_queue,
                    timeout,
                    bear_instance,
                    filename,
                    file_dict[filename],
                    **kwargs)

def run_global_bear(message_queue,
                    timeout,
                    global_bear_instance,
                    dependency_results):
    """
    Runs an instance of a global bear. Checks if bear_instance is of type
    GlobalBear and then passes it to the run_bear to execute.

    :param message_queue:        A queue that contains messages of type
                                 errors/warnings/debug statements to be printed
                                 in the Log.
    :param timeout:              The queue blocks at most timeout seconds for a
                                 free slot to execute the put operation on.
                                 After the timeout it returns queue Full
                                 exception.
    :param global_bear_instance: Instance of GlobalBear to run.
    :param dependency_results:   The results of all the bears on which the
                                 instance of the passed bear to be run depends
                                 on.
    :return:                     Returns a list of results generated by the
                                 passed bear_instance.
    """
    if not isinstance(global_bear_instance, GlobalBear) \
            or global_bear_instance.kind() != BEAR_KIND.GLOBAL:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.WARNING,
                 _("A given global bear ({}) is not valid. Leaving it "
                   "out...")
                 .format(global_bear_instance.__class__.__name__),
                 StringConstants.THIS_IS_A_BUG)

        return None

    kwargs = {"dependency_results": dependency_results}
    return run_bear(message_queue,
                    timeout,
                    global_bear_instance,
                    **kwargs)

class BearRunner(multiprocessing.Process):
    def __init__(self,
                 file_name_queue,
                 local_bear_list,
                 global_bear_list,
                 global_bear_queue,
                 file_dict,
                 local_result_dict,
                 global_result_dict,
                 message_queue,
                 control_queue,
                 TIMEOUT=0):
        """
        This is the object that actually runs on the processes

        If parameters type is 'queue (read)' this means it has to implement the
        get(timeout=TIMEOUT) method and it shall raise queue.Empty if the queue
        is empty up until the end of the timeout. If the queue has the
        (optional!) task_done() attribute, BearRunner will call it after
        processing each item.

        If parameters type is 'queue (write)' it shall implement the
        put(object, timeout=TIMEOUT) method.

        If the queues raise any exception not specified here the user will get
        an 'unknown error' message. So beware of that.

        :param file_name_queue:    queue (read) of file names to check with
                                   local bears. Every BearRunner takes one of
                                   those and checks it with all local bears.
                                   (Repeat until queue empty.)
        :param local_bear_list:    list of local bear instances
        :param global_bear_list:   list of global bear instances
        :param global_bear_queue:  queue (read, write) of indexes of global
                                   bear instances in the global_bear_list
        :param file_dict:          dict of all files as {filename:file}, file
                                   as in file.readlines()
        :param local_result_dict:  A Manager.dict that will be used to store
                                   local results. A list of all local results
                                   will be stored with the filename as key.
        :param global_result_dict: A Manager.dict that will be used to store
                                   global results. The list of results of one
                                   global bear will be stored with the bear
                                   name as key.
        :param message_queue:      queue (write) for debug/warning/error
                                   messages (type LogMessage)
        :param control_queue:      queue (write). If any result gets written to
                                   the result_dict a tuple containing a
                                   CONTROL_ELEMENT (to indicate what kind of
                                   event happened) and either a bear name
                                   (for global results) or a file name to
                                   indicate the result will be put to the
                                   queue. If this BearRunner finished all its
                                   local bears it will put
                                   (CONTROL_ELEMENT.LOCAL_FINISHED, None) to
                                   the queue, if it finished all global ones,
                                   (CONTROL_ELEMENT.GLOBAL_FINISHED, None) will
                                   be put there.
        :param TIMEOUT:            in seconds for all queue actions
        """
        multiprocessing.Process.__init__(self)

        self.filename_queue = file_name_queue
        self.local_bear_list = local_bear_list
        self.global_bear_queue = global_bear_queue
        self.global_bear_list = global_bear_list

        self.file_dict = file_dict

        self.local_result_dict = local_result_dict
        self.global_result_dict = global_result_dict
        self.message_queue = message_queue
        self.control_queue = control_queue

        self.TIMEOUT = TIMEOUT

        # Will be used to hold local results when they are not yet stored in
        # the result dict
        self._local_result_list = []

    def run(self):
        self.run_local_bears()
        self.control_queue.put((CONTROL_ELEMENT.LOCAL_FINISHED, None))

        self.run_global_bears()
        self.control_queue.put((CONTROL_ELEMENT.GLOBAL_FINISHED, None))

    def run_local_bears(self):
        try:
            while True:
                filename = self.filename_queue.get(timeout=self.TIMEOUT)
                self.__run_local_bears(filename)
                if hasattr(self.filename_queue, "task_done"):
                    self.filename_queue.task_done()
        except queue.Empty:
            return

    def _get_global_dependency_results(self, bear_instance):
        """
        Retreives dependency results for a global bear.

        :return: None if bear has no dependencies, False if dependencies are
        not met, the dependency dict otherwise.
        """
        try:
            deps = bear_instance.get_dependencies()
            if deps == []:
                return None
        except AttributeError:
            # When this occurs we have an invalid bear and a warning will be
            # emitted later.
            return None

        dependency_results = {}
        for dep in deps:
            depname = dep.__name__
            if depname not in self.global_result_dict:
                return False

            dependency_results[depname] = self.global_result_dict[depname]

        return dependency_results

    def _get_next_global_bear(self):
        """
        Retrieves the next global bear.

        :return: (bear, bearname, dependency_results)
        """
        dependency_results = False

        while dependency_results is False:
            bear_id = self.global_bear_queue.get(timeout=self.TIMEOUT)
            bear = self.global_bear_list[bear_id]

            dependency_results = self._get_global_dependency_results(bear)
            if dependency_results is False:
                self.global_bear_queue.put(bear_id)

        return bear, bear.__class__.__name__, dependency_results

    def run_global_bears(self):
        try:
            while True:
                bear, bearname, dep_results = self._get_next_global_bear()
                result = run_global_bear(self.message_queue,
                                         self.TIMEOUT,
                                         bear,
                                         dep_results)
                if result:
                    self.global_result_dict[bearname] = result
                    self.control_queue.put((CONTROL_ELEMENT.GLOBAL,
                                            bearname))
                else:
                    self.global_result_dict[bearname] = None
                if hasattr(self.global_bear_queue, "task_done"):
                    self.global_bear_queue.task_done()
        except queue.Empty:
            return

    def __run_local_bears(self, filename):
        if filename not in self.file_dict:
            send_msg(self.message_queue,
                     self.TIMEOUT,
                     LOG_LEVEL.ERROR,
                     _("An internal error occurred."),
                     StringConstants.THIS_IS_A_BUG)
            send_msg(self.message_queue,
                     self.TIMEOUT,
                     LOG_LEVEL.DEBUG,
                     _("The given file through the queue is not in the file "
                       "dictionary."))

            return

        self._local_result_list = []
        for bear_instance in self.local_bear_list:
            r = run_local_bear(self.message_queue,
                               self.TIMEOUT,
                               self._local_result_list,
                               self.file_dict,
                               bear_instance,
                               filename)
            if r is not None:
                self._local_result_list.extend(r)

        self.local_result_dict[filename] = self._local_result_list
        self.control_queue.put((CONTROL_ELEMENT.LOCAL, filename))
<commit_msg>BearRunner: __run_local_bears -> run_local_bears_on_file<commit_after>import queue
import multiprocessing
import traceback

from coalib.bears.BEAR_KIND import BEAR_KIND
from coalib.bears.GlobalBear import GlobalBear
from coalib.bears.LocalBear import LocalBear
from coalib.misc.StringConstants import StringConstants
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.processes.communication.LogMessage import LogMessage, LOG_LEVEL
from coalib.misc.i18n import _
from coalib.results.Result import Result

def send_msg(message_queue, timeout, log_level, *args, delimiter=' ', end=''):
    """
    Puts message into message queue for a LogPrinter to present to the user.

    :param message_queue: The queue to put the message into and which the
                          LogPrinter reads.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param log_level:     The log_level i.e Error,Debug or Warning.It is sent
                          to the LogPrinter depending on the message.
    :param args:          This includes the elements of the message.
    :param delimiter:     It is the value placed between each arg. By default
                          it is a ' '.
    :param end:           It is the value placed at the end of the message.
    """
    output = str(delimiter).join(str(arg) for arg in args) + str(end)
    message_queue.put(LogMessage(log_level, output),
                      timeout=timeout)

def validate_results(message_queue, timeout, result_list, name, args, kwargs):
    """
    Validates if the result_list passed to it contains valid set of results.
    That is the result_list must itself be a list and contain objects of the
    instance of Result object. If any irregularity is found a message is put in
    the message_queue to present the irregularity to the user. Each result_list
    belongs to an execution of a bear.

    :param message_queue: A queue that contains messages of type
                          errors/warnings/debug statements to be printed in the
                          Log.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param result_list:   The list of results to validate.
    :param name:          The name of the bear executed.
    :param args:          The args with which the bear was executed.
    :param kwargs:        The kwargs with which the bear was executed.
    :return:              Returns None if the result_list is invalid. Else it
                          returns the result_list itself.
    """
    if result_list is None:
        return None

    if not isinstance(result_list, list):
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.ERROR,
                 _("The results from the bear {bear} couldn't be processed "
                   "with arguments {arglist}, {kwarglist}.")
                 .format(bear=name, arglist=args, kwarglist=kwargs))
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.DEBUG,
                 _("The return value of the {bear} is an instance of {ret}"
                   " but should be an instance of list.")
                 .format(bear=name, ret=result_list.__class__))
        return None

    for result in result_list:
        if not isinstance(result, Result):
            send_msg(message_queue,
                     timeout,
                     LOG_LEVEL.ERROR,
                     _("The results from the bear {bear} could only be "
                       "partially processed with arguments {arglist}, "
                       "{kwarglist}")
                     .format(bear=name, arglist=args, kwarglist=kwargs))
            send_msg(message_queue,
                     timeout,
                     LOG_LEVEL.DEBUG,
                     _("One of the results in the list for the bear {bear} is "
                       "an instance of {ret} but it should be an instance of "
                       "Result")
                     .format(bear=name, ret=result.__class__))
            result_list.remove(result)

    return result_list

def run_bear(message_queue, timeout, bear_instance, *args, **kwargs):
    """
    This method is responsible for executing the instance of a bear. It also
    reports or logs errors if any occur during the execution of that bear
    instance.

    :param message_queue: A queue that contains messages of type
                          errors/warnings/debug statements to be printed in the
                          Log.
    :param timeout:       The queue blocks at most timeout seconds for a free
                          slot to execute the put operation on. After the
                          timeout it returns queue Full exception.
    :param bear_instance: The instance of the bear to be executed.
    :param args:          The arguments that are to be passed to the bear.
    :param kwargs:        The keyword arguments that are to be passed to the
                          bear.
    :return:              Returns a valid list of objects of the type Result
                          if the bear executed succesfully. None otherwise.
    """
    if kwargs.get("dependency_results", True) is None:
        del kwargs["dependency_results"]

    name = bear_instance.__class__.__name__

    try:
        result_list = bear_instance.execute(*args,
                                            **kwargs)
    except:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.ERROR,
                 _("The bear {bear} failed to run with the arguments "
                   "{arglist}, {kwarglist}. Skipping bear...")
                 .format(bear=name, arglist=args, kwarglist=kwargs))
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.DEBUG,
                 _("Traceback for error in bear {bear}:")
                 .format(bear=name),
                 traceback.format_exc(),
                 delimiter="\n")

        return None

    return validate_results(message_queue,
                            timeout,
                            result_list,
                            name,
                            args,
                            kwargs)

def get_local_dependency_results(local_result_list, bear_instance):
    """
    This method gets all the results originating from the dependencies of a
    bear_instance. Each bear_instance may or may not have dependencies.

    :param local_result_list: The list of results out of which the dependency
                              results are picked.
    :param bear_instance:     The instance of a local bear to get the
                              dependencies from.
    :return:                  Return none if their are no dependencies for the
                              bear. Else return a dictionary containing
                              dependency results.
    """
    deps = bear_instance.get_dependencies()
    if deps == []:
        return None

    dependency_results = {}
    dep_strings = []
    for dep in deps:
        dep_strings.append(dep.__name__)

    for result in local_result_list:
        if result.origin in dep_strings:
            results = dependency_results.get(result.origin, [])
            results.append(result)
            dependency_results[result.origin] = results

    return dependency_results

def run_local_bear(message_queue,
                   timeout,
                   local_result_list,
                   file_dict,
                   bear_instance,
                   filename):
    """
    Runs an instance of a local bear. Checks if bear_instance is of type
    LocalBear and then passes it to the run_bear to execute.

    :param message_queue:     A queue that contains messages of type
                              errors/warnings/debug statements to be printed in
                              the Log.
    :param timeout:           The queue blocks at most timeout seconds for a
                              free slot to execute the put operation on. After
                              the timeout it returns queue Full exception.
    :param local_result_list: Its a list that stores the results of all local
                              bears.
    :param file_dict:         Dictionary containing contents of file.
    :param bear_instance:     Instance of LocalBear the run.
    :param filename:          Name of the file to run it on.
    :return:                  Returns a list of results generated by the passed
                              bear_instance.
    """
    if not isinstance(bear_instance, LocalBear) or \
            bear_instance.kind() != BEAR_KIND.LOCAL:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.WARNING,
                 _("A given local bear ({}) is not valid. Leaving "
                   "it out...").format(bear_instance.__class__.__name__),
                 StringConstants.THIS_IS_A_BUG)

        return None

    kwargs = {"dependency_results":
                  get_local_dependency_results(local_result_list,
                                               bear_instance)}
    return run_bear(message_queue,
                    timeout,
                    bear_instance,
                    filename,
                    file_dict[filename],
                    **kwargs)

def run_global_bear(message_queue,
                    timeout,
                    global_bear_instance,
                    dependency_results):
    """
    Runs an instance of a global bear. Checks if bear_instance is of type
    GlobalBear and then passes it to the run_bear to execute.

    :param message_queue:        A queue that contains messages of type
                                 errors/warnings/debug statements to be printed
                                 in the Log.
    :param timeout:              The queue blocks at most timeout seconds for a
                                 free slot to execute the put operation on.
                                 After the timeout it returns queue Full
                                 exception.
    :param global_bear_instance: Instance of GlobalBear to run.
    :param dependency_results:   The results of all the bears on which the
                                 instance of the passed bear to be run depends
                                 on.
    :return:                     Returns a list of results generated by the
                                 passed bear_instance.
    """
    if not isinstance(global_bear_instance, GlobalBear) \
            or global_bear_instance.kind() != BEAR_KIND.GLOBAL:
        send_msg(message_queue,
                 timeout,
                 LOG_LEVEL.WARNING,
                 _("A given global bear ({}) is not valid. Leaving it "
                   "out...")
                 .format(global_bear_instance.__class__.__name__),
                 StringConstants.THIS_IS_A_BUG)

        return None

    kwargs = {"dependency_results": dependency_results}
    return run_bear(message_queue,
                    timeout,
                    global_bear_instance,
                    **kwargs)

class BearRunner(multiprocessing.Process):
    def __init__(self,
                 file_name_queue,
                 local_bear_list,
                 global_bear_list,
                 global_bear_queue,
                 file_dict,
                 local_result_dict,
                 global_result_dict,
                 message_queue,
                 control_queue,
                 TIMEOUT=0):
        """
        This is the object that actually runs on the processes

        If parameters type is 'queue (read)' this means it has to implement the
        get(timeout=TIMEOUT) method and it shall raise queue.Empty if the queue
        is empty up until the end of the timeout. If the queue has the
        (optional!) task_done() attribute, BearRunner will call it after
        processing each item.

        If parameters type is 'queue (write)' it shall implement the
        put(object, timeout=TIMEOUT) method.

        If the queues raise any exception not specified here the user will get
        an 'unknown error' message. So beware of that.

        :param file_name_queue:    queue (read) of file names to check with
                                   local bears. Every BearRunner takes one of
                                   those and checks it with all local bears.
                                   (Repeat until queue empty.)
        :param local_bear_list:    list of local bear instances
        :param global_bear_list:   list of global bear instances
        :param global_bear_queue:  queue (read, write) of indexes of global
                                   bear instances in the global_bear_list
        :param file_dict:          dict of all files as {filename:file}, file
                                   as in file.readlines()
        :param local_result_dict:  A Manager.dict that will be used to store
                                   local results. A list of all local results
                                   will be stored with the filename as key.
        :param global_result_dict: A Manager.dict that will be used to store
                                   global results. The list of results of one
                                   global bear will be stored with the bear
                                   name as key.
        :param message_queue:      queue (write) for debug/warning/error
                                   messages (type LogMessage)
        :param control_queue:      queue (write). If any result gets written to
                                   the result_dict a tuple containing a
                                   CONTROL_ELEMENT (to indicate what kind of
                                   event happened) and either a bear name
                                   (for global results) or a file name to
                                   indicate the result will be put to the
                                   queue. If this BearRunner finished all its
                                   local bears it will put
                                   (CONTROL_ELEMENT.LOCAL_FINISHED, None) to
                                   the queue, if it finished all global ones,
                                   (CONTROL_ELEMENT.GLOBAL_FINISHED, None) will
                                   be put there.
        :param TIMEOUT:            in seconds for all queue actions
        """
        multiprocessing.Process.__init__(self)

        self.filename_queue = file_name_queue
        self.local_bear_list = local_bear_list
        self.global_bear_queue = global_bear_queue
        self.global_bear_list = global_bear_list

        self.file_dict = file_dict

        self.local_result_dict = local_result_dict
        self.global_result_dict = global_result_dict
        self.message_queue = message_queue
        self.control_queue = control_queue

        self.TIMEOUT = TIMEOUT

        # Will be used to hold local results when they are not yet stored in
        # the result dict
        self._local_result_list = []

    def run(self):
        self.run_local_bears()
        self.control_queue.put((CONTROL_ELEMENT.LOCAL_FINISHED, None))

        self.run_global_bears()
        self.control_queue.put((CONTROL_ELEMENT.GLOBAL_FINISHED, None))

    def run_local_bears(self):
        try:
            while True:
                filename = self.filename_queue.get(timeout=self.TIMEOUT)
                self.run_local_bears_on_file(filename)
                if hasattr(self.filename_queue, "task_done"):
                    self.filename_queue.task_done()
        except queue.Empty:
            return

    def _get_global_dependency_results(self, bear_instance):
        """
        Retreives dependency results for a global bear.

        :return: None if bear has no dependencies, False if dependencies are
        not met, the dependency dict otherwise.
        """
        try:
            deps = bear_instance.get_dependencies()
            if deps == []:
                return None
        except AttributeError:
            # When this occurs we have an invalid bear and a warning will be
            # emitted later.
            return None

        dependency_results = {}
        for dep in deps:
            depname = dep.__name__
            if depname not in self.global_result_dict:
                return False

            dependency_results[depname] = self.global_result_dict[depname]

        return dependency_results

    def _get_next_global_bear(self):
        """
        Retrieves the next global bear.

        :return: (bear, bearname, dependency_results)
        """
        dependency_results = False

        while dependency_results is False:
            bear_id = self.global_bear_queue.get(timeout=self.TIMEOUT)
            bear = self.global_bear_list[bear_id]

            dependency_results = self._get_global_dependency_results(bear)
            if dependency_results is False:
                self.global_bear_queue.put(bear_id)

        return bear, bear.__class__.__name__, dependency_results

    def run_global_bears(self):
        try:
            while True:
                bear, bearname, dep_results = self._get_next_global_bear()
                result = run_global_bear(self.message_queue,
                                         self.TIMEOUT,
                                         bear,
                                         dep_results)
                if result:
                    self.global_result_dict[bearname] = result
                    self.control_queue.put((CONTROL_ELEMENT.GLOBAL,
                                            bearname))
                else:
                    self.global_result_dict[bearname] = None
                if hasattr(self.global_bear_queue, "task_done"):
                    self.global_bear_queue.task_done()
        except queue.Empty:
            return

    def run_local_bears_on_file(self, filename):
        if filename not in self.file_dict:
            send_msg(self.message_queue,
                     self.TIMEOUT,
                     LOG_LEVEL.ERROR,
                     _("An internal error occurred."),
                     StringConstants.THIS_IS_A_BUG)
            send_msg(self.message_queue,
                     self.TIMEOUT,
                     LOG_LEVEL.DEBUG,
                     _("The given file through the queue is not in the file "
                       "dictionary."))

            return

        self._local_result_list = []
        for bear_instance in self.local_bear_list:
            r = run_local_bear(self.message_queue,
                               self.TIMEOUT,
                               self._local_result_list,
                               self.file_dict,
                               bear_instance,
                               filename)
            if r is not None:
                self._local_result_list.extend(r)

        self.local_result_dict[filename] = self._local_result_list
        self.control_queue.put((CONTROL_ELEMENT.LOCAL, filename))
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import getopt
import math
import tempfile
import stat
import shlex
import subprocess
from subprocess import Popen
from optparse import OptionParser
from experiments.core.util import get_new_file, sweep_mult, fancify_cmd
from experiments.core.util import head_sentences
import platform
from glob import glob
from experiments.core.experiment_runner import ExpParamsRunner
from experiments.core import experiment_runner
import re

def get_root_dir():
    scripts_dir =  os.path.abspath(sys.path[0])
    root_dir =  os.path.dirname(os.path.dirname(scripts_dir))
    print "Using root_dir: " + root_dir
    return root_dir;

def get_dev_data(data_dir, file_prefix, name=None):
    return get_some_data(data_dir, file_prefix, name, "dev")

def get_test_data(data_dir, file_prefix, name=None):
    return get_some_data(data_dir, file_prefix, name, "test")

def get_some_data(data_dir, file_prefix, name, test_suffix):
    data = DPExpParams()
    if name == None:
        name = file_prefix.replace("/", "-")
    data.set("dataset", name,True,False)
    data.set("train","%s/%s" % (data_dir, file_prefix),False,True)
    #data.set("dev","%s/%s.%s" % (data_dir, file_prefix, test_suffix),False,True)
    return data

class DPExpParams(experiment_runner.JavaExpParams):

    def __init__(self, **keywords):
        experiment_runner.ExpParams.__init__(self,keywords)

    def get_initial_keys(self):
        return "dataSet model k s".split()

    def get_instance(self):
        return DPExpParams()

    def create_experiment_script(self, exp_dir, eprunner):
        script = ""
        #script += "export CLASSPATH=%s/classes:%s/lib/*\n" % (eprunner.root_dir, eprunner.root_dir)
        script += "echo 'CLASSPATH=$CLASSPATH'\n"
        cmd = "java -cp $CLASSPATH " + self.get_java_args(eprunner) + " edu.jhu.hltcoe.PipelineRunner  %s \n" % (self.get_args())
        script += fancify_cmd(cmd)
        return script

    def get_java_args(self, eprunner):
        # Allot the available memory to the JVM, ILP solver, and ZIMPL
        total_work_mem_megs = eprunner.work_mem_megs
        if (eprunner.queue != None):
            zimpl_mem = int(total_work_mem_megs * 0.5)
        else:
            zimpl_mem = 0
        java_mem = int((total_work_mem_megs - zimpl_mem) * 0.5)
        ilp_mem = total_work_mem_megs - java_mem - zimpl_mem
        # Subtract off some overhead for CPLEX
        ilp_mem -= 128
        assert (zimpl_mem + java_mem + ilp_mem <= total_work_mem_megs)

        self.update(ilpWorkMemMegs=ilp_mem)

        # Create the JVM args
        java_args = self._get_java_args(java_mem)
        if True:
            # HACK: revert back to this if-clause after adding real parser for eval: self.get("ilpSolver") == "cplex":
            mac_jlp = "/Users/mgormley/installed/ILOG/CPLEX_Studio_AcademicResearch122/cplex/bin/x86-64_darwin9_gcc4.0"
            coe_jlp = "/home/hltcoe/mgormley/installed/ILOG/CPLEX_Studio_AcademicResearch122/cplex/bin/x86-64_sles10_4.1"
            if os.path.exists(mac_jlp): jlp = mac_jlp
            elif os.path.exists(coe_jlp): jlp = coe_jlp
            else: raise Exception("Could not find java.library.path for CPLEX")
            java_args += " -Djava.library.path=%s " % (jlp)
        return java_args

class DepParseExpParamsRunner(ExpParamsRunner):

    def __init__(self, options):
        ExpParamsRunner.__init__(self, "dp", options.queue)
        self.root_dir = os.path.abspath(get_root_dir())
        self.fast = options.fast
        self.expname = options.expname
        self.data = options.data
        if options.test:
            self.get_data = get_test_data
        else:
            self.get_data = get_dev_data

        if self.queue == "mem" or self.queue == "himem":
            # Override qsub_args to exclude "-l h_vmem=%dM"
            # Old way: self.qsub_args = re.sub("-l h_vmem=", "-l virtual_free=", self.qsub_args)
            self.qsub_args = re.sub("-l h_vmem=\S+", "", self.qsub_args)

    def get_experiments(self):
        all = DPExpParams()
        all.set("expname", self.expname, False, False)
        all.update(threads=self.threads)
        all.update(formulation="deptree-flow-nonproj",
                   parser="ilp-corpus",
                   model="dmv",
                   algorithm="viterbi",
                   ilpSolver="cplex",
                   convergenceRatio=0.99999,
                   epsilon=0.1)
        all.set("lambda",0.1)
        all.update(printModel="./model.txt")
        # Only keeping sentences that contain a verb
        all.update(mustContainVerb=None)

        dgFixedInterval = DPExpParams(deltaGenerator="fixed-interval",interval=0.01,numPerSide=2)
        dgFactor = DPExpParams(deltaGenerator="factor",factor=1.1,numPerSide=2)

        if self.fast:       all.update(iterations=1,
                                       maxSentenceLength=7,
                                       maxNumSentences=2)
        else:               all.update(iterations=25)

        # Data sets
        data_dir = os.path.join(self.root_dir, "data")
        wsj_00 = self.get_data(data_dir, "treebank_3/wsj/00")
        wsj_full = self.get_data(data_dir, "treebank_3/wsj")
        brown_cf = self.get_data(data_dir, "treebank_3/brown/cf")
        brown_full = self.get_data(data_dir, "treebank_3/brown")
        synthetic = DPExpParams(synthetic="simplest")

        # Reducing tagset explicitly
        for ptbdata in [wsj_00, wsj_full, brown_cf, brown_full]:
            ptbdata.update(reduceTags="%s/data/universal_pos_tags.1.02/en-ptb.map" % (self.root_dir))

        if self.data == "synthetic":
            datasets = [synthetic]
            all.update(printSentences="./data.txt",
                       syntheticSeed=123454321)
        elif self.fast:       datasets = [brown_cf]
        else:               datasets = [brown_full]

        experiments = []
        if self.expname == "formulations":
            all.update(parser="ilp-corpus")
            for dataset in datasets:
                formulations = ["deptree-dp-proj", "deptree-explicit-proj", "deptree-flow-nonproj", "deptree-flow-proj", "deptree-multiflow-nonproj", "deptree-multiflow-proj" ]
                for formulation in formulations:
                    ilpform = DPExpParams(formulation=formulation)
                    experiments.append(all + dataset + ilpform)
        elif self.expname == "bnb":
            all.update(algorithm="bnb")
            for dataset in datasets:
                for maxSentenceLength in [3,5]:
                    msl = DPExpParams(maxSentenceLength=maxSentenceLength)
                    for maxNumSentences in [10,100]:
                        mns = DPExpParams(maxNumSentences=maxNumSentences)
                experiments.append(all + dataset + msl + mns)
        elif self.expname == "corpus-size":
            # For ilp-corpus testing:
            #  all.update(iterations=1)
            all.set("lambda",0.0)
            # For SIMPLEX testing
            #  all.update(formulation="deptree-flow-nonproj-lprelax")
            for parser in ["cky"]: #["ilp-corpus","ilp-sentence"]:
                par = DPExpParams(parser=parser)
                for dataset in datasets:
                    for maxSentenceLength in [7,10,20]:
                        msl = DPExpParams(maxSentenceLength=maxSentenceLength)
                        # Limiting to max  7 words/sent there are 2394 sentences
                        # Limiting to max 10 words/sent there are 5040 sentences
                        # Limiting to max 20 words/sent there are 20570 sentences
                        if (maxSentenceLength == 7):
                            mns_list = range(200,2200,200)
                        elif (maxSentenceLength == 10):
                            mns_list = range(100,5040,100)
                        else: # for 20
                            mns_list = range(1000,20570,1000)
                        for maxNumSentences in mns_list:
                            mns = DPExpParams(maxNumSentences=maxNumSentences)
                            experiments.append(all + dataset + msl + par + mns)
        elif self.expname == "deltas":
            for dataset in datasets:
                for maxSentenceLength in [5,7]:
                    msl = DPExpParams(maxSentenceLength=maxSentenceLength)
#                    if (maxSentenceLength == 5):
#                        mns_list = range(100,1500,100)
#                    elif (maxSentenceLength == 7):
#                        mns_list = range(100,2200,100)
                    mns_list = [2,4,8,16,32,64,128,256,512,1024]
                    for maxNumSentences in mns_list:
                        mns = DPExpParams(maxNumSentences=maxNumSentences)
                        # It doesn't make sense to do D-W for ilp-corpus, because there are no coupling constraints
                        experiments.append(all + dataset + msl + mns + DPExpParams(parser="ilp-corpus"))
                        for dataGen in [dgFixedInterval, dgFactor]:
                            for ilpSolver in ["cplex","dip-milpblock-cpm","dip-milpblock-pc"]:
                                experiments.append(all + dataset + msl + mns + DPExpParams(parser="ilp-deltas", ilpSolver=ilpSolver) + dataGen)
                                if ilpSolver == "cplex":
                                    experiments.append(all + dataset + msl + mns + DPExpParams(parser="ilp-deltas-init", ilpSolver=ilpSolver) + dataGen)

        else:
            raise Exception("Unknown expname: " + str(self.expname))

        return experiments

    def create_post_processing_script(self, top_dir, exp_tuples):
        #return "python %s/scripts/experiments/pp_scrape_expout.py %s" % (self.tagging_dir, top_dir)
        return None;

if __name__ == "__main__":
    usage = "%prog "

    parser = OptionParser(usage=usage)
    parser.add_option('-q', '--queue', help="Which SGE queue to use")
    parser.add_option('-f', '--fast', action="store_true", help="Run a fast version")
    parser.add_option('--test', action="store_true", help="Use test data")
    parser.add_option('--expname',  help="Experiment name")
    parser.add_option('--data',  help="Dataset to use")
    (options, args) = parser.parse_args(sys.argv)

    if len(args) != 1:
        parser.print_help()
        sys.exit(1)

    runner = DepParseExpParamsRunner(options)
    experiments = runner.get_experiments()
    runner.run_experiments(experiments)

<commit_msg>fixing indentation<commit_after>#!/usr/bin/python

import sys
import os
import getopt
import math
import tempfile
import stat
import shlex
import subprocess
from subprocess import Popen
from optparse import OptionParser
from experiments.core.util import get_new_file, sweep_mult, fancify_cmd
from experiments.core.util import head_sentences
import platform
from glob import glob
from experiments.core.experiment_runner import ExpParamsRunner
from experiments.core import experiment_runner
import re

def get_root_dir():
    scripts_dir =  os.path.abspath(sys.path[0])
    root_dir =  os.path.dirname(os.path.dirname(scripts_dir))
    print "Using root_dir: " + root_dir
    return root_dir;

def get_dev_data(data_dir, file_prefix, name=None):
    return get_some_data(data_dir, file_prefix, name, "dev")

def get_test_data(data_dir, file_prefix, name=None):
    return get_some_data(data_dir, file_prefix, name, "test")

def get_some_data(data_dir, file_prefix, name, test_suffix):
    data = DPExpParams()
    if name == None:
        name = file_prefix.replace("/", "-")
    data.set("dataset", name,True,False)
    data.set("train","%s/%s" % (data_dir, file_prefix),False,True)
    #data.set("dev","%s/%s.%s" % (data_dir, file_prefix, test_suffix),False,True)
    return data

class DPExpParams(experiment_runner.JavaExpParams):

    def __init__(self, **keywords):
        experiment_runner.ExpParams.__init__(self,keywords)

    def get_initial_keys(self):
        return "dataSet model k s".split()

    def get_instance(self):
        return DPExpParams()

    def create_experiment_script(self, exp_dir, eprunner):
        script = ""
        #script += "export CLASSPATH=%s/classes:%s/lib/*\n" % (eprunner.root_dir, eprunner.root_dir)
        script += "echo 'CLASSPATH=$CLASSPATH'\n"
        cmd = "java -cp $CLASSPATH " + self.get_java_args(eprunner) + " edu.jhu.hltcoe.PipelineRunner  %s \n" % (self.get_args())
        script += fancify_cmd(cmd)
        return script

    def get_java_args(self, eprunner):
        # Allot the available memory to the JVM, ILP solver, and ZIMPL
        total_work_mem_megs = eprunner.work_mem_megs
        if (eprunner.queue != None):
            zimpl_mem = int(total_work_mem_megs * 0.5)
        else:
            zimpl_mem = 0
        java_mem = int((total_work_mem_megs - zimpl_mem) * 0.5)
        ilp_mem = total_work_mem_megs - java_mem - zimpl_mem
        # Subtract off some overhead for CPLEX
        ilp_mem -= 128
        assert (zimpl_mem + java_mem + ilp_mem <= total_work_mem_megs)

        self.update(ilpWorkMemMegs=ilp_mem)

        # Create the JVM args
        java_args = self._get_java_args(java_mem)
        if True:
            # HACK: revert back to this if-clause after adding real parser for eval: self.get("ilpSolver") == "cplex":
            mac_jlp = "/Users/mgormley/installed/ILOG/CPLEX_Studio_AcademicResearch122/cplex/bin/x86-64_darwin9_gcc4.0"
            coe_jlp = "/home/hltcoe/mgormley/installed/ILOG/CPLEX_Studio_AcademicResearch122/cplex/bin/x86-64_sles10_4.1"
            if os.path.exists(mac_jlp): jlp = mac_jlp
            elif os.path.exists(coe_jlp): jlp = coe_jlp
            else: raise Exception("Could not find java.library.path for CPLEX")
            java_args += " -Djava.library.path=%s " % (jlp)
        return java_args

class DepParseExpParamsRunner(ExpParamsRunner):

    def __init__(self, options):
        ExpParamsRunner.__init__(self, "dp", options.queue)
        self.root_dir = os.path.abspath(get_root_dir())
        self.fast = options.fast
        self.expname = options.expname
        self.data = options.data
        if options.test:
            self.get_data = get_test_data
        else:
            self.get_data = get_dev_data

        if self.queue == "mem" or self.queue == "himem":
            # Override qsub_args to exclude "-l h_vmem=%dM"
            # Old way: self.qsub_args = re.sub("-l h_vmem=", "-l virtual_free=", self.qsub_args)
            self.qsub_args = re.sub("-l h_vmem=\S+", "", self.qsub_args)

    def get_experiments(self):
        all = DPExpParams()
        all.set("expname", self.expname, False, False)
        all.update(threads=self.threads)
        all.update(formulation="deptree-flow-nonproj",
                   parser="ilp-corpus",
                   model="dmv",
                   algorithm="viterbi",
                   ilpSolver="cplex",
                   convergenceRatio=0.99999,
                   epsilon=0.1)
        all.set("lambda",0.1)
        all.update(printModel="./model.txt")
        # Only keeping sentences that contain a verb
        all.update(mustContainVerb=None)

        dgFixedInterval = DPExpParams(deltaGenerator="fixed-interval",interval=0.01,numPerSide=2)
        dgFactor = DPExpParams(deltaGenerator="factor",factor=1.1,numPerSide=2)

        if self.fast:       all.update(iterations=1,
                                       maxSentenceLength=7,
                                       maxNumSentences=2)
        else:               all.update(iterations=25)

        # Data sets
        data_dir = os.path.join(self.root_dir, "data")
        wsj_00 = self.get_data(data_dir, "treebank_3/wsj/00")
        wsj_full = self.get_data(data_dir, "treebank_3/wsj")
        brown_cf = self.get_data(data_dir, "treebank_3/brown/cf")
        brown_full = self.get_data(data_dir, "treebank_3/brown")
        synthetic = DPExpParams(synthetic="simplest")

        # Reducing tagset explicitly
        for ptbdata in [wsj_00, wsj_full, brown_cf, brown_full]:
            ptbdata.update(reduceTags="%s/data/universal_pos_tags.1.02/en-ptb.map" % (self.root_dir))

        if self.data == "synthetic":
            datasets = [synthetic]
            all.update(printSentences="./data.txt",
                       syntheticSeed=123454321)
        elif self.fast:       datasets = [brown_cf]
        else:               datasets = [brown_full]

        experiments = []
        if self.expname == "formulations":
            all.update(parser="ilp-corpus")
            for dataset in datasets:
                formulations = ["deptree-dp-proj", "deptree-explicit-proj", "deptree-flow-nonproj", "deptree-flow-proj", "deptree-multiflow-nonproj", "deptree-multiflow-proj" ]
                for formulation in formulations:
                    ilpform = DPExpParams(formulation=formulation)
                    experiments.append(all + dataset + ilpform)
        elif self.expname == "bnb":
            all.update(algorithm="bnb")
            for dataset in datasets:
                for maxSentenceLength in [3,5]:
                    msl = DPExpParams(maxSentenceLength=maxSentenceLength)
                    for maxNumSentences in [10,100]:
                        mns = DPExpParams(maxNumSentences=maxNumSentences)
                        experiments.append(all + dataset + msl + mns)
        elif self.expname == "corpus-size":
            # For ilp-corpus testing:
            #  all.update(iterations=1)
            all.set("lambda",0.0)
            # For SIMPLEX testing
            #  all.update(formulation="deptree-flow-nonproj-lprelax")
            for parser in ["cky"]: #["ilp-corpus","ilp-sentence"]:
                par = DPExpParams(parser=parser)
                for dataset in datasets:
                    for maxSentenceLength in [7,10,20]:
                        msl = DPExpParams(maxSentenceLength=maxSentenceLength)
                        # Limiting to max  7 words/sent there are 2394 sentences
                        # Limiting to max 10 words/sent there are 5040 sentences
                        # Limiting to max 20 words/sent there are 20570 sentences
                        if (maxSentenceLength == 7):
                            mns_list = range(200,2200,200)
                        elif (maxSentenceLength == 10):
                            mns_list = range(100,5040,100)
                        else: # for 20
                            mns_list = range(1000,20570,1000)
                        for maxNumSentences in mns_list:
                            mns = DPExpParams(maxNumSentences=maxNumSentences)
                            experiments.append(all + dataset + msl + par + mns)
        elif self.expname == "deltas":
            for dataset in datasets:
                for maxSentenceLength in [5,7]:
                    msl = DPExpParams(maxSentenceLength=maxSentenceLength)
#                    if (maxSentenceLength == 5):
#                        mns_list = range(100,1500,100)
#                    elif (maxSentenceLength == 7):
#                        mns_list = range(100,2200,100)
                    mns_list = [2,4,8,16,32,64,128,256,512,1024]
                    for maxNumSentences in mns_list:
                        mns = DPExpParams(maxNumSentences=maxNumSentences)
                        # It doesn't make sense to do D-W for ilp-corpus, because there are no coupling constraints
                        experiments.append(all + dataset + msl + mns + DPExpParams(parser="ilp-corpus"))
                        for dataGen in [dgFixedInterval, dgFactor]:
                            for ilpSolver in ["cplex","dip-milpblock-cpm","dip-milpblock-pc"]:
                                experiments.append(all + dataset + msl + mns + DPExpParams(parser="ilp-deltas", ilpSolver=ilpSolver) + dataGen)
                                if ilpSolver == "cplex":
                                    experiments.append(all + dataset + msl + mns + DPExpParams(parser="ilp-deltas-init", ilpSolver=ilpSolver) + dataGen)

        else:
            raise Exception("Unknown expname: " + str(self.expname))

        return experiments

    def create_post_processing_script(self, top_dir, exp_tuples):
        #return "python %s/scripts/experiments/pp_scrape_expout.py %s" % (self.tagging_dir, top_dir)
        return None;

if __name__ == "__main__":
    usage = "%prog "

    parser = OptionParser(usage=usage)
    parser.add_option('-q', '--queue', help="Which SGE queue to use")
    parser.add_option('-f', '--fast', action="store_true", help="Run a fast version")
    parser.add_option('--test', action="store_true", help="Use test data")
    parser.add_option('--expname',  help="Experiment name")
    parser.add_option('--data',  help="Dataset to use")
    (options, args) = parser.parse_args(sys.argv)

    if len(args) != 1:
        parser.print_help()
        sys.exit(1)

    runner = DepParseExpParamsRunner(options)
    experiments = runner.get_experiments()
    runner.run_experiments(experiments)

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Author : Himanshu Shekhar < https://github.com/himanshub16 > (2017)
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Checks for broken links in file names specified as command line parameters.

There are a ton of a solutions available for validating URLs in string using
regex, but less for searching, of which very few are accurate.
This snippet is intended to just do the required work, and avoid complexities.
Django Validator has pretty good regex for validation, but we have to find
urls instead of validating them. (REFERENCES [7])
There's always room for improvement.

Method:
* Match URLs using regex (REFERENCES [1]])
* Some URLs need to be fixed, as they have < (or) > due to inefficient regex.
* Remove duplicates (because regex is not 100% efficient as of now).
* Check validity of URL, using HEAD request. (HEAD to save bandwidth)
  Uses requests module for others are painful to use. REFERENCES[9]
  Handles redirects, http, https, ftp as well.

REFERENCES:
Using [1] with some modificatons for including ftp
[1] http://stackoverflow.com/questions/6883049/regex-to-find-urls-in-string-in-python
[2] http://stackoverflow.com/a/31952097/5163807
[3] http://daringfireball.net/2010/07/improved_regex_for_matching_urls
[4] https://mathiasbynens.be/demo/url-regex
[5] https://github.com/django/django/blob/master/django/core/validators.py
[6] https://data.iana.org/TLD/tlds-alpha-by-domain.txt
[7] https://codereview.stackexchange.com/questions/19663/http-url-validating
[8] https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
[9] http://docs.python-requests.org/

"""

from __future__ import print_function

import os
import re
import sys
import requests

HERE = os.path.abspath(os.path.dirname(__file__))

URL_REGEX = '(?:http|ftp|https)?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'

def get_urls(filename):
    """Extracts all URLs available in specified filename
    """
    fname = os.path.abspath(os.path.join(HERE, filename))
    print (fname)
    text = ''
    with open(fname) as f:
        text = f.read()

    urls = re.findall(URL_REGEX, text)
    # remove duplicates, list for sets are not iterable
    urls = list(set(urls))
    # correct urls which are between < and/or >
    i = 0
    while i < len(urls):
        urls[i] = re.sub("[<>]", '', urls[i])
        i += 1

    return urls

def validate_url(url):
    """Validate the URL by attempting an HTTP connection.
    Makes an HTTP-HEAD request for each URL.
    Uses requests module.
    """
    try:
        res = requests.head(url)
        return res.ok
    except Exception as e:
        return False

def main():
    """Main function
    """
    files = sys.argv[1:]
    fails = []
    for fname in files:
        urls = get_urls(fname)
        i = 0
        last = len(urls)
        for url in urls:
            i += 1
            if not validate_url(url):
                fails.append((url, fname))
            sys.stdout.write("\r " + fname + " : " + str(i) + " / " + str(last))
            sys.stdout.flush()

    print()
    if len(fails) == 0:
        print("All URLs are valid. Cheers!")
    else:
        print ("Total :", len(fails), "fails!")
        print ("Writing failed urls to fails.txt")
        with open("../../fails.txt", 'w') as f:
            for fail in fails:
                f.write(fail[1] + ' : ' + fail[0] + os.linesep)
            f.write('-' * 20)
            f.write(os.linesep*2)

if __name__ == '__main__':
    main()
<commit_msg>expecting absolute path in script<commit_after>#!/usr/bin/env python

# Author : Himanshu Shekhar < https://github.com/himanshub16 > (2017)
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Checks for broken links in file names specified as command line parameters.

There are a ton of a solutions available for validating URLs in string using
regex, but less for searching, of which very few are accurate.
This snippet is intended to just do the required work, and avoid complexities.
Django Validator has pretty good regex for validation, but we have to find
urls instead of validating them. (REFERENCES [7])
There's always room for improvement.

Method:
* Match URLs using regex (REFERENCES [1]])
* Some URLs need to be fixed, as they have < (or) > due to inefficient regex.
* Remove duplicates (because regex is not 100% efficient as of now).
* Check validity of URL, using HEAD request. (HEAD to save bandwidth)
  Uses requests module for others are painful to use. REFERENCES[9]
  Handles redirects, http, https, ftp as well.

REFERENCES:
Using [1] with some modificatons for including ftp
[1] http://stackoverflow.com/questions/6883049/regex-to-find-urls-in-string-in-python
[2] http://stackoverflow.com/a/31952097/5163807
[3] http://daringfireball.net/2010/07/improved_regex_for_matching_urls
[4] https://mathiasbynens.be/demo/url-regex
[5] https://github.com/django/django/blob/master/django/core/validators.py
[6] https://data.iana.org/TLD/tlds-alpha-by-domain.txt
[7] https://codereview.stackexchange.com/questions/19663/http-url-validating
[8] https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
[9] http://docs.python-requests.org/

"""

from __future__ import print_function

import os
import re
import sys
import requests

HERE = os.path.abspath(os.path.dirname(__file__))

URL_REGEX = '(?:http|ftp|https)?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'

def get_urls(filename):
    """Extracts all URLs available in specified filename
    """
    # fname = os.path.abspath(os.path.join(HERE, filename))
    # expecting absolute path
    fname = os.path.abspath(filename)
    print (fname)
    text = ''
    with open(fname) as f:
        text = f.read()

    urls = re.findall(URL_REGEX, text)
    # remove duplicates, list for sets are not iterable
    urls = list(set(urls))
    # correct urls which are between < and/or >
    i = 0
    while i < len(urls):
        urls[i] = re.sub("[<>]", '', urls[i])
        i += 1

    return urls

def validate_url(url):
    """Validate the URL by attempting an HTTP connection.
    Makes an HTTP-HEAD request for each URL.
    Uses requests module.
    """
    try:
        res = requests.head(url)
        return res.ok
    except Exception as e:
        return False

def main():
    """Main function
    """
    files = sys.argv[1:]
    fails = []
    for fname in files:
        urls = get_urls(fname)
        i = 0
        last = len(urls)
        for url in urls:
            i += 1
            if not validate_url(url):
                fails.append((url, fname))
            sys.stdout.write("\r " + fname + " : " + str(i) + " / " + str(last))
            sys.stdout.flush()

    print()
    if len(fails) == 0:
        print("All URLs are valid. Cheers!")
    else:
        print ("Total :", len(fails), "fails!")
        print ("Writing failed urls to fails.txt")
        with open("../../fails.txt", 'w') as f:
            for fail in fails:
                f.write(fail[1] + ' : ' + fail[0] + os.linesep)
            f.write('-' * 20)
            f.write(os.linesep*2)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from collections import namedtuple

import pytz
from backdrop.core.timeseries import parse_period
from backdrop.core.timeutils import now, parse_time_as_utc
from backdrop.read.response import *

def if_present(func, value):
    """Apply the given function to the value and return if it exists"""
    if value is not None:
        return func(value)

def parse_request_args(request_args):
    args = dict()

    args['start_at'] = if_present(parse_time_as_utc,
                                  request_args.get('start_at'))

    args['end_at'] = if_present(parse_time_as_utc,
                                request_args.get('end_at'))

    args['duration'] = if_present(int, request_args.get('duration'))

    args['period'] = if_present(parse_period,
                                request_args.get('period'))

    def boolify(value):
        return {
            "true": True,
            "false": False,
        }.get(value, value)

    def parse_filter_by(filter_by):
        key, value = filter_by.split(':', 1)

        return [key, boolify(value)]

    args['filter_by'] = map(parse_filter_by, request_args.getlist('filter_by'))

    args['group_by'] = request_args.get('group_by')

    args['sort_by'] = if_present(lambda sort_by: sort_by.split(':', 1),
                                 request_args.get('sort_by'))

    args['limit'] = if_present(int, request_args.get('limit'))

    args['collect'] = []
    for collect_arg in request_args.getlist('collect'):
        if ':' in collect_arg:
            args['collect'].append(tuple(collect_arg.split(':')))
        else:
            args['collect'].append((collect_arg, 'default'))

    return args

_Query = namedtuple(
    '_Query',
    ['start_at', 'end_at', 'delta', 'period',
     'filter_by', 'group_by', 'sort_by', 'limit', 'collect'])

class Query(_Query):
    @classmethod
    def create(cls,
               start_at=None, end_at=None, duration=None, delta=None,
               period=None, filter_by=None, group_by=None,
               sort_by=None, limit=None, collect=None):
        delta = None
        if duration is not None:
            date = start_at or end_at or now()
            delta = duration if start_at else -duration
            start_at, end_at = cls.__calculate_start_and_end(period, date,
                                                             delta)
        return Query(start_at, end_at, delta, period,
                     filter_by or [], group_by, sort_by, limit, collect or [])

    @classmethod
    def parse(cls, request_args):
        args = parse_request_args(request_args)
        return Query.create(**args)

    @staticmethod
    def __calculate_start_and_end(period, date, delta):
        duration = period.delta * delta
        start_of_period = period.start(date)

        start_at, end_at = sorted(
            [start_of_period, start_of_period + duration])

        return start_at, end_at

    def __skip_blank_periods(self, results, repository):
        amount_to_shift = results.amount_to_shift(self.delta)
        if amount_to_shift != 0:
            query = self.get_shifted_query(shift=amount_to_shift)
            results = query.execute(repository)

        return results

    def get_shifted_query(self, shift):
        """Return a new Query where the date is shifted by n periods"""
        args = self._asdict()

        args['start_at'] = args['start_at'] + (self.period.delta * shift)
        args['end_at'] = args['end_at'] + (self.period.delta * shift)

        return Query.create(**args)

    def to_mongo_query(self):
        mongo_query = {}
        if self.start_at or self.end_at:
            mongo_query["_timestamp"] = {}
            if self.end_at:
                mongo_query["_timestamp"]["$lt"] = self.end_at
            if self.start_at:
                mongo_query["_timestamp"]["$gte"] = self.start_at
        if self.filter_by:
            mongo_query.update(self.filter_by)
        return mongo_query

    def execute(self, repository):
        if self.group_by and self.period:
            result = self.__execute_period_group_query(repository)
        elif self.group_by:
            result = self.__execute_grouped_query(repository)
        elif self.period:
            result = self.__execute_period_query(repository)
        else:
            result = self.__execute_query(repository)

        if self.delta:
            result = self.__skip_blank_periods(result, repository)

        return result

    def __get_period_key(self):
        return self.period.start_at_key

    def __execute_period_group_query(self, repository):
        period_key = self.__get_period_key()

        cursor = repository.multi_group(
            self.group_by, period_key, self,
            sort=self.sort_by, limit=self.limit,
            collect=self.collect
        )

        results = PeriodGroupedData(cursor, period=self.period)

        if self.start_at and self.end_at:
            results.fill_missing_periods(
                self.start_at, self.end_at, collect=self.collect)

        return results

    def __execute_grouped_query(self, repository):
        cursor = repository.group(self.group_by, self, self.sort_by,
                                  self.limit, self.collect)

        results = GroupedData(cursor)
        return results

    def __execute_period_query(self, repository):
        period_key = self.__get_period_key()
        sort = [period_key, "ascending"]
        cursor = repository.group(
            period_key, self,
            sort=sort, limit=self.limit, collect=self.collect
        )

        results = PeriodData(cursor, period=self.period)

        if self.start_at and self.end_at:
            results.fill_missing_periods(
                self.start_at, self.end_at, collect=self.collect)

        return results

    def __execute_query(self, repository):
        cursor = repository.find(
            self, sort=self.sort_by, limit=self.limit)

        results = SimpleData(cursor)
        return results
<commit_msg>Add in quick Query internals explanation<commit_after>from collections import namedtuple

import pytz
from backdrop.core.timeseries import parse_period
from backdrop.core.timeutils import now, parse_time_as_utc
from backdrop.read.response import *

def if_present(func, value):
    """Apply the given function to the value and return if it exists"""
    if value is not None:
        return func(value)

def parse_request_args(request_args):
    args = dict()

    args['start_at'] = if_present(parse_time_as_utc,
                                  request_args.get('start_at'))

    args['end_at'] = if_present(parse_time_as_utc,
                                request_args.get('end_at'))

    args['duration'] = if_present(int, request_args.get('duration'))

    args['period'] = if_present(parse_period,
                                request_args.get('period'))

    def boolify(value):
        return {
            "true": True,
            "false": False,
        }.get(value, value)

    def parse_filter_by(filter_by):
        key, value = filter_by.split(':', 1)

        return [key, boolify(value)]

    args['filter_by'] = map(parse_filter_by, request_args.getlist('filter_by'))

    args['group_by'] = request_args.get('group_by')

    args['sort_by'] = if_present(lambda sort_by: sort_by.split(':', 1),
                                 request_args.get('sort_by'))

    args['limit'] = if_present(int, request_args.get('limit'))

    args['collect'] = []
    for collect_arg in request_args.getlist('collect'):
        if ':' in collect_arg:
            args['collect'].append(tuple(collect_arg.split(':')))
        else:
            args['collect'].append((collect_arg, 'default'))

    return args

"""
This is the internal Query object
 - Create list of attributes to build the query from
 - We use delta interally, but the end user will use 'duration'
"""
_Query = namedtuple(
    '_Query',
    ['start_at', 'end_at', 'delta', 'period',
     'filter_by', 'group_by', 'sort_by', 'limit', 'collect'])

class Query(_Query):
    @classmethod
    def create(cls,
               start_at=None, end_at=None, duration=None, delta=None,
               period=None, filter_by=None, group_by=None,
               sort_by=None, limit=None, collect=None):
        delta = None
        if duration is not None:
            date = start_at or end_at or now()
            delta = duration if start_at else -duration
            start_at, end_at = cls.__calculate_start_and_end(period, date,
                                                             delta)
        return Query(start_at, end_at, delta, period,
                     filter_by or [], group_by, sort_by, limit, collect or [])

    @classmethod
    def parse(cls, request_args):
        args = parse_request_args(request_args)
        return Query.create(**args)

    @staticmethod
    def __calculate_start_and_end(period, date, delta):
        duration = period.delta * delta
        start_of_period = period.start(date)

        start_at, end_at = sorted(
            [start_of_period, start_of_period + duration])

        return start_at, end_at

    def __skip_blank_periods(self, results, repository):
        amount_to_shift = results.amount_to_shift(self.delta)
        if amount_to_shift != 0:
            query = self.get_shifted_query(shift=amount_to_shift)
            results = query.execute(repository)

        return results

    def get_shifted_query(self, shift):
        """Return a new Query where the date is shifted by n periods"""
        args = self._asdict()

        args['start_at'] = args['start_at'] + (self.period.delta * shift)
        args['end_at'] = args['end_at'] + (self.period.delta * shift)

        return Query.create(**args)

    def to_mongo_query(self):
        mongo_query = {}
        if self.start_at or self.end_at:
            mongo_query["_timestamp"] = {}
            if self.end_at:
                mongo_query["_timestamp"]["$lt"] = self.end_at
            if self.start_at:
                mongo_query["_timestamp"]["$gte"] = self.start_at
        if self.filter_by:
            mongo_query.update(self.filter_by)
        return mongo_query

    def execute(self, repository):
        if self.group_by and self.period:
            result = self.__execute_period_group_query(repository)
        elif self.group_by:
            result = self.__execute_grouped_query(repository)
        elif self.period:
            result = self.__execute_period_query(repository)
        else:
            result = self.__execute_query(repository)

        if self.delta:
            result = self.__skip_blank_periods(result, repository)

        return result

    def __get_period_key(self):
        return self.period.start_at_key

    def __execute_period_group_query(self, repository):
        period_key = self.__get_period_key()

        cursor = repository.multi_group(
            self.group_by, period_key, self,
            sort=self.sort_by, limit=self.limit,
            collect=self.collect
        )

        results = PeriodGroupedData(cursor, period=self.period)

        if self.start_at and self.end_at:
            results.fill_missing_periods(
                self.start_at, self.end_at, collect=self.collect)

        return results

    def __execute_grouped_query(self, repository):
        cursor = repository.group(self.group_by, self, self.sort_by,
                                  self.limit, self.collect)

        results = GroupedData(cursor)
        return results

    def __execute_period_query(self, repository):
        period_key = self.__get_period_key()
        sort = [period_key, "ascending"]
        cursor = repository.group(
            period_key, self,
            sort=sort, limit=self.limit, collect=self.collect
        )

        results = PeriodData(cursor, period=self.period)

        if self.start_at and self.end_at:
            results.fill_missing_periods(
                self.start_at, self.end_at, collect=self.collect)

        return results

    def __execute_query(self, repository):
        cursor = repository.find(
            self, sort=self.sort_by, limit=self.limit)

        results = SimpleData(cursor)
        return results
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse, os, collections

parser = argparse.ArgumentParser(description='''
This script generates a country specific databases for use with
libpostal routines. Its a driver that runs build_country_db.sh
for every country found in the countries_languages.txt .

The driver will generate databases only for the countries that are
missing in the output directory''')

parser.add_argument('geodata_dir', type=str,
                    help='path to the directory containing geonames.tsv and postal_codes.tsv files')

parser.add_argument('addrdata_dir', type=str,
                    help='path to the directory containing files required for address parser training (formatted_addresses_tagged.random.tsv formatted_places_tagged.random.tsv)')

parser.add_argument('output_root_dir', type=str,
                    help='path to the directory where country specific subdirectory will be created')

args = parser.parse_args()

# load list of countries
countries = []
for l in open("countries_languages.txt", "r"):
    countries.append( l.split(':')[0] )

print "Loaded list of countries [%d]: " % len(countries),
for c in countries:
    print c,
print

countries_done = []
for l in os.listdir(args.output_root_dir):
    if l != "compressed" and os.path.exists(os.path.join(args.output_root_dir, l, "address_parser", "address_parser.dat")):
        countries_done.append( l.lower() )
countries_done.sort()

print "\nList of countries that are ready [%d]: " % len(countries_done),
for c in countries_done:
    print c,
print

countries_todo = []
for l in countries:
    if l not in countries_done:
        countries_todo.append(l)

print "\nCountries not covered yet [%d]: " % len(countries_todo),
for c in countries_todo:
    print c,
print

##########################################

script = "generator_script.sh"
f = open(script, "w")
f.write("#!/bin/bash\n\nset -e\n")
for C in countries_todo:
    f.write('./build_country_db.sh "%s" "%s" "%s" %s\n' % (args.geodata_dir,
                                                         args.addrdata_dir,
                                                         args.output_root_dir,
                                                         C) )

print "\nExamine and run " + script + "\n"
<commit_msg>switching to Make for building databases<commit_after>#!/usr/bin/env python

import argparse, os, collections

parser = argparse.ArgumentParser(description='''
This script generates a country specific databases for use with
libpostal routines. Its a driver that runs build_country_db.sh
for every country found in the countries_languages.txt .

The driver will generate databases only for the countries that are
missing in the output directory''')

parser.add_argument('geodata_dir', type=str,
                    help='path to the directory containing geonames.tsv and postal_codes.tsv files')

parser.add_argument('addrdata_dir', type=str,
                    help='path to the directory containing files required for address parser training (formatted_addresses_tagged.random.tsv formatted_places_tagged.random.tsv)')

parser.add_argument('output_root_dir', type=str,
                    help='path to the directory where country specific subdirectory will be created')

args = parser.parse_args()

# load list of countries
countries = []
for l in open("countries_languages.txt", "r"):
    countries.append( l.split(':')[0] )

print "Loaded list of countries [%d]: " % len(countries),
for c in countries:
    print c,
print

countries_done = []
for l in os.listdir(args.output_root_dir):
    if l != "compressed" and os.path.exists(os.path.join(args.output_root_dir, l, "address_parser", "address_parser.dat")):
        countries_done.append( l.lower() )
countries_done.sort()

print "\nList of countries that are ready [%d]: " % len(countries_done),
for c in countries_done:
    print c,
print

countries_todo = []
for l in countries:
    if l not in countries_done:
        countries_todo.append(l)

print "\nCountries not covered yet [%d]: " % len(countries_todo),
for c in countries_todo:
    print c,
print

##########################################

script = "Makefile"
f = open(script, "w")
f.write("# Generated by build_all_country_db\n\nall: ")
for C in countries_todo:
    f.write(C + " ")

f.write("\n\techo All done\n\n")
for C in countries_todo:
    f.write(C + ":\n" +
            '\t./build_country_db.sh "%s" "%s" "%s" %s\n\n' % (args.geodata_dir,
                                                               args.addrdata_dir,
                                                               args.output_root_dir,
                                                               C) )

print "\nExamine and run " + script + "\n"
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import sys

from rosdistro import get_distribution_file
from rosdistro import get_index

from ros_buildfarm.argument import add_argument_arch
from ros_buildfarm.argument import add_argument_build_name
from ros_buildfarm.argument import add_argument_cache_dir
from ros_buildfarm.argument import add_argument_config_url
from ros_buildfarm.argument import add_argument_os_code_name
from ros_buildfarm.argument import add_argument_rosdistro_name
from ros_buildfarm.common import get_debian_package_name
from ros_buildfarm.common import Target
from ros_buildfarm.config import get_index as get_config_index
from ros_buildfarm.config import get_release_build_files
from ros_buildfarm.debian_repo import get_debian_repo_index

def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description='Check if the sync criteria are matched to sync ' +
                    'packages from the building to the testing repo')
    add_argument_config_url(parser)
    add_argument_rosdistro_name(parser)
    add_argument_build_name(parser, 'release')
    add_argument_os_code_name(parser)
    add_argument_arch(parser)
    add_argument_cache_dir(parser, '/tmp/debian_repo_cache')
    args = parser.parse_args(argv)

    success = check_sync_criteria(
        args.config_url, args.rosdistro_name, args.release_build_name,
        args.os_code_name, args.arch, args.cache_dir)
    return 0 if success else 1

def check_sync_criteria(
        config_url, rosdistro_name, release_build_name, os_code_name, arch,
        cache_dir):
    # fetch debian package list
    config = get_config_index(config_url)
    index = get_index(config.rosdistro_index_url)
    dist_file = get_distribution_file(index, rosdistro_name)
    build_files = get_release_build_files(config, rosdistro_name)
    build_file = build_files[release_build_name]

    target = Target('ubuntu', os_code_name, arch)

    repo_index = get_debian_repo_index(
        build_file.target_repository, target, cache_dir)

    # for each release package which matches the release build file
    # check if a binary package exists
    binary_packages = {}
    all_pkg_names = dist_file.release_packages.keys()
    pkg_names = build_file.filter_packages(all_pkg_names)
    for pkg_name in sorted(pkg_names):
        debian_pkg_name = get_debian_package_name(rosdistro_name, pkg_name)
        binary_packages[pkg_name] = debian_pkg_name in repo_index

    # check that all elements from whitelist are present
    if build_file.sync_packages:
        missing_binary_packages = len([
            pkg_name
            for pkg_name, has_binary_package in binary_packages.items()
            if has_binary_package])
        if missing_binary_packages:
            print('The following binary packages are missing to sync:',
                  file=sys.stderr)
            for pkg_name in sorted(missing_binary_packages):
                print('-', pkg_name, file=sys.stderr)
            return False
        print('All required binary packages are available:')
        for pkg_name in sorted(build_file.sync_packages):
            print('-', pkg_name)

    # check that count is satisfied
    if build_file.sync_package_count is not None:
        binary_package_count = len([
            pkg_name
            for pkg_name, has_binary_package in binary_packages.items()
            if has_binary_package])
        if binary_package_count < build_file.sync_package_count:
            print('Only %d binary packages available ' % binary_package_count +
                  '(at least %d are required to sync)' %
                  build_file.sync_package_count, file=sys.stderr)
            return False
        print('%d binary packages available ' % binary_package_count +
              '(more or equal then the configured sync limit of %d)' %
              build_file.sync_package_count)

    return True

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Fix logic for ensuring all packages in sync: packages are present<commit_after>#!/usr/bin/env python3

import argparse
import sys

from rosdistro import get_distribution_file
from rosdistro import get_index

from ros_buildfarm.argument import add_argument_arch
from ros_buildfarm.argument import add_argument_build_name
from ros_buildfarm.argument import add_argument_cache_dir
from ros_buildfarm.argument import add_argument_config_url
from ros_buildfarm.argument import add_argument_os_code_name
from ros_buildfarm.argument import add_argument_rosdistro_name
from ros_buildfarm.common import get_debian_package_name
from ros_buildfarm.common import Target
from ros_buildfarm.config import get_index as get_config_index
from ros_buildfarm.config import get_release_build_files
from ros_buildfarm.debian_repo import get_debian_repo_index

def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description='Check if the sync criteria are matched to sync ' +
                    'packages from the building to the testing repo')
    add_argument_config_url(parser)
    add_argument_rosdistro_name(parser)
    add_argument_build_name(parser, 'release')
    add_argument_os_code_name(parser)
    add_argument_arch(parser)
    add_argument_cache_dir(parser, '/tmp/debian_repo_cache')
    args = parser.parse_args(argv)

    success = check_sync_criteria(
        args.config_url, args.rosdistro_name, args.release_build_name,
        args.os_code_name, args.arch, args.cache_dir)
    return 0 if success else 1

def check_sync_criteria(
        config_url, rosdistro_name, release_build_name, os_code_name, arch,
        cache_dir):
    # fetch debian package list
    config = get_config_index(config_url)
    index = get_index(config.rosdistro_index_url)
    dist_file = get_distribution_file(index, rosdistro_name)
    build_files = get_release_build_files(config, rosdistro_name)
    build_file = build_files[release_build_name]

    target = Target('ubuntu', os_code_name, arch)

    repo_index = get_debian_repo_index(
        build_file.target_repository, target, cache_dir)

    # for each release package which matches the release build file
    # check if a binary package exists
    binary_packages = {}
    all_pkg_names = dist_file.release_packages.keys()
    pkg_names = build_file.filter_packages(all_pkg_names)
    for pkg_name in sorted(pkg_names):
        debian_pkg_name = get_debian_package_name(rosdistro_name, pkg_name)
        binary_packages[pkg_name] = debian_pkg_name in repo_index

    # check that all elements from whitelist are present
    if build_file.sync_packages:
        missing_binary_packages = [
            pkg_name
            for pkg_name in build_file.sync_packages
            if pkg_name not in binary_packages or not binary_packages[pkg_name]]
        if missing_binary_packages:
            print('The following binary packages are missing to sync:',
                  file=sys.stderr)
            for pkg_name in sorted(missing_binary_packages):
                print('-', pkg_name, file=sys.stderr)
            return False
        print('All required binary packages are available:')
        for pkg_name in sorted(build_file.sync_packages):
            print('-', pkg_name)

    # check that count is satisfied
    if build_file.sync_package_count is not None:
        binary_package_count = len([
            pkg_name
            for pkg_name, has_binary_package in binary_packages.items()
            if has_binary_package])
        if binary_package_count < build_file.sync_package_count:
            print('Only %d binary packages available ' % binary_package_count +
                  '(at least %d are required to sync)' %
                  build_file.sync_package_count, file=sys.stderr)
            return False
        print('%d binary packages available ' % binary_package_count +
              '(more or equal then the configured sync limit of %d)' %
              build_file.sync_package_count)

    return True

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import RPi.GPIO as GPIO
import time
import os
import sys
from hx711 import HX711
from Adafruit_IO import Client

def cleanAndExit():
    print("Cleaning...")
    GPIO.cleanup()
    print("Bye!")
    sys.exit()

hx1 = HX711(5, 6)
hx2 = HX711(8,7)

aio_key = os.environ['AIOKEY']
aio = Client(aio_key)

feed_name1 = 'scalepos2'
feed_name2 = 'scalepos3'

# I've found out that, for some reason, the order of the bytes is not always the same between versions of python, nump$
# Still need to figure out why does it change.
# If you're experiencing super random values, change these values to MSB or LSB until to get more stable values.
# There is some code below to debug and log the order of the bits and the bytes.
# The first parameter is the order in which the bytes are used to build the "long" value.
# The second paramter is the order of the bits inside each byte.
# According to the HX711 Datasheet, the second parameter is MSB so you shouldn't need to modify it.
hx1.set_reading_format("LSB","MSB")
hx2.set_reading_format("LSB","MSB")

# HOW TO CALCULATE THE REFFERENCE UNIT
# To set the reference unit to 1. Put 1kg on your sensor or anything you have and know exactly how much it weights.
# In this case, 92 is 1 gram because, with 1 as a reference unit I got numbers near 0 without any weight
# and I got numbers around 184000 when I added 2kg. So, according to the rule of thirds:
# If 2000 grams is 184000 then 1000 grams is 184000 / 2000 = 92.
#hx.set_reference_unit(113)
hx1.set_reference_unit(-2037)
hx2.set_reference_unit(-2037)

hx1.reset()
hx1.tare()
hx2.reset()
hx2.tare()

while True:
    try:
        # These three lines are usefull to debug wether to use MSB or LSB in the reading formats
        # for the first parameter of "hx.set_reading_format("LSB", "MSB")".
        # Comment the two lines "val = hx.get_weight(5)" and "print val" and uncomment the three lines to see what it $
        #np_arr8_string = hx.get_np_arr8_string()
        #binary_string = hx.get_binary_string()
        #print binary_string + " " + np_arr8_string

        # Prints the weight. Comment if you're debbuging the MSB and LSB issue.
        val1 = hx1.get_weight(5)
        print("val1:" + str(val1))
        #print(int(val1))
        aio.send(feed_name1, int(val1))

        hx1.power_down()
        hx1.power_up()

        val2 = hx2.get_weight(5)
        print("val2:" + str(val2))
        aio.send(feed_name2, int(val2))

        hx2.power_down()
        hx2.power_up()
        #time.sleep(0.5)

        time.sleep(60)
    except (KeyboardInterrupt, SystemExit):
        cleanAndExit()

<commit_msg>Update fridgescale.py<commit_after># Retrieve weight from weight scales using HX711.
#
# Code is based on the HX711 Python Library for Raspberry Pi
# written by Philip Whitfield "https://github.com/underdoeg/"
# modified by tatobari "https://github.com/tatobari/hx711py"
# and further adapted to Python 3 in the MonoHelix Labs fork
# "https://github.com/MonoHelixLabs/hx711py"
#
# Note that you need the HX711.py from repo mentione above
# to be in the same folder as this file.

import RPi.GPIO as GPIO
import time
import os
import sys
from hx711 import HX711
from Adafruit_IO import Client

def cleanAndExit():
    print("Cleaning...")
    GPIO.cleanup()
    print("Bye!")
    sys.exit()

hx1 = HX711(5, 6)
hx2 = HX711(8, 7)

aio_key = os.environ['AIOKEY']
aio = Client(aio_key)

feed_name1 = 'scalepos2'
feed_name2 = 'scalepos3'

hx1.set_reading_format("LSB","MSB")
hx2.set_reading_format("LSB","MSB")

hx1.set_reference_unit(-2037)
hx2.set_reference_unit(-2037)

hx1.reset()
hx1.tare()
hx2.reset()
hx2.tare()

while True:
    try:
        val1 = hx1.get_weight(5)
        print("val1:" + str(val1))
        aio.send(feed_name1, int(val1))

        hx1.power_down()
        hx1.power_up()

        val2 = hx2.get_weight(5)
        print("val2:" + str(val2))
        aio.send(feed_name2, int(val2))

        hx2.power_down()
        hx2.power_up()

        time.sleep(60)
    except (KeyboardInterrupt, SystemExit):
        cleanAndExit()

<|endoftext|>


<commit_before>""" CLI for the analysis subcommand
"""
import click
from taca.analysis import analysis as an

@click.group()
def analysis():
	""" Analysis methods entry point """
	pass

# analysis subcommands
@analysis.command()
@click.option('-r', '--run', type=click.Path(exists=True), default=None,
				 help='Demultiplex only a particular run')
def demultiplex(run):
	""" Demultiplex all runs present in the data directories
	"""
	an.run_preprocessing(run)

@analysis.command()
@click.argument('rundir')
def transfer(rundir):
    """Transfers the run without qc"""
    an.transfer_run(rundir)
<commit_msg>Add option for triggering or not the analysis<commit_after>""" CLI for the analysis subcommand
"""
import click
from taca.analysis import analysis as an

@click.group()
def analysis():
	""" Analysis methods entry point """
	pass

# analysis subcommands
@analysis.command()
@click.option('-r', '--run', type=click.Path(exists=True), default=None,
				 help='Demultiplex only a particular run')
def demultiplex(run):
	""" Demultiplex all runs present in the data directories
	"""
	an.run_preprocessing(run)

@analysis.command()
@click.option('-a','--analysis', is_flag=True, help='Trigger the analysis for the transferred flowcell')
@click.argument('rundir')
def transfer(rundir, analysis):
    """Transfers the run without qc"""
    an.transfer_run(rundir, analysis=analysis)
<|endoftext|>


<commit_before>"""
stringjumble.py
Author: Avery Wallis
Credit: None so far

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
string= str(input("Please enter a string of text (the bigger the better): "))

print(string[-1] string[-2])<commit_msg>Updated from Brython Server: 2/6/2016 4:19:34 PM<commit_after>"""
stringjumble.py
Author: Avery Wallis
Credit: None so far

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
string= str(input("Please enter a string of text (the bigger the better): "))

print(string[(-1,-2)])<|endoftext|>