<commit_before>#!/usr/bin/env python
# encoding=utf8
from __future__ import print_function
from flexbe_core import EventState, Logger
from geometry_msgs.msg import Pose
from sara_moveit.srv import *
import rospy
import roslib; roslib.load_manifest('lu4r_ros')
import rospy
from lu4r_ros.srv import *
from lu4r_ros.msg import *
import socket
import sys
import requests
import json
import re
from subprocess import call

reload(sys)
sys.setdefaultencoding('utf8')
semantic_map = {}
HEADERS = {'content-type': 'application/json'}

class LU4R_Parser(EventState):
    '''
    MoveArm receive a ROS pose as input and launch a ROS service with the same pose

    ># sentence     string      sentence to parse

    <# actions     lu4r      list of actions

    <= done     Finish job.
    '''

    def __init__(self):
        # See example_state.py for basic explanations.
        super(LU4R_Parser, self).__init__(outcomes=['done', 'fail'], input_keys=['sentence', 'HighFIFO','MedFIFO','LowFIFO','DoNow'])

    def execute(self, userdata):
        # This method is called periodically while the state is active.
        # Main purpose is to check state conditions and trigger a corresponding outcome.
        # If no outcome is returned, the state will stay active.

        Logger.loginfo("sentence: "+userdata.sentence)

        lu4r_ip = '127.0.0.1'
        lu4r_port = '9001'
        lu4r_url = 'http://' + str(lu4r_ip) + ':' + str(lu4r_port) + '/service/nlu'

        HYPO = {'hypotheses':[{"transcription":userdata.sentence,"confidence":"0.9","rank":"1"}]}
        ENT = {'entities': [{"atom":"book1","type":"book","preferredLexicalReference":"book","alternativeLexicalReferences":["volume","manual","dictionary","text"],"coordinate":{"x":"13.0","y":"0.0","z":"0.0","angle":"0"}},{"atom":"table1","type":"table","preferredLexicalReference":"table","alternativeLexicalReferences":["counter","desk"],"coordinate":{"x":"16.0","y":"0.0","z":"0.0","angle":"0.671"}},{"atom":"glass1","type":"glass","preferredLexicalReference":"glass","alternativeLexicalReferences":["cup","decanter","chalice"],"coordinate":{"x":"5.0","y":"8.0","z":"0.0","angle":"0"}},{"atom":"bedroom1","type":"bedroom","preferredLexicalReference":"bedroom","alternativeLexicalReferences":["chamber","cubicle","bedchamber"],"coordinate":{"x":"11.0","y":"6.0","z":"0.0","angle":"0"}},{"atom":"studio1","type":"studio","preferredLexicalReference":"studio","alternativeLexicalReferences":["library","office"],"coordinate":{"x":"9.0","y":"14.0","z":"0.0","angle":"0"}},{"atom":"person1","type":"person","preferredLexicalReference":"person","alternativeLexicalReferences":["body","character","guy","man","woman"],"coordinate":{"x":"2.0","y":"2.0","z":"0.0","angle":"0"}}]}
        r = requests.post(lu4r_url, data={'hypo':str(HYPO) , 'entities':str(ENT)}, headers=HEADERS)

        lu4r = Lu4r()
        opItem = Operation()

        for line in r.iter_lines():
            if line:
                operation = re.compile('(\s*:op\d\s)?\([a-zA-Z0-9\-]+ / ([a-zA-Z\-]+)')
                m = operation.match(line)
                if m:
                    if opItem.action != "" and opItem.action != 'and':
                        lu4r.opList.append(opItem)
                    opItem = Operation()
                    opItem.action = m.group(2)
                else:
                    action = re.compile('\s*:([a-zA-Z0-9]*) \([a-zA-Z0-9]* / ([a-zA-Z0-9]*)\)')
                    n = action.match(line)
                    if n:
                        arg = args()
                        arg.type = n.group(1)
                        arg.content = n.group(2)
                        opItem.args.append(arg)

        if opItem.action != "":
            lu4r.opList.append(opItem)

        Logger.loginfo("Returning operation list - count : " + str(len(lu4r.opList)))
        if len(lu4r.opList) == 0:
            Logger.loginfo("fail")
            return 'fail'

        test = re.compile('.*[Ss]top.*')
        n = test.match(lu4r.opList[0].action)
        if n:
            Logger.loginfo("Action: Stop now!")
            userdata.DoNow = []
            userdata.DoNow.append('Stop')
            userdata.DoNow.append(0)
            userdata.HighFIFO = []
            userdata.MedFIFO = []
            userdata.LowFIFO = []

        for opitem in lu4r.opList:
            priority = 2
            Logger.loginfo("Action: "+opitem.action)
            ActionForm = self.ActionFormBuilder( opitem, [
                'MoveBase',
                '.*[Mm]otion.*',
                '.*([Tt]heme)|([Aa]rea)|([Gg]oal).*',
                '.*([Dd]irection).*',
                '.*([Dd]istance).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Pick',
                    '.*[Tt]aking.*',
                    '.*([Tt]heme).*',
                    '.*([Aa]rea).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Bring',
                    '.*[Bb]ring.*',
                    '.*([Tt]heme).*',
                    '.*([Aa]rea)|([Ss]ource).*',
                    '.*([Bb]eneficiary).*'])

            if ActionForm != None:
                if priority == 1:
                    userdata.LowFIFO.append(ActionForm)
                if priority == 2:
                    userdata.MedFIFO.append(ActionForm)
                if priority == 3:
                    userdata.HighFIFO.append(ActionForm)

        return 'done'  # One of the outcomes declared above.

    def on_enter(self, userdata):
        # This method is called when the state becomes active, a transition from another state to this one is taken.
        # It is primarily used to start actions which are associated with this state.

        Logger.loginfo('Enter lu4r_parser')

        global semantic_map

    def ActionFormBuilder(self, opitem, positions):

        PreActionForm = None
        # - Generic ActionForm builder
        test = re.compile(positions[1])
        n = test.match(opitem.action)
        if n:
            PreActionForm = []
            PreActionForm.append(positions[0])
            i=2
            while i<len(positions):
                PreActionForm.append('')
                i = i + 1
            for arg in opitem.args:
                Logger.loginfo("arg-type: " + arg.type)
                Logger.loginfo("arg-content: " + arg.content)
                i = 2
                while i < len(positions):
                    test = re.compile(positions[i])
                    n = test.match(arg.type)
                    if n:
                        PreActionForm[i-1] = arg.content
                    i = i + 1
        return PreActionForm<commit_msg>ajout de toute les actions possibles<commit_after>#!/usr/bin/env python
# encoding=utf8
from __future__ import print_function
from flexbe_core import EventState, Logger
from geometry_msgs.msg import Pose
from sara_moveit.srv import *
import rospy
import roslib; roslib.load_manifest('lu4r_ros')
import rospy
from lu4r_ros.srv import *
from lu4r_ros.msg import *
import socket
import sys
import requests
import json
import re
from subprocess import call

reload(sys)
sys.setdefaultencoding('utf8')
semantic_map = {}
HEADERS = {'content-type': 'application/json'}

class LU4R_Parser(EventState):
    '''
    MoveArm receive a ROS pose as input and launch a ROS service with the same pose

    ># sentence     string      sentence to parse

    <# actions     lu4r      list of actions

    <= done     Finish job.
    '''

    def __init__(self):
        # See example_state.py for basic explanations.
        super(LU4R_Parser, self).__init__(outcomes=['done', 'fail'], input_keys=['sentence', 'HighFIFO','MedFIFO','LowFIFO','DoNow'])

    def execute(self, userdata):
        # This method is called periodically while the state is active.
        # Main purpose is to check state conditions and trigger a corresponding outcome.
        # If no outcome is returned, the state will stay active.

        Logger.loginfo("sentence: "+userdata.sentence)

        lu4r_ip = '127.0.0.1'
        lu4r_port = '9001'
        lu4r_url = 'http://' + str(lu4r_ip) + ':' + str(lu4r_port) + '/service/nlu'

        HYPO = {'hypotheses':[{"transcription":userdata.sentence,"confidence":"0.9","rank":"1"}]}
        ENT = {'entities': [{"atom":"book1","type":"book","preferredLexicalReference":"book","alternativeLexicalReferences":["volume","manual","dictionary","text"],"coordinate":{"x":"13.0","y":"0.0","z":"0.0","angle":"0"}},{"atom":"table1","type":"table","preferredLexicalReference":"table","alternativeLexicalReferences":["counter","desk"],"coordinate":{"x":"16.0","y":"0.0","z":"0.0","angle":"0.671"}},{"atom":"glass1","type":"glass","preferredLexicalReference":"glass","alternativeLexicalReferences":["cup","decanter","chalice"],"coordinate":{"x":"5.0","y":"8.0","z":"0.0","angle":"0"}},{"atom":"bedroom1","type":"bedroom","preferredLexicalReference":"bedroom","alternativeLexicalReferences":["chamber","cubicle","bedchamber"],"coordinate":{"x":"11.0","y":"6.0","z":"0.0","angle":"0"}},{"atom":"studio1","type":"studio","preferredLexicalReference":"studio","alternativeLexicalReferences":["library","office"],"coordinate":{"x":"9.0","y":"14.0","z":"0.0","angle":"0"}},{"atom":"person1","type":"person","preferredLexicalReference":"person","alternativeLexicalReferences":["body","character","guy","man","woman"],"coordinate":{"x":"2.0","y":"2.0","z":"0.0","angle":"0"}}]}
        r = requests.post(lu4r_url, data={'hypo':str(HYPO) , 'entities':str(ENT)}, headers=HEADERS)

        lu4r = Lu4r()
        opItem = Operation()

        for line in r.iter_lines():
            if line:
                operation = re.compile('(\s*:op\d\s)?\([a-zA-Z0-9\-]+ / ([a-zA-Z\-]+)')
                m = operation.match(line)
                if m:
                    if opItem.action != "" and opItem.action != 'and':
                        lu4r.opList.append(opItem)
                    opItem = Operation()
                    opItem.action = m.group(2)
                else:
                    action = re.compile('\s*:([a-zA-Z0-9]*) \([a-zA-Z0-9]* / ([a-zA-Z0-9]*)\)')
                    n = action.match(line)
                    if n:
                        arg = args()
                        arg.type = n.group(1)
                        arg.content = n.group(2)
                        opItem.args.append(arg)

        if opItem.action != "":
            lu4r.opList.append(opItem)

        Logger.loginfo("Returning operation list - count : " + str(len(lu4r.opList)))
        if len(lu4r.opList) == 0:
            Logger.loginfo("fail")
            return 'fail'

        test = re.compile('.*[Ss]top.*')
        n = test.match(lu4r.opList[0].action)
        if n:
            Logger.loginfo("Action: Stop now!")
            userdata.DoNow = []
            userdata.DoNow.append('Stop')
            userdata.DoNow.append(0)
            userdata.HighFIFO = []
            userdata.MedFIFO = []
            userdata.LowFIFO = []

        for opitem in lu4r.opList:
            priority = 2
            Logger.loginfo("Action: "+opitem.action)
            ActionForm = None

            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Bring',
                    '.*[Bb]ring.*',
                    '.*([Tt]heme).*',
                    '.*([Aa]rea)|([Ss]ource).*',
                    '.*([Bb]eneficiary).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Follow',
                    '.*[Cc]otheme.*',
                    '.*([Tt]heme).*',
                    '.*([Gg]oal).*',
                    '.*([Aa]rea).*'
                    '.*([Pp]ath)|([Rr]oad).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder( opitem, [
                    'MoveBase',
                    '.*[Mm]otion.*',
                    '.*([Tt]heme)|([Aa]rea)|([Gg]oal).*',
                    '.*([Dd]irection).*',
                    '.*([Dd]istance).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Attach',
                    '.*[Aa]ttaching.*',
                    '.*([Ii]tem).*',
                    '.*([Gg]oal)|([Cc]onnector).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'LookAt',
                    '[Ii]specting'
                    '.*[Gg]round.*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Find',
                    '.*[Ll]ocating.*',
                    '.*[Ss]ought_entity.*',
                    '.*([Gg]round)|([[Ll]ocation]).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Place',
                    '.*[Pp]lacing.*',
                    '.*[Gg]oal.*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Give',
                    '.*[Gg]iving.*',
                    '.*[Rr]ecipient.*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Pick',
                    '.*[Mm]anipulation.*',
                    '.*([Ee]tity).*',
                    '.*([Aa]rea).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'LookAt',
                    '[Pp]erception'
                    '.*[Dd]irection.*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Pick',
                    '.*[Tt]aking.*',
                    '.*([Tt]heme).*',
                    '.*([Aa]rea).*'])
            if ActionForm == None:
                ActionForm = self.ActionFormBuilder(opitem, [
                    'Turn',
                    '.*[Cc]ange [Dd]irection.*',
                    '.*([Dd]irection).*'])

            if ActionForm != None:
                if priority == 1:
                    userdata.LowFIFO.append(ActionForm)
                if priority == 2:
                    userdata.MedFIFO.append(ActionForm)
                if priority == 3:
                    userdata.HighFIFO.append(ActionForm)

        return 'done'  # One of the outcomes declared above.

    def on_enter(self, userdata):
        # This method is called when the state becomes active, a transition from another state to this one is taken.
        # It is primarily used to start actions which are associated with this state.

        Logger.loginfo('Enter lu4r_parser')

        global semantic_map

    def ActionFormBuilder(self, opitem, positions):

        PreActionForm = None
        # - Generic ActionForm builder
        test = re.compile(positions[1])
        n = test.match(opitem.action)
        if n:
            PreActionForm = []
            PreActionForm.append(positions[0])
            i=2
            while i<len(positions):
                PreActionForm.append('')
                i = i + 1
            for arg in opitem.args:
                Logger.loginfo("arg-type: " + arg.type)
                Logger.loginfo("arg-content: " + arg.content)
                i = 2
                while i < len(positions):
                    test = re.compile(positions[i])
                    n = test.match(arg.type)
                    if n:
                        PreActionForm[i-1] = arg.content
                    i = i + 1
        return PreActionForm<|endoftext|>


<commit_before>import atexit
import weakref
from concurrent.futures import _base
from concurrent.futures import thread
from concurrent.futures.thread import ThreadPoolExecutor, BrokenThreadPool, _WorkItem
import typing as tp
import threading

from satella.time import measure

from satella.instrumentation.metrics.metric_types import EmptyMetric, SummaryMetric, HistogramMetric, MetricLevel, \
    CallableMetric
from ..aggregate import AggregateMetric

def _worker(executor_reference, work_queue, initializer, initargs):
    if initializer is not None:
        try:
            initializer(*initargs)
        except BaseException:
            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
            executor = executor_reference()
            if executor is not None:
                executor._initializer_failed()
            return
    try:
        while True:
            work_item = work_queue.get(block=True)
            if work_item is not None:
                # Measure the time spent in waiting
                executor = executor_reference()
                work_item.measure.stop()
                executor.waiting_time_metric.handle(executor.metric_level, work_item.measure())
                del executor

                with measure() as measurement:
                    work_item.run()

                executor = executor_reference()
                executor.executing_time_metric.handle(executor.metric_level, measurement())
                # Delete references to object. See issue16284
                del work_item

                # attempt to increment idle count

                if executor is not None:
                    executor._idle_semaphore.release()
                del executor
                continue

            executor = executor_reference()
            # Exit if:
            #   - The interpreter is shutting down OR
            #   - The executor that owns the worker has been collected OR
            #   - The executor that owns the worker has been shutdown.
            if thread._shutdown or executor is None or executor._shutdown:
                # Flag the executor as shutting down as early as possible if it
                # is not gc-ed yet.
                if executor is not None:
                    executor._shutdown = True
                # Notice other workers
                work_queue.put(None)
                return
            del executor
    except BaseException:
        _base.LOGGER.critical('Exception in worker', exc_info=True)

class MetrifiedThreadPoolExecutor(ThreadPoolExecutor):
    """
    A thread pool executor that provides execution statistics as metrics.

    :param time_spent_waiting: a metric (can be aggregate) to which times spent waiting in the queue will be deposited
    :param time_spent_executing: a metric (can be aggregate) to which times spent executing will be deposited
    :param waiting_tasks: a fresh CallableMetric that will be patched to yield the number of currently waiting tasks
    :param metric_level: a level with which to log to these two metrics
    """

    def __init__(self, max_workers=None, thread_name_prefix='',
                 initializer=None, initargs=(),
                 time_spent_waiting=None,
                 time_spent_executing=None,
                 waiting_tasks: tp.Optional[CallableMetric] = None,
                 metric_level: MetricLevel = MetricLevel.RUNTIME):
        super().__init__(max_workers, thread_name_prefix, initializer, initargs)
        self.waiting_time_metric = time_spent_waiting or EmptyMetric()
        self.executing_time_metric = time_spent_executing or EmptyMetric()
        self.metric_level = metric_level
        if waiting_tasks is not None:
            waiting_tasks.callable = lambda: self._work_queue.qsize()

    def submit(*args, **kwargs):
        if len(args) >= 2:
            self, fn, *args = args
        elif not args:
            raise TypeError("descriptor 'submit' of 'ThreadPoolExecutor' object "
                            "needs an argument")
        elif 'fn' in kwargs:
            fn = kwargs.pop('fn')
            self, *args = args
            import warnings
            warnings.warn("Passing 'fn' as keyword argument is deprecated",
                          DeprecationWarning, stacklevel=2)
        else:
            raise TypeError('submit expected at least 1 positional argument, '
                            'got %d' % (len(args)-1))

        with self._shutdown_lock:
            if self._broken:
                raise BrokenThreadPool(self._broken)

            if self._shutdown:
                raise RuntimeError('cannot schedule new futures after shutdown')
            if thread._shutdown:
                raise RuntimeError('cannot schedule new futures after '
                                   'interpreter shutdown')

            f = _base.Future()
            w = _WorkItem(f, fn, args, kwargs)
            w.measure = measure()
            self._work_queue.put(w)
            self._adjust_thread_count()
            return f
<commit_msg>patch-around for python 3.5<commit_after>import atexit
import weakref
from concurrent.futures import _base
from concurrent.futures import thread
from concurrent.futures.thread import ThreadPoolExecutor, _WorkItem
try:
    from concurrent.futures.thread import BrokenThreadPool
except ImportError:
    BrokenThreadPool = RuntimeError
import typing as tp
import threading

from satella.time import measure

from satella.instrumentation.metrics.metric_types import EmptyMetric, SummaryMetric, HistogramMetric, MetricLevel, \
    CallableMetric
from ..aggregate import AggregateMetric

def _worker(executor_reference, work_queue, initializer, initargs):
    if initializer is not None:
        try:
            initializer(*initargs)
        except BaseException:
            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
            executor = executor_reference()
            if executor is not None:
                executor._initializer_failed()
            return
    try:
        while True:
            work_item = work_queue.get(block=True)
            if work_item is not None:
                # Measure the time spent in waiting
                executor = executor_reference()
                work_item.measure.stop()
                executor.waiting_time_metric.handle(executor.metric_level, work_item.measure())
                del executor

                with measure() as measurement:
                    work_item.run()

                executor = executor_reference()
                executor.executing_time_metric.handle(executor.metric_level, measurement())
                # Delete references to object. See issue16284
                del work_item

                # attempt to increment idle count

                if executor is not None:
                    executor._idle_semaphore.release()
                del executor
                continue

            executor = executor_reference()
            # Exit if:
            #   - The interpreter is shutting down OR
            #   - The executor that owns the worker has been collected OR
            #   - The executor that owns the worker has been shutdown.
            if thread._shutdown or executor is None or executor._shutdown:
                # Flag the executor as shutting down as early as possible if it
                # is not gc-ed yet.
                if executor is not None:
                    executor._shutdown = True
                # Notice other workers
                work_queue.put(None)
                return
            del executor
    except BaseException:
        _base.LOGGER.critical('Exception in worker', exc_info=True)

class MetrifiedThreadPoolExecutor(ThreadPoolExecutor):
    """
    A thread pool executor that provides execution statistics as metrics.

    :param time_spent_waiting: a metric (can be aggregate) to which times spent waiting in the queue will be deposited
    :param time_spent_executing: a metric (can be aggregate) to which times spent executing will be deposited
    :param waiting_tasks: a fresh CallableMetric that will be patched to yield the number of currently waiting tasks
    :param metric_level: a level with which to log to these two metrics
    """

    def __init__(self, max_workers=None, thread_name_prefix='',
                 initializer=None, initargs=(),
                 time_spent_waiting=None,
                 time_spent_executing=None,
                 waiting_tasks: tp.Optional[CallableMetric] = None,
                 metric_level: MetricLevel = MetricLevel.RUNTIME):
        super().__init__(max_workers, thread_name_prefix, initializer, initargs)
        self.waiting_time_metric = time_spent_waiting or EmptyMetric()
        self.executing_time_metric = time_spent_executing or EmptyMetric()
        self.metric_level = metric_level
        if waiting_tasks is not None:
            waiting_tasks.callable = lambda: self._work_queue.qsize()

    def submit(*args, **kwargs):
        if len(args) >= 2:
            self, fn, *args = args
        elif not args:
            raise TypeError("descriptor 'submit' of 'ThreadPoolExecutor' object "
                            "needs an argument")
        elif 'fn' in kwargs:
            fn = kwargs.pop('fn')
            self, *args = args
            import warnings
            warnings.warn("Passing 'fn' as keyword argument is deprecated",
                          DeprecationWarning, stacklevel=2)
        else:
            raise TypeError('submit expected at least 1 positional argument, '
                            'got %d' % (len(args)-1))

        with self._shutdown_lock:
            if self._broken:
                raise BrokenThreadPool(self._broken)

            if self._shutdown:
                raise RuntimeError('cannot schedule new futures after shutdown')
            if thread._shutdown:
                raise RuntimeError('cannot schedule new futures after '
                                   'interpreter shutdown')

            f = _base.Future()
            w = _WorkItem(f, fn, args, kwargs)
            w.measure = measure()
            self._work_queue.put(w)
            self._adjust_thread_count()
            return f
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/trackHub",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/PXD000625-10090.pogo"
  } ]
}
"""

import time
# App imports
import config_manager
from pipelines.template_pipeline import Director, DirectorConfigurationManager

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def set_pipeline_arguments(pipeline_arguments):
    global __pipeline_arguments
    if __pipeline_arguments is None:
        __pipeline_arguments = pipeline_arguments
    return __pipeline_arguments

def get_pipeline_director():
    global __pipeline_director
    if __pipeline_director is None:
        __pipeline_director = TrackhubCreatorForProject(config_manager.read_config_from_file(__configuration_file),
                                                        __configuration_file,
                                                        __pipeline_arguments)
    return __pipeline_director

class ConfigManager(DirectorConfigurationManager):
    # Command Line Arguments for this pipeline look like
    #   # This is a JSON formatted file that contains all the relevant information needed for processing the project
    #   # data and create its trackhub
    #   project_data_file=project_data.json

    # Command Line Argument keys
    _CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE = 'project_data_file'

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(ConfigManager, self).__init__(configuration_object, configuration_file, pipeline_arguments)
        # Lazy Process command line arguments
        self.__pipeline_arguments_object = None
        self.__running_mode = None

# Models for dealing with the data file that describes the project
class ProjectTrackDescriptor:
    # TODO
    pass

class ProjectDescriptor:
    # Project Data File keys
    _PROJECT_DATA_FILE_KEY_TRACKHUB_NAME = 'trackHubName'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL = 'trackHubShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL = 'trackHubLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE = 'trackHubType'

    def __init__(self, project_data_file_path):
        self.__project_data_file_path = project_data_file_path

class TrackhubCreatorForProject(Director):
    # TODO
    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        runner_id = "{}-{}".format(__name__, time.time())
        super(TrackhubCreatorForProject, self).__init__(runner_id)
        self.__config_manager = ConfigManager(configuration_object, configuration_file, pipeline_arguments)
        self.__trackhub_destination_folder = None

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>_PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL key<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/trackHub",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/PXD000625-10090.pogo"
  } ]
}
"""

import time
# App imports
import config_manager
from pipelines.template_pipeline import Director, DirectorConfigurationManager

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def set_pipeline_arguments(pipeline_arguments):
    global __pipeline_arguments
    if __pipeline_arguments is None:
        __pipeline_arguments = pipeline_arguments
    return __pipeline_arguments

def get_pipeline_director():
    global __pipeline_director
    if __pipeline_director is None:
        __pipeline_director = TrackhubCreatorForProject(config_manager.read_config_from_file(__configuration_file),
                                                        __configuration_file,
                                                        __pipeline_arguments)
    return __pipeline_director

class ConfigManager(DirectorConfigurationManager):
    # Command Line Arguments for this pipeline look like
    #   # This is a JSON formatted file that contains all the relevant information needed for processing the project
    #   # data and create its trackhub
    #   project_data_file=project_data.json

    # Command Line Argument keys
    _CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE = 'project_data_file'

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(ConfigManager, self).__init__(configuration_object, configuration_file, pipeline_arguments)
        # Lazy Process command line arguments
        self.__pipeline_arguments_object = None
        self.__running_mode = None

# Models for dealing with the data file that describes the project
class ProjectTrackDescriptor:
    # TODO
    pass

class ProjectDescriptor:
    # Project Data File keys
    _PROJECT_DATA_FILE_KEY_TRACKHUB_NAME = 'trackHubName'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL = 'trackHubShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL = 'trackHubLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE = 'trackHubType'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL = 'trackHubEmail'

    def __init__(self, project_data_file_path):
        self.__project_data_file_path = project_data_file_path

class TrackhubCreatorForProject(Director):
    # TODO
    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        runner_id = "{}-{}".format(__name__, time.time())
        super(TrackhubCreatorForProject, self).__init__(runner_id)
        self.__config_manager = ConfigManager(configuration_object, configuration_file, pipeline_arguments)
        self.__trackhub_destination_folder = None

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2014 Ben Kurtovic <ben.kurtovic@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from gzip import GzipFile
from socket import timeout
from StringIO import StringIO
from time import sleep, time
from urllib2 import build_opener, URLError

from earwigbot import exceptions, importer
from earwigbot.wiki.copyvios.markov import MarkovChain, MarkovChainIntersection
from earwigbot.wiki.copyvios.parsers import ArticleTextParser, HTMLTextParser
from earwigbot.wiki.copyvios.result import CopyvioCheckResult
from earwigbot.wiki.copyvios.search import YahooBOSSSearchEngine

oauth = importer.new("oauth2")

__all__ = ["CopyvioMixIn"]

class CopyvioMixIn(object):
    """
    **EarwigBot: Wiki Toolset: Copyright Violation MixIn**

    This is a mixin that provides two public methods, :py:meth:`copyvio_check`
    and :py:meth:`copyvio_compare`. The former checks the page for copyright
    violations using a search engine API, and the latter compares the page
    against a given URL. Credentials for the search engine API are stored in
    the :py:class:`~earwigbot.wiki.site.Site`'s config.
    """
    EMPTY = MarkovChain("")
    EMPTY_INTERSECTION = MarkovChainIntersection(EMPTY, EMPTY)

    def __init__(self, site):
        self._search_config = site._search_config
        self._exclusions_db = self._search_config.get("exclusions_db")
        self._opener = build_opener()
        self._opener.addheaders = site._opener.addheaders

    def _open_url_ignoring_errors(self, url):
        """Open a URL and return its parsed content, or None.

        First, we will decompress the content if the headers contain "gzip" as
        its content encoding. Then, we will return the content stripped using
        an HTML parser if the headers indicate it is HTML, or return the
        content directly if it is plain text. If we don't understand the
        content type, we'll return None.

        If a URLError was raised while opening the URL or an IOError was raised
        while decompressing, None will be returned.
        """
        try:
            response = self._opener.open(url.encode("utf8"), timeout=5)
            result = response.read()
        except (URLError, timeout):
            return None

        if response.headers.get("Content-Encoding") == "gzip":
            stream = StringIO(result)
            gzipper = GzipFile(fileobj=stream)
            try:
                result = gzipper.read()
            except IOError:
                return None

        ctype_full = response.headers.get("Content-Type", "text/plain")
        ctype = ctype_full.split(" ", 1)[0]
        if ctype in ["text/html", "application/xhtml+xml"]:
            return HTMLTextParser(result).strip()
        elif ctype == "text/plain":
            return result.strip()
        else:
            return None

    def _select_search_engine(self):
        """Return a function that can be called to do web searches.

        The function takes one argument, a search query, and returns a list of
        URLs, ranked by importance. The underlying logic depends on the
        *engine* argument within our config; for example, if *engine* is
        "Yahoo! BOSS", we'll use YahooBOSSSearchEngine for querying.

        Raises UnknownSearchEngineError if the 'engine' listed in our config is
        unknown to us, and UnsupportedSearchEngineError if we are missing a
        required package or module, like oauth2 for "Yahoo! BOSS".
        """
        engine = self._search_config["engine"]
        credentials = self._search_config["credentials"]

        if engine == "Yahoo! BOSS":
            try:
                oauth.__version__  # Force-load the lazy module
            except ImportError:
                e = "Yahoo! BOSS requires the 'oauth2' package: https://github.com/simplegeo/python-oauth2"
                raise exceptions.UnsupportedSearchEngineError(e)
            return YahooBOSSSearchEngine(credentials, self._opener)

        raise exceptions.UnknownSearchEngineError(engine)

    def _copyvio_compare_content(self, article, url):
        """Return a number comparing an article and a URL.

        The *article* is a Markov chain, whereas the *url* is just a string
        that we'll try to open and read ourselves.
        """
        text = self._open_url_ignoring_errors(url)
        if not text:
            return 0, (self.EMPTY, self.EMPTY_INTERSECTION)

        source = MarkovChain(text)
        delta = MarkovChainIntersection(article, source)
        return float(delta.size()) / article.size(), (source, delta)

    def copyvio_check(self, min_confidence=0.5, max_queries=-1, max_time=-1,
                      interquery_sleep=1):
        """Check the page for copyright violations.

        Returns a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object
        with information on the results of the check.

        *max_queries* is self-explanatory; we will never make more than this
        number of queries in a given check. If it's lower than 0, we will not
        limit the number of queries.

        *max_time* can be set to prevent copyvio checks from taking longer than
        a set amount of time (generally around a minute), which can be useful
        if checks are called through a web server with timeouts. We will stop
        checking new URLs as soon as this limit is reached.

        *interquery_sleep* is the minimum amount of time we will sleep between
        search engine queries, in seconds.

        Raises :py:exc:`~earwigbot.exceptions.CopyvioCheckError` or subclasses
        (:py:exc:`~earwigbot.exceptions.UnknownSearchEngineError`,
        :py:exc:`~earwigbot.exceptions.SearchQueryError`, ...) on errors.
        """
        start_time = time()
        searcher = self._select_search_engine()
        if self._exclusions_db:
            self._exclusions_db.sync(self.site.name)
        handled_urls = []
        best_confidence = 0
        best_match = None
        num_queries = 0
        best_chains = (self.EMPTY, self.EMPTY_INTERSECTION)
        parser = ArticleTextParser(self.get())
        clean = parser.strip()
        chunks = parser.chunk(self._search_config["nltk_dir"], max_queries)
        article_chain = MarkovChain(clean)
        last_query = time()

        if article_chain.size() < 20:  # Auto-fail very small articles
            return CopyvioCheckResult(False, best_confidence, best_match,
                                      num_queries, 0, article_chain,
                                      best_chains)

        while (chunks and best_confidence < min_confidence and
               (max_queries < 0 or num_queries < max_queries)):
            chunk = chunks.pop(0)
            log = u"[[{0}]] -> querying {1} for {2!r}"
            self._logger.debug(log.format(self.title, searcher.name, chunk))
            urls = searcher.search(chunk)
            urls = [url for url in urls if url not in handled_urls]
            for url in urls:
                handled_urls.append(url)
                if self._exclusions_db:
                    if self._exclusions_db.check(self.site.name, url):
                        continue
                conf, chns = self._copyvio_compare_content(article_chain, url)
                if conf > best_confidence:
                    best_confidence = conf
                    best_match = url
                    best_chains = chns
                if time() - start_time > max_time:
                    break
            num_queries += 1
            if time() - start_time > max_time:
                break
            diff = time() - last_query
            if diff < interquery_sleep:
                sleep(interquery_sleep - diff)
            last_query = time()

        ctime = time() - start_time
        if best_confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2}; using {3} queries in {4} seconds)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          best_match, num_queries, ctime))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; using {2} queries in {3} seconds)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          num_queries, ctime))

        return CopyvioCheckResult(is_violation, best_confidence, best_match,
                                  num_queries, ctime, article_chain,
                                  best_chains)

    def copyvio_compare(self, url, min_confidence=0.5):
        """Check the page like :py:meth:`copyvio_check` against a specific URL.

        This is essentially a reduced version of the above - a copyivo
        comparison is made using Markov chains and the result is returned in a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object -
        but without using a search engine, since the suspected "violated" URL
        is supplied from the start.

        Its primary use is to generate a result when the URL is retrieved from
        a cache, like the one used in EarwigBot's Toolserver site. After a
        search is done, the resulting URL is stored in a cache for 24 hours so
        future checks against that page will not require another set of
        time-and-money-consuming search engine queries. However, the comparison
        itself (which includes the article's and the source's content) cannot
        be stored for data retention reasons, so a fresh comparison is made
        using this function.

        Since no searching is done, neither
        :py:exc:`~earwigbot.exceptions.UnknownSearchEngineError` nor
        :py:exc:`~earwigbot.exceptions.SearchQueryError` will be raised.
        """
        start_time = time()
        content = self.get()
        clean = ArticleTextParser(content).strip()
        article_chain = MarkovChain(clean)

        if not url:
            chns = (self.EMPTY, self.EMPTY_INTERSECTION)
            return CopyvioCheckResult(False, 0, url, 0, 0, article_chain, chns)

        confidence, chains = self._copyvio_compare_content(article_chain, url)
        ctime = time() - start_time
        if confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2}; {3} seconds)"
            self._logger.debug(log.format(self.title, confidence, url, ctime))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; URL: {2}; {3} seconds)"
            self._logger.debug(log.format(self.title, confidence, url, ctime))

        return CopyvioCheckResult(is_violation, confidence, url, 0, ctime,
                                  article_chain, chains)
<commit_msg>Split content type correctly.<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2014 Ben Kurtovic <ben.kurtovic@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from gzip import GzipFile
from socket import timeout
from StringIO import StringIO
from time import sleep, time
from urllib2 import build_opener, URLError

from earwigbot import exceptions, importer
from earwigbot.wiki.copyvios.markov import MarkovChain, MarkovChainIntersection
from earwigbot.wiki.copyvios.parsers import ArticleTextParser, HTMLTextParser
from earwigbot.wiki.copyvios.result import CopyvioCheckResult
from earwigbot.wiki.copyvios.search import YahooBOSSSearchEngine

oauth = importer.new("oauth2")

__all__ = ["CopyvioMixIn"]

class CopyvioMixIn(object):
    """
    **EarwigBot: Wiki Toolset: Copyright Violation MixIn**

    This is a mixin that provides two public methods, :py:meth:`copyvio_check`
    and :py:meth:`copyvio_compare`. The former checks the page for copyright
    violations using a search engine API, and the latter compares the page
    against a given URL. Credentials for the search engine API are stored in
    the :py:class:`~earwigbot.wiki.site.Site`'s config.
    """
    EMPTY = MarkovChain("")
    EMPTY_INTERSECTION = MarkovChainIntersection(EMPTY, EMPTY)

    def __init__(self, site):
        self._search_config = site._search_config
        self._exclusions_db = self._search_config.get("exclusions_db")
        self._opener = build_opener()
        self._opener.addheaders = site._opener.addheaders

    def _open_url_ignoring_errors(self, url):
        """Open a URL and return its parsed content, or None.

        First, we will decompress the content if the headers contain "gzip" as
        its content encoding. Then, we will return the content stripped using
        an HTML parser if the headers indicate it is HTML, or return the
        content directly if it is plain text. If we don't understand the
        content type, we'll return None.

        If a URLError was raised while opening the URL or an IOError was raised
        while decompressing, None will be returned.
        """
        try:
            response = self._opener.open(url.encode("utf8"), timeout=5)
            result = response.read()
        except (URLError, timeout):
            return None

        if response.headers.get("Content-Encoding") == "gzip":
            stream = StringIO(result)
            gzipper = GzipFile(fileobj=stream)
            try:
                result = gzipper.read()
            except IOError:
                return None

        ctype_full = response.headers.get("Content-Type", "text/plain")
        ctype = ctype_full.split("; ", 1)[0]
        if ctype in ["text/html", "application/xhtml+xml"]:
            return HTMLTextParser(result).strip()
        elif ctype == "text/plain":
            return result.strip()
        else:
            return None

    def _select_search_engine(self):
        """Return a function that can be called to do web searches.

        The function takes one argument, a search query, and returns a list of
        URLs, ranked by importance. The underlying logic depends on the
        *engine* argument within our config; for example, if *engine* is
        "Yahoo! BOSS", we'll use YahooBOSSSearchEngine for querying.

        Raises UnknownSearchEngineError if the 'engine' listed in our config is
        unknown to us, and UnsupportedSearchEngineError if we are missing a
        required package or module, like oauth2 for "Yahoo! BOSS".
        """
        engine = self._search_config["engine"]
        credentials = self._search_config["credentials"]

        if engine == "Yahoo! BOSS":
            try:
                oauth.__version__  # Force-load the lazy module
            except ImportError:
                e = "Yahoo! BOSS requires the 'oauth2' package: https://github.com/simplegeo/python-oauth2"
                raise exceptions.UnsupportedSearchEngineError(e)
            return YahooBOSSSearchEngine(credentials, self._opener)

        raise exceptions.UnknownSearchEngineError(engine)

    def _copyvio_compare_content(self, article, url):
        """Return a number comparing an article and a URL.

        The *article* is a Markov chain, whereas the *url* is just a string
        that we'll try to open and read ourselves.
        """
        text = self._open_url_ignoring_errors(url)
        if not text:
            return 0, (self.EMPTY, self.EMPTY_INTERSECTION)

        source = MarkovChain(text)
        delta = MarkovChainIntersection(article, source)
        return float(delta.size()) / article.size(), (source, delta)

    def copyvio_check(self, min_confidence=0.5, max_queries=-1, max_time=-1,
                      interquery_sleep=1):
        """Check the page for copyright violations.

        Returns a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object
        with information on the results of the check.

        *max_queries* is self-explanatory; we will never make more than this
        number of queries in a given check. If it's lower than 0, we will not
        limit the number of queries.

        *max_time* can be set to prevent copyvio checks from taking longer than
        a set amount of time (generally around a minute), which can be useful
        if checks are called through a web server with timeouts. We will stop
        checking new URLs as soon as this limit is reached.

        *interquery_sleep* is the minimum amount of time we will sleep between
        search engine queries, in seconds.

        Raises :py:exc:`~earwigbot.exceptions.CopyvioCheckError` or subclasses
        (:py:exc:`~earwigbot.exceptions.UnknownSearchEngineError`,
        :py:exc:`~earwigbot.exceptions.SearchQueryError`, ...) on errors.
        """
        start_time = time()
        searcher = self._select_search_engine()
        if self._exclusions_db:
            self._exclusions_db.sync(self.site.name)
        handled_urls = []
        best_confidence = 0
        best_match = None
        num_queries = 0
        best_chains = (self.EMPTY, self.EMPTY_INTERSECTION)
        parser = ArticleTextParser(self.get())
        clean = parser.strip()
        chunks = parser.chunk(self._search_config["nltk_dir"], max_queries)
        article_chain = MarkovChain(clean)
        last_query = time()

        if article_chain.size() < 20:  # Auto-fail very small articles
            return CopyvioCheckResult(False, best_confidence, best_match,
                                      num_queries, 0, article_chain,
                                      best_chains)

        while (chunks and best_confidence < min_confidence and
               (max_queries < 0 or num_queries < max_queries)):
            chunk = chunks.pop(0)
            log = u"[[{0}]] -> querying {1} for {2!r}"
            self._logger.debug(log.format(self.title, searcher.name, chunk))
            urls = searcher.search(chunk)
            urls = [url for url in urls if url not in handled_urls]
            for url in urls:
                handled_urls.append(url)
                if self._exclusions_db:
                    if self._exclusions_db.check(self.site.name, url):
                        continue
                conf, chns = self._copyvio_compare_content(article_chain, url)
                if conf > best_confidence:
                    best_confidence = conf
                    best_match = url
                    best_chains = chns
                if time() - start_time > max_time:
                    break
            num_queries += 1
            if time() - start_time > max_time:
                break
            diff = time() - last_query
            if diff < interquery_sleep:
                sleep(interquery_sleep - diff)
            last_query = time()

        ctime = time() - start_time
        if best_confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2}; using {3} queries in {4} seconds)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          best_match, num_queries, ctime))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; using {2} queries in {3} seconds)"
            self._logger.debug(log.format(self.title, best_confidence,
                                          num_queries, ctime))

        return CopyvioCheckResult(is_violation, best_confidence, best_match,
                                  num_queries, ctime, article_chain,
                                  best_chains)

    def copyvio_compare(self, url, min_confidence=0.5):
        """Check the page like :py:meth:`copyvio_check` against a specific URL.

        This is essentially a reduced version of the above - a copyivo
        comparison is made using Markov chains and the result is returned in a
        :py:class:`~earwigbot.wiki.copyvios.result.CopyvioCheckResult` object -
        but without using a search engine, since the suspected "violated" URL
        is supplied from the start.

        Its primary use is to generate a result when the URL is retrieved from
        a cache, like the one used in EarwigBot's Toolserver site. After a
        search is done, the resulting URL is stored in a cache for 24 hours so
        future checks against that page will not require another set of
        time-and-money-consuming search engine queries. However, the comparison
        itself (which includes the article's and the source's content) cannot
        be stored for data retention reasons, so a fresh comparison is made
        using this function.

        Since no searching is done, neither
        :py:exc:`~earwigbot.exceptions.UnknownSearchEngineError` nor
        :py:exc:`~earwigbot.exceptions.SearchQueryError` will be raised.
        """
        start_time = time()
        content = self.get()
        clean = ArticleTextParser(content).strip()
        article_chain = MarkovChain(clean)

        if not url:
            chns = (self.EMPTY, self.EMPTY_INTERSECTION)
            return CopyvioCheckResult(False, 0, url, 0, 0, article_chain, chns)

        confidence, chains = self._copyvio_compare_content(article_chain, url)
        ctime = time() - start_time
        if confidence >= min_confidence:
            is_violation = True
            log = u"Violation detected for [[{0}]] (confidence: {1}; URL: {2}; {3} seconds)"
            self._logger.debug(log.format(self.title, confidence, url, ctime))
        else:
            is_violation = False
            log = u"No violation for [[{0}]] (confidence: {1}; URL: {2}; {3} seconds)"
            self._logger.debug(log.format(self.title, confidence, url, ctime))

        return CopyvioCheckResult(is_violation, confidence, url, 0, ctime,
                                  article_chain, chains)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
    Quicktime/MP4 Fast Start
    ------------------------
    Enable streaming and pseudo-streaming of Quicktime and MP4 files by
    moving metadata and offset information to the front of the file.

    This program is based on qt-faststart.c from the ffmpeg project, which is
    released into the public domain, as well as ISO 14496-12:2005 (the official
    spec for MP4), which can be obtained from the ISO or found online.

    The goals of this project are to run anywhere without compilation (in
    particular, many Windows and Mac OS X users have trouble getting
    qt-faststart.c compiled), to run about as fast as the C version, to be more
    user friendly, and to use less actual lines of code doing so.

    Features
    --------

        * Works everywhere Python can be installed
        * Handles both 32-bit (stco) and 64-bit (co64) atoms
        * Handles any file where the mdat atom is before the moov atom
        * Preserves the order of other atoms
        * Can replace the original file (if given no output file)

    Usage
    -----
    See `qt-faststart.py --help` for more info! If outfile is not present then
    the infile is overwritten.

        $ qtfaststart.py infile [outfile]

    History
    -------
     * 2009-11-05: Add --sample option. Version bump to 1.3.
     * 2009-03-13: Update to be more library-friendly by using logging module,
                   rename fast_start => process, version bump to 1.2
     * 2008-10-04: Bug fixes, support multiple atoms of the same type,
                   version bump to 1.1
     * 2008-09-02: Initial release

    License
    -------
    Copyright (C) 2008 - 2009  Daniel G. Taylor <dan@programmer-art.org>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import logging
import os
import shutil
import struct
import sys
import tempfile

from optparse import OptionParser
from StringIO import StringIO

VERSION = "1.3"
CHUNK_SIZE = 8192

log = logging.getLogger("qtfaststart")

class FastStartException(Exception):
    pass

def read_atom(datastream):
    """
        Read an atom and return a tuple of (size, type) where size is the size
        in bytes (including the 8 bytes already read) and type is a "fourcc"
        like "ftyp" or "moov".
    """
    return struct.unpack(">L4s", datastream.read(8))

def get_index(datastream):
    """
        Return an index of top level atoms, their absolute byte-position in the
        file and their size in a list:

        index = [
            ("ftyp", 0, 24),
            ("moov", 25, 2658),
            ("free", 2683, 8),
            ...
        ]

        The tuple elements will be in the order that they appear in the file.
    """
    index = []

    log.debug("Getting index of top level atoms...")

    # Read atoms until we catch an error
    while(datastream):
        try:
            skip = 8
            atom_size, atom_type = read_atom(datastream)
            if atom_size == 1:
                atom_size = struct.unpack(">Q", datastream.read(8))[0]
                skip = 16
            log.debug("%s: %s" % (atom_type, atom_size))
        except:
            break

        index.append((atom_type, datastream.tell() - skip, atom_size))
        datastream.seek(atom_size - skip, os.SEEK_CUR)

    # Make sure the atoms we need exist
    top_level_atoms = set([item[0] for item in index])
    for key in ["ftyp", "moov", "mdat"]:
        if key not in top_level_atoms:
            log.error("%s atom not found, is this a valid MOV/MP4 file?" % key)
            raise FastStartException()

    return index

def find_atoms(size, datastream):
    """
        This function is a generator that will yield either "stco" or "co64"
        when either atom is found. datastream can be assumed to be 8 bytes
        into the stco or co64 atom when the value is yielded.

        It is assumed that datastream will be at the end of the atom after
        the value has been yielded and processed.

        size is the number of bytes to the end of the atom in the datastream.
    """
    stop = datastream.tell() + size

    while datastream.tell() < stop:
        try:
            atom_size, atom_type = read_atom(datastream)
        except:
            log.exception("Error reading next atom!")
            raise FastStartException()

        if atom_type in ["trak", "mdia", "minf", "stbl"]:
            # Known ancestor atom of stco or co64, search within it!
            for atype in find_atoms(atom_size - 8, datastream):
                yield atype
        elif atom_type in ["stco", "co64"]:
            yield atom_type
        else:
            # Ignore this atom, seek to the end of it.
            datastream.seek(atom_size - 8, os.SEEK_CUR)

def process(infilename, outfilename, limit = 0):
    """
        Convert a Quicktime/MP4 file for streaming by moving the metadata to
        the front of the file. This method writes a new file.

        If limit is set to something other than zero it will be used as the
        number of bytes to write of the atoms following the moov atom. This
        is very useful to create a small sample of a file with full headers,
        which can then be used in bug reports and such.
    """
    datastream = open(infilename, "rb")

    # Get the top level atom index
    index = get_index(datastream)

    # Make sure moov occurs AFTER mdat, otherwise no need to run!
    for atom, pos, size in index:
        # The atoms are guaranteed to exist from get_index above!
        if atom == "moov":
            moov_pos = pos
            moov_size = size
        elif atom == "mdat":
            mdat_pos = pos

    if moov_pos < mdat_pos:
        log.error("This file appears to already be setup for streaming!")
        raise FastStartException()

    # Read and fix moov
    datastream.seek(moov_pos)
    moov = StringIO(datastream.read(moov_size))

    # Ignore moov identifier and size, start reading children
    moov.seek(8)

    for atom_type in find_atoms(moov_size - 8, moov):
        # Read either 32-bit or 64-bit offsets
        ctype, csize = atom_type == "stco" and ("L", 4) or ("Q", 8)

        # Get number of entries
        version, entry_count = struct.unpack(">2L", moov.read(8))

        log.info("Patching %s with %d entries" % (atom_type, entry_count))

        # Read entries
        entries = struct.unpack(">" + ctype * entry_count,
                                moov.read(csize * entry_count))

        # Patch and write entries
        moov.seek(-csize * entry_count, os.SEEK_CUR)
        moov.write(struct.pack(">" + ctype * entry_count,
                               *[entry + moov_size for entry in entries]))

    log.info("Writing output...")
    outfile = open(outfilename, "wb")

    # Write ftype
    for atom, pos, size in index:
        if atom == "ftyp":
            datastream.seek(pos)
            outfile.write(datastream.read(size))

    # Write moov
    moov.seek(0)
    outfile.write(moov.read())

    # Write the rest
    written = 0
    atoms = [item for item in index if item[0] not in ["ftyp", "moov"]]
    for atom, pos, size in atoms:
        datastream.seek(pos)

        # Write in chunks to not use too much memory
        for x in range(size / CHUNK_SIZE):
            outfile.write(datastream.read(CHUNK_SIZE))
            written += CHUNK_SIZE
            if limit and written >= limit:
                # A limit was set and we've just passed it, stop writing!
                break

        if size % CHUNK_SIZE:
            outfile.write(datastream.read(size % CHUNK_SIZE))
            written += (size % CHUNK_SIZE)
            if limit and written >= limit:
                # A limit was set and we've just passed it, stop writing!
                break

if __name__ == "__main__":
    logging.basicConfig(level = logging.INFO, stream = sys.stdout,
                        format = "%(message)s")

    parser = OptionParser(usage="%prog [options] infile [outfile]",
                          version="%prog " + VERSION)

    parser.add_option("-d", "--debug", dest="debug", default=False,
                      action="store_true",
                      help="Enable debug output")
    parser.add_option("-l", "--list", dest="list", default=False,
                      action="store_true",
                      help="List top level atoms")
    parser.add_option("-s", "--sample", dest="sample", default=False,
                      action="store_true",
                      help="Create a small sample of the input file")

    options, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        raise SystemExit(1)

    if options.debug:
        logging.getLogger().setLevel(logging.DEBUG)

    if options.list:
        index = get_index(open(args[0], "rb"))

        for atom, pos, size in index:
            print atom, "(" + str(size) + " bytes)"

        raise SystemExit

    if len(args) == 1:
        # Replace the original file!
        if options.sample:
            print "Please pass an output filename when used with --sample!"
            raise SystemExit(1)

        tmp, outfile = tempfile.mkstemp()
        os.close(tmp)
    else:
        outfile = args[1]

    limit = 0
    if options.sample:
        # Create a small sample (4 MiB)
        limit = 4 * (1024 ** 2)

    try:
        process(args[0], outfile, limit = limit)
    except FastStartException:
        # A log message was printed, so exit with an error code
        raise SystemExit(1)

    if len(args) == 1:
        # Move temp file to replace original
        shutil.move(outfile, args[0])

<commit_msg>Some files seem to be missing an ftyp atom and fail. Ignore the missing atom for now as long as those files still have moov and mdat atoms. This fixes a couple samples I ran into using rgb24 raw video in mov<commit_after>#!/usr/bin/env python

"""
    Quicktime/MP4 Fast Start
    ------------------------
    Enable streaming and pseudo-streaming of Quicktime and MP4 files by
    moving metadata and offset information to the front of the file.

    This program is based on qt-faststart.c from the ffmpeg project, which is
    released into the public domain, as well as ISO 14496-12:2005 (the official
    spec for MP4), which can be obtained from the ISO or found online.

    The goals of this project are to run anywhere without compilation (in
    particular, many Windows and Mac OS X users have trouble getting
    qt-faststart.c compiled), to run about as fast as the C version, to be more
    user friendly, and to use less actual lines of code doing so.

    Features
    --------

        * Works everywhere Python can be installed
        * Handles both 32-bit (stco) and 64-bit (co64) atoms
        * Handles any file where the mdat atom is before the moov atom
        * Preserves the order of other atoms
        * Can replace the original file (if given no output file)

    Usage
    -----
    See `qt-faststart.py --help` for more info! If outfile is not present then
    the infile is overwritten.

        $ qtfaststart.py infile [outfile]

    History
    -------
     * 2009-11-05: Add --sample option. Version bump to 1.3.
     * 2009-03-13: Update to be more library-friendly by using logging module,
                   rename fast_start => process, version bump to 1.2
     * 2008-10-04: Bug fixes, support multiple atoms of the same type,
                   version bump to 1.1
     * 2008-09-02: Initial release

    License
    -------
    Copyright (C) 2008 - 2009  Daniel G. Taylor <dan@programmer-art.org>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import logging
import os
import shutil
import struct
import sys
import tempfile

from optparse import OptionParser
from StringIO import StringIO

VERSION = "1.3"
CHUNK_SIZE = 8192

log = logging.getLogger("qtfaststart")

class FastStartException(Exception):
    pass

def read_atom(datastream):
    """
        Read an atom and return a tuple of (size, type) where size is the size
        in bytes (including the 8 bytes already read) and type is a "fourcc"
        like "ftyp" or "moov".
    """
    return struct.unpack(">L4s", datastream.read(8))

def get_index(datastream):
    """
        Return an index of top level atoms, their absolute byte-position in the
        file and their size in a list:

        index = [
            ("ftyp", 0, 24),
            ("moov", 25, 2658),
            ("free", 2683, 8),
            ...
        ]

        The tuple elements will be in the order that they appear in the file.
    """
    index = []

    log.debug("Getting index of top level atoms...")

    # Read atoms until we catch an error
    while(datastream):
        try:
            skip = 8
            atom_size, atom_type = read_atom(datastream)
            if atom_size == 1:
                atom_size = struct.unpack(">Q", datastream.read(8))[0]
                skip = 16
            log.debug("%s: %s" % (atom_type, atom_size))
        except:
            break

        index.append((atom_type, datastream.tell() - skip, atom_size))
        datastream.seek(atom_size - skip, os.SEEK_CUR)

    # Make sure the atoms we need exist
    top_level_atoms = set([item[0] for item in index])
    for key in ["moov", "mdat"]:
        if key not in top_level_atoms:
            log.error("%s atom not found, is this a valid MOV/MP4 file?" % key)
            raise FastStartException()

    return index

def find_atoms(size, datastream):
    """
        This function is a generator that will yield either "stco" or "co64"
        when either atom is found. datastream can be assumed to be 8 bytes
        into the stco or co64 atom when the value is yielded.

        It is assumed that datastream will be at the end of the atom after
        the value has been yielded and processed.

        size is the number of bytes to the end of the atom in the datastream.
    """
    stop = datastream.tell() + size

    while datastream.tell() < stop:
        try:
            atom_size, atom_type = read_atom(datastream)
        except:
            log.exception("Error reading next atom!")
            raise FastStartException()

        if atom_type in ["trak", "mdia", "minf", "stbl"]:
            # Known ancestor atom of stco or co64, search within it!
            for atype in find_atoms(atom_size - 8, datastream):
                yield atype
        elif atom_type in ["stco", "co64"]:
            yield atom_type
        else:
            # Ignore this atom, seek to the end of it.
            datastream.seek(atom_size - 8, os.SEEK_CUR)

def process(infilename, outfilename, limit = 0):
    """
        Convert a Quicktime/MP4 file for streaming by moving the metadata to
        the front of the file. This method writes a new file.

        If limit is set to something other than zero it will be used as the
        number of bytes to write of the atoms following the moov atom. This
        is very useful to create a small sample of a file with full headers,
        which can then be used in bug reports and such.
    """
    datastream = open(infilename, "rb")

    # Get the top level atom index
    index = get_index(datastream)

    # Make sure moov occurs AFTER mdat, otherwise no need to run!
    for atom, pos, size in index:
        # The atoms are guaranteed to exist from get_index above!
        if atom == "moov":
            moov_pos = pos
            moov_size = size
        elif atom == "mdat":
            mdat_pos = pos

    if moov_pos < mdat_pos:
        log.error("This file appears to already be setup for streaming!")
        raise FastStartException()

    # Read and fix moov
    datastream.seek(moov_pos)
    moov = StringIO(datastream.read(moov_size))

    # Ignore moov identifier and size, start reading children
    moov.seek(8)

    for atom_type in find_atoms(moov_size - 8, moov):
        # Read either 32-bit or 64-bit offsets
        ctype, csize = atom_type == "stco" and ("L", 4) or ("Q", 8)

        # Get number of entries
        version, entry_count = struct.unpack(">2L", moov.read(8))

        log.info("Patching %s with %d entries" % (atom_type, entry_count))

        # Read entries
        entries = struct.unpack(">" + ctype * entry_count,
                                moov.read(csize * entry_count))

        # Patch and write entries
        moov.seek(-csize * entry_count, os.SEEK_CUR)
        moov.write(struct.pack(">" + ctype * entry_count,
                               *[entry + moov_size for entry in entries]))

    log.info("Writing output...")
    outfile = open(outfilename, "wb")

    # Write ftype
    for atom, pos, size in index:
        if atom == "ftyp":
            datastream.seek(pos)
            outfile.write(datastream.read(size))

    # Write moov
    moov.seek(0)
    outfile.write(moov.read())

    # Write the rest
    written = 0
    atoms = [item for item in index if item[0] not in ["ftyp", "moov"]]
    for atom, pos, size in atoms:
        datastream.seek(pos)

        # Write in chunks to not use too much memory
        for x in range(size / CHUNK_SIZE):
            outfile.write(datastream.read(CHUNK_SIZE))
            written += CHUNK_SIZE
            if limit and written >= limit:
                # A limit was set and we've just passed it, stop writing!
                break

        if size % CHUNK_SIZE:
            outfile.write(datastream.read(size % CHUNK_SIZE))
            written += (size % CHUNK_SIZE)
            if limit and written >= limit:
                # A limit was set and we've just passed it, stop writing!
                break

if __name__ == "__main__":
    logging.basicConfig(level = logging.INFO, stream = sys.stdout,
                        format = "%(message)s")

    parser = OptionParser(usage="%prog [options] infile [outfile]",
                          version="%prog " + VERSION)

    parser.add_option("-d", "--debug", dest="debug", default=False,
                      action="store_true",
                      help="Enable debug output")
    parser.add_option("-l", "--list", dest="list", default=False,
                      action="store_true",
                      help="List top level atoms")
    parser.add_option("-s", "--sample", dest="sample", default=False,
                      action="store_true",
                      help="Create a small sample of the input file")

    options, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        raise SystemExit(1)

    if options.debug:
        logging.getLogger().setLevel(logging.DEBUG)

    if options.list:
        index = get_index(open(args[0], "rb"))

        for atom, pos, size in index:
            print atom, "(" + str(size) + " bytes)"

        raise SystemExit

    if len(args) == 1:
        # Replace the original file!
        if options.sample:
            print "Please pass an output filename when used with --sample!"
            raise SystemExit(1)

        tmp, outfile = tempfile.mkstemp()
        os.close(tmp)
    else:
        outfile = args[1]

    limit = 0
    if options.sample:
        # Create a small sample (4 MiB)
        limit = 4 * (1024 ** 2)

    try:
        process(args[0], outfile, limit = limit)
    except FastStartException:
        # A log message was printed, so exit with an error code
        raise SystemExit(1)

    if len(args) == 1:
        # Move temp file to replace original
        shutil.move(outfile, args[0])

<|endoftext|>


<commit_before>## Justin FlicK
## Twitter Scraping and Sentiment Analysis
## Copyright 2017 Licensed under MIT License
## A GUI program for running twitter data through a Naive Bayes sentiment analysis to determine a positive or negative response.
##This program assumes you already have your dataset as a JSON file. If not, just use scraper_and_analysis.py to do everything!

import easygui as g #library for GUI
import collections
import sys
import webbrowser
import json ## library for manipulating JSON files
from datetime import datetime
from os.path import isfile
from json import dump
import logging
from twitterscraper import query_tweets #library for scraping
from twitterscraper.query import query_all_tweets
from textblob import TextBlob #library for sentiment analysis
import csv #library for reading/writing CSV

##Opens a GUI on start

version = 'Twitter Sentiment Analysis Tool 1.1'

options = ['Start', 'Developer Page', 'Exit']

button = g.buttonbox('Welcome to Twitter Sentiment Analysis Tool' + '\n' + '\n'  + '\n' + '\n' + '\n' + 'Created by Justin Flick, Copyright 2017 Licensed Under MIT License' , title = version, choices = options)

if button == options[0]:
    pass
if button == options[1]:
    webbrowser.open('https://github.com/Jflick58', new=0, autoraise=True)
if button == options[2]:
    sys.exit()

msg = "Enter your query information. Output will be in the form of a .json file"
title = version
fieldNames = ["Input file path (use absolute path)", "Output file name"]
fieldValues = []  # we start with blanks for the values
fieldValues = g.multenterbox(msg,title, fieldNames)

input = fieldValues[0]
output = fieldValues[1]

#converts json to python objects and prepares it to be written to a csv

data_json = open(input, mode='r').read() #reads in the JSON file into Python as a string
data_python = json.loads(data_json) #turns the string into a json Python object

csv_out = open(output + '.csv', mode='w') #opens csv file
writer = csv.writer(csv_out) #create the csv writer object

fields = ['text', 'timestamp', 'polarity', 'subjectivity', 'sentiment'] #field names
writer.writerow(fields) #writes field

for line in data_python:

   #performs the sentiment analysis and classifies it

    print(line.get('text').encode('unicode_escape'))
    analysis = TextBlob(line.get('text'))

    def get_label(analysis, threshold = 0):
        if analysis.sentiment[0]>threshold:
            return 'Positive'
        elif analysis.sentiment[0]<threshold:
            return 'Negative'
        else:
            return 'Neutral'
    print(analysis.sentiment,get_label(analysis)) #print the results
    print("  ")

    #writes a row and gets the fields from the json object and the sentiment analysis
    writer.writerow([
                     line.get('text').encode('unicode_escape'), #unicode escape to fix emoji issue
                     line.get('timestamp'),
                     analysis.sentiment.polarity,
                     analysis.sentiment.subjectivity,
                     get_label(analysis)])
else:
	csv_out.close() #saves the file and closes it
	print('Thank you for using this program')
	sys.exit('goodbye')#end program<commit_msg>chg: dev: reformat for flake8<commit_after>"""sentiment analysis module."""
# Justin FlicK
# Twitter Scraping and Sentiment Analysis
# Copyright 2017 Licensed under MIT License
# A GUI program for running twitter data through a Naive Bayes sentiment analysis
# to determine a positive or negative response.
# This program assumes you already have your dataset as a JSON file.
# If not, just use scraper_and_analysis.py to do everything!

# from datetime import datetime  # imported but unused
# from json import dump  # imported but unused
# from os.path import isfile  # imported but unused
# import collections
import csv  # library for reading/writing CSV
import json  # library for manipulating JSON files
# import logging  # imported but unused
import sys
import webbrowser

from textblob import TextBlob  # library for sentiment analysis
# from twitterscraper import query_tweets  # library for scraping  # imported but unused
# from twitterscraper.query import query_all_tweets  # imported but unused
import easygui as g  # library for GUI

def analyze_sentiment():
    """analyze sentiment."""
    # Opens a GUI on start

    version = 'Twitter Sentiment Analysis Tool 1.1'

    options = ['Start', 'Developer Page', 'Exit']

    button = g.buttonbox(
        'Welcome to Twitter Sentiment Analysis Tool' +
        '\n' + '\n' + '\n' + '\n' + '\n' +
        'Created by Justin Flick, Copyright 2017 Licensed Under MIT License',
        title=version, choices=options)

    if button == options[0]:
        pass
    if button == options[1]:
        webbrowser.open('https://github.com/Jflick58', new=0, autoraise=True)
    if button == options[2]:
        sys.exit()

    msg = "Enter your query information. Output will be in the form of a .json file"
    title = version
    fieldNames = ["Input file path (use absolute path)", "Output file name"]  # NOQA
    fieldValues = []  # we start with blanks for the values  # NOQA
    fieldValues = g.multenterbox(msg, title, fieldNames)  # NOQA

    input = fieldValues[0]
    output = fieldValues[1]

    # converts json to python objects and prepares it to be written to a csv

    data_json = open(input, mode='r').read()  # reads in the JSON file into Python as a string
    data_python = json.loads(data_json)  # turns the string into a json Python object

    csv_out = open(output + '.csv', mode='w')  # opens csv file
    writer = csv.writer(csv_out)  # create the csv writer object

    fields = ['text', 'timestamp', 'polarity', 'subjectivity', 'sentiment']  # field names
    writer.writerow(fields)  # writes field

    for line in data_python:

        # performs the sentiment analysis and classifies it

        print(line.get('text').encode('unicode_escape'))
        analysis = TextBlob(line.get('text'))

        def get_label(analysis, threshold=0):
            if analysis.sentiment[0] > threshold:
                return 'Positive'
            elif analysis.sentiment[0] < threshold:
                return 'Negative'
            else:
                return 'Neutral'
        print(analysis.sentiment, get_label(analysis))  # print the results
        print("  ")

        # writes a row and gets the fields from the json object and the sentiment analysis
        writer.writerow([
            line.get('text').encode('unicode_escape'),  # unicode escape to fix emoji issue
            line.get('timestamp'),
            analysis.sentiment.polarity,
            analysis.sentiment.subjectivity,
            get_label(analysis)
        ])
    else:
        csv_out.close()  # saves the file and closes it
        print('Thank you for using this program')
        sys.exit('goodbye')  # end program
<|endoftext|>