<commit_before>from .socket import Socket
from socket import timeout as sockTimeout
from ... import Logger, DummyLog
import threading

_QUEUEDSOCKET_IDENTIFIER_ = '<agutil.io.queuedsocket:1.0.0>'

class QueuedSocket(Socket):
    def __init__(self, socket, logmethod=DummyLog, _skipIdentifier=False, _useIdentifier=_QUEUEDSOCKET_IDENTIFIER_):
        if not isinstance(socket, Socket):
            raise TypeError("socket argument must be of type agutil.io.Socket")
        super().__init__(socket.addr, socket.port, socket.sock, True)
        self.incoming = {'__orphan__': []}
        self.outgoing = {}
        self.outgoing_channels = []
        self._shutdown = False
        self.datalock = threading.Condition()
        self.iolock = threading.Condition()
        self.log = logmethod
        if isinstance(self.log, Logger):
            self.log = self.log.bindToSender("QueuedSocket")
        self.log("The underlying Socket has been initialized.  Starting background thread...")
        self._thread = threading.Thread(
            target=QueuedSocket._worker,
            args=(self,),
            name="QueuedSocket background thread",
            daemon=True
        )
        self._thread.start()
        if not _skipIdentifier:
            QueuedSocket.send(self, _useIdentifier, '__protocol__')
            remoteID = QueuedSocket.recv(self, '__protocol__', True)
            if remoteID != _useIdentifier:
                self.log("The remote socket provided an invalid QueuedSocket protocol identifier. (Theirs: %s) (Ours: %s)" % (
                    remoteID,
                    _useIdentifier
                ), "WARN")
                self.close()
                raise ValueError("The remote socket provided an invalid identifier at the QueuedSocket level")

    def close(self, timeout=1):
        if self._shutdown:
            return
        self.log("Shutdown initiated")
        self.datalock.acquire()
        self._shutdown = True
        self.datalock.release()
        self._thread.join(timeout)
        super().close()
        self.log.close()

    def send(self, msg, channel='__orphan__'):
        if self._shutdown:
            self.log("Attempt to use the QueuedSocket after shutdown", "WARN")
            raise IOError("This QueuedSocket has already been closed")
        if '|' in channel:
            self.log("Attempt to send message over illegal channel name", "WARN")
            raise ValueError("Channel names cannot contain '|' characters (ascii 124)")
        if type(msg)==str:
            msg=msg.encode()
        elif type(msg)!=bytes:
            raise TypeError("msg argument must be str or bytes")
        self.datalock.acquire()
        if channel not in self.outgoing:
            self.outgoing[channel] = [msg]
        else:
            self.outgoing[channel].append(msg)
        self.log("Message Queued on channel '%s'"%channel, "DEBUG")
        self.outgoing_channels.append(""+channel)
        self.datalock.release()

    def recv(self, channel='__orphan__', decode=False, timeout=None):
        if self._shutdown:
            self.log("Attempt to use the QueuedSocket after shutdown", "WARN")
            raise IOError("This QueuedSocket has already been closed")
        self.datalock.acquire()
        if channel not in self.incoming:
            self.incoming[channel] = []
        if not self._check_channel(channel):
            self.log("Waiting for input on channel '%s'" %channel, "DEBUG")
            result = self.datalock.wait_for(lambda :self._check_channel(channel), timeout)
            if not result:
                self.datalock.release()
                raise sockTimeout()
            self.log("Input dequeued from channel '%s'"%channel, "DETAIL")
        msg = self.incoming[channel].pop(0)
        self.datalock.release()
        if decode:
            msg = msg.decode()
        return msg

    def _sends(self, msg, channel):
        channel = ":ch#"+channel+"|"
        if type(msg)==bytes:
            channel = channel.encode()
        msg = channel + msg
        # print("Sends:", msg)
        self.iolock.acquire()
        super().send(msg)
        self.iolock.release()

    def _recvs(self):
        self.iolock.acquire()
        msg = super().recv()
        self.iolock.release()
        # print("Recvs:", msg)
        if msg[:4] == b':ch#':
            channel = b""
            i = 4
            while msg[i:i+1] != b'|':
                channel += msg[i:i+1]
                i+=1
            return (channel.decode(), msg[i+1:])
        return ("__orphan__", msg)

    def _check_channel(self, channel):
        return len(self.incoming[channel])

    def _worker(self):
        self.log("QueuedSocket background thread initialized")
        while True:
            self.datalock.acquire()
            status = self._shutdown
            outqueue = len(self.outgoing_channels)
            self.datalock.release()
            if status:
                self.log("QueuedSocket background thread halted")
                return
            if outqueue:
                self.datalock.acquire()
                target = self.outgoing_channels.pop(0)
                payload = self.outgoing[target].pop(0)
                self.datalock.release()
                self.log("Outgoing payload on channel '%s'" %target, "DEBUG")
                try:
                    self._sends(payload, target)
                except (OSError, BrokenPipeError, IOError) as e:
                    if self._shutdown:
                        self.log("QueuedSocket background thread halted (attempted to send after shutdown)")
                        return
                    else:
                        self.log("QueuedSocket encountered an error: "+str(e), "ERROR")
                        raise e
            super().settimeout(.05)
            try:
                (channel, payload) = self._recvs()
                self.log("Incoming payload on channel '%s'" %channel, "DEBUG")
                self.datalock.acquire()
                if channel not in self.incoming:
                    self.incoming[channel] = []
                self.incoming[channel].append(payload)
                self.datalock.notify_all()
                self.log("Threads waiting on '%s' have been notified" %channel, "DETAIL")
                self.datalock.release()
            except sockTimeout:
                pass
            except OSError as e:
                if self._shutdown:
                    self.log("QueuedSocket background thread halted")
                    return
                else:
                    self.log("QueuedSocket encountered an error: "+str(e), "ERROR")
                    raise e
<commit_msg>Removed IOError catch, as it would already be caught by OSError<commit_after>from .socket import Socket
from socket import timeout as sockTimeout
from ... import Logger, DummyLog
import threading

_QUEUEDSOCKET_IDENTIFIER_ = '<agutil.io.queuedsocket:1.0.0>'

class QueuedSocket(Socket):
    def __init__(self, socket, logmethod=DummyLog, _skipIdentifier=False, _useIdentifier=_QUEUEDSOCKET_IDENTIFIER_):
        if not isinstance(socket, Socket):
            raise TypeError("socket argument must be of type agutil.io.Socket")
        super().__init__(socket.addr, socket.port, socket.sock, True)
        self.incoming = {'__orphan__': []}
        self.outgoing = {}
        self.outgoing_channels = []
        self._shutdown = False
        self.datalock = threading.Condition()
        self.iolock = threading.Condition()
        self.log = logmethod
        if isinstance(self.log, Logger):
            self.log = self.log.bindToSender("QueuedSocket")
        self.log("The underlying Socket has been initialized.  Starting background thread...")
        self._thread = threading.Thread(
            target=QueuedSocket._worker,
            args=(self,),
            name="QueuedSocket background thread",
            daemon=True
        )
        self._thread.start()
        if not _skipIdentifier:
            QueuedSocket.send(self, _useIdentifier, '__protocol__')
            remoteID = QueuedSocket.recv(self, '__protocol__', True)
            if remoteID != _useIdentifier:
                self.log("The remote socket provided an invalid QueuedSocket protocol identifier. (Theirs: %s) (Ours: %s)" % (
                    remoteID,
                    _useIdentifier
                ), "WARN")
                self.close()
                raise ValueError("The remote socket provided an invalid identifier at the QueuedSocket level")

    def close(self, timeout=1):
        if self._shutdown:
            return
        self.log("Shutdown initiated")
        self.datalock.acquire()
        self._shutdown = True
        self.datalock.release()
        self._thread.join(timeout)
        super().close()
        self.log.close()

    def send(self, msg, channel='__orphan__'):
        if self._shutdown:
            self.log("Attempt to use the QueuedSocket after shutdown", "WARN")
            raise IOError("This QueuedSocket has already been closed")
        if '|' in channel:
            self.log("Attempt to send message over illegal channel name", "WARN")
            raise ValueError("Channel names cannot contain '|' characters (ascii 124)")
        if type(msg)==str:
            msg=msg.encode()
        elif type(msg)!=bytes:
            raise TypeError("msg argument must be str or bytes")
        self.datalock.acquire()
        if channel not in self.outgoing:
            self.outgoing[channel] = [msg]
        else:
            self.outgoing[channel].append(msg)
        self.log("Message Queued on channel '%s'"%channel, "DEBUG")
        self.outgoing_channels.append(""+channel)
        self.datalock.release()

    def recv(self, channel='__orphan__', decode=False, timeout=None):
        if self._shutdown:
            self.log("Attempt to use the QueuedSocket after shutdown", "WARN")
            raise IOError("This QueuedSocket has already been closed")
        self.datalock.acquire()
        if channel not in self.incoming:
            self.incoming[channel] = []
        if not self._check_channel(channel):
            self.log("Waiting for input on channel '%s'" %channel, "DEBUG")
            result = self.datalock.wait_for(lambda :self._check_channel(channel), timeout)
            if not result:
                self.datalock.release()
                raise sockTimeout()
            self.log("Input dequeued from channel '%s'"%channel, "DETAIL")
        msg = self.incoming[channel].pop(0)
        self.datalock.release()
        if decode:
            msg = msg.decode()
        return msg

    def _sends(self, msg, channel):
        channel = ":ch#"+channel+"|"
        if type(msg)==bytes:
            channel = channel.encode()
        msg = channel + msg
        # print("Sends:", msg)
        self.iolock.acquire()
        super().send(msg)
        self.iolock.release()

    def _recvs(self):
        self.iolock.acquire()
        msg = super().recv()
        self.iolock.release()
        # print("Recvs:", msg)
        if msg[:4] == b':ch#':
            channel = b""
            i = 4
            while msg[i:i+1] != b'|':
                channel += msg[i:i+1]
                i+=1
            return (channel.decode(), msg[i+1:])
        return ("__orphan__", msg)

    def _check_channel(self, channel):
        return len(self.incoming[channel])

    def _worker(self):
        self.log("QueuedSocket background thread initialized")
        while True:
            self.datalock.acquire()
            status = self._shutdown
            outqueue = len(self.outgoing_channels)
            self.datalock.release()
            if status:
                self.log("QueuedSocket background thread halted")
                return
            if outqueue:
                self.datalock.acquire()
                target = self.outgoing_channels.pop(0)
                payload = self.outgoing[target].pop(0)
                self.datalock.release()
                self.log("Outgoing payload on channel '%s'" %target, "DEBUG")
                try:
                    self._sends(payload, target)
                except (OSError, BrokenPipeError) as e:
                    if self._shutdown:
                        self.log("QueuedSocket background thread halted (attempted to send after shutdown)")
                        return
                    else:
                        self.log("QueuedSocket encountered an error: "+str(e), "ERROR")
                        raise e
            super().settimeout(.05)
            try:
                (channel, payload) = self._recvs()
                self.log("Incoming payload on channel '%s'" %channel, "DEBUG")
                self.datalock.acquire()
                if channel not in self.incoming:
                    self.incoming[channel] = []
                self.incoming[channel].append(payload)
                self.datalock.notify_all()
                self.log("Threads waiting on '%s' have been notified" %channel, "DETAIL")
                self.datalock.release()
            except sockTimeout:
                pass
            except OSError as e:
                if self._shutdown:
                    self.log("QueuedSocket background thread halted")
                    return
                else:
                    self.log("QueuedSocket encountered an error: "+str(e), "ERROR")
                    raise e
<|endoftext|>


<commit_before># Imports operators dynamically while keeping the package API clean,
# abstracting the underlying modules
from airflow.utils import import_module_attrs as _import_module_attrs

# These need to be integrated first as other operators depend on them
_import_module_attrs(globals(), {
    'check_operator': [
        'CheckOperator',
        'ValueCheckOperator',
        'IntervalCheckOperator',
    ],
})

_operators = {
    'bash_operator': ['BashOperator'],
    'python_operator': ['PythonOperator', 'BranchPythonOperator'],
    'hive_operator': ['HiveOperator'],
    'presto_check_operator': [
        'PrestoCheckOperator',
        'PrestoValueCheckOperator',
        'PrestoIntervalCheckOperator',
    ],
    'dummy_operator': ['DummyOperator'],
    'email_operator': ['EmailOperator'],
    'hive_to_samba_operator': ['Hive2SambaOperator'],
    'mysql_operator': ['MySqlOperator'],
    'sqlite_operator': ['SqliteOperator'],
    'mysql_to_hive': ['MySqlToHiveTransfer'],
    'postgres_operator': ['PostgresOperator'],
    'sensors': [
        'SqlSensor',
        'ExternalTaskSensor',
        'HivePartitionSensor',
        'S3KeySensor',
        'S3PrefixSensor',
        'HdfsSensor',
        'TimeSensor',
        'TimeDeltaSensor',
        'HttpSensor'
    ],
    'subdag_operator': ['SubDagOperator'],
    'hive_stats_operator': ['HiveStatsCollectionOperator'],
    's3_to_hive_operator': ['S3ToHiveTransfer'],
    'hive_to_mysql': ['HiveToMySqlTransfer'],
    's3_file_transform_operator': ['S3FileTransformOperator'],
    'http_operator': ['SimpleHttpOperator'],
    'hive_to_druid': ['HiveToDruidTransfer'],
    'jdbc_operator': ['JdbcOperator'],
    'mssql_operator': ['MsSqlOperator'],
    'mssql_to_hive': ['MsSqlToHiveTransfer'],
    'slack_operator': ['SlackAPIPostOperator', 'SlackAPIOperator'],
    'generic_transfer': ['GenericTransfer'],
}

_import_module_attrs(globals(), _operators)
from airflow.models import BaseOperator

def integrate_plugins():
    """Integrate plugins to the context"""
    from airflow.plugins_manager import operators as _operators
    for _operator in _operators:
        globals()[_operator.__name__] = _operator
<commit_msg>update operators<commit_after># Imports operators dynamically while keeping the package API clean,
# abstracting the underlying modules
from airflow.utils import import_module_attrs as _import_module_attrs

# These need to be integrated first as other operators depend on them
_import_module_attrs(globals(), {
    'check_operator': [
        'CheckOperator',
        'ValueCheckOperator',
        'IntervalCheckOperator',
    ],
})

_operators = {
    'bash_operator': ['BashOperator'],
    'python_operator': ['PythonOperator', 'BranchPythonOperator'],
    'hive_operator': ['HiveOperator'],
    'presto_check_operator': [
        'PrestoCheckOperator',
        'PrestoValueCheckOperator',
        'PrestoIntervalCheckOperator',
    ],
    'dummy_operator': ['DummyOperator'],
    'email_operator': ['EmailOperator'],
    'hive_to_samba_operator': ['Hive2SambaOperator'],
    'mysql_operator': ['MySqlOperator'],
    'sqlite_operator': ['SqliteOperator'],
    'mysql_to_hive': ['MySqlToHiveTransfer'],
    'postgres_operator': ['PostgresOperator'],
    'sensors': [
        'SqlSensor',
        'ExternalTaskSensor',
        'HivePartitionSensor',
        'S3KeySensor',
        'S3PrefixSensor',
        'HdfsSensor',
        'TimeSensor',
        'TimeDeltaSensor',
        'HttpSensor'
    ],
    'subdag_operator': ['SubDagOperator'],
    'hive_stats_operator': ['HiveStatsCollectionOperator'],
    's3_to_hive_operator': ['S3ToHiveTransfer'],
    'hive_to_mysql': ['HiveToMySqlTransfer'],
    's3_file_transform_operator': ['S3FileTransformOperator'],
    'http_operator': ['SimpleHttpOperator'],
    'hive_to_druid': ['HiveToDruidTransfer'],
    'jdbc_operator': ['JdbcOperator'],
    'mssql_operator': ['MsSqlOperator'],
    'mssql_to_hive': ['MsSqlToHiveTransfer'],
    'slack_operator': ['SlackAPIPostOperator', 'SlackAPIOperator'],
    'generic_transfer': ['GenericTransfer'],
    'redshift_operator': ['RedshiftOperator'],
}

_import_module_attrs(globals(), _operators)
from airflow.models import BaseOperator

def integrate_plugins():
    """Integrate plugins to the context"""
    from airflow.plugins_manager import operators as _operators
    for _operator in _operators:
        globals()[_operator.__name__] = _operator
<|endoftext|>


<commit_before>'''
BFS time complexity : O(|E|)
BFS space complexity : O(|V|)

do BFS from (0,0) of the grid and get the minimum number of steps needed to get to the lower right column

only step on the columns whose value is 1

if there is no path, it returns -1

Ex 1)
If grid is
[[1,0,1,1,1,1],
 [1,0,1,0,1,0],
 [1,0,1,0,1,1],
 [1,1,1,0,1,1]],
the answer is: 14

Ex 2)
If grid is
[[1,0,0],
 [0,1,1],
 [0,1,1]],
the answer is: -1
'''

def maze_search(grid):
    dx = [0,0,-1,1]
    dy = [-1,1,0,0]
    n = len(grid)
    m = len(grid[0])
    q = [(0,0,0)]
    visit = [[0]*m for _ in range(n)]
    if grid[0][0] == 0:
        return -1
    visit[0][0] = 1
    while q:
        i, j, step = q.pop(0)
        if i == n-1 and j == m-1:
            return step
        for k in range(4):
            x = i + dx[k]
            y = j + dy[k]
            if x>=0 and x<n and y>=0 and y<m:
                if grid[x][y] ==1 and visit[x][y] == 0:
                    visit[x][y] = 1
                    q.append((x,y,step+1))
    return -1
<commit_msg>Reformat maze_search (#555)<commit_after>from collections import deque

'''
BFS time complexity : O(|E| + |V|)
BFS space complexity : O(|E| + |V|)

do BFS from (0,0) of the grid and get the minimum number of steps needed to get to the lower right column

only step on the columns whose value is 1

if there is no path, it returns -1

Ex 1)
If grid is
[[1,0,1,1,1,1],
 [1,0,1,0,1,0],
 [1,0,1,0,1,1],
 [1,1,1,0,1,1]],
the answer is: 14

Ex 2)
If grid is
[[1,0,0],
 [0,1,1],
 [0,1,1]],
the answer is: -1
'''

def maze_search(maze):
    BLOCKED, ALLOWED = 0, 1
    UNVISITED, VISITED = 0, 1

    initial_x, initial_y = 0, 0

    if maze[initial_x][initial_y] == BLOCKED:
        return -1

    directions = [(0, -1), (0, 1), (-1, 0), (1, 0)]

    height, width = len(maze), len(maze[0])

    target_x, target_y = height - 1, width - 1

    queue = deque([(initial_x, initial_y, 0)])

    is_visited = [[UNVISITED for w in range(width)] for h in range(height)]
    is_visited[initial_x][initial_y] = VISITED

    while queue:
        x, y, steps = queue.popleft()

        if x == target_x and y == target_y:
            return steps

        for dx, dy in directions:
            new_x = x + dx
            new_y = y + dy

            if not (0 <= new_x < height and 0 <= new_y < width):
                continue

            if maze[new_x][new_y] == ALLOWED and is_visited[new_x][new_y] == UNVISITED:
                queue.append((new_x, new_y, steps + 1))
                is_visited[new_x][new_y] = VISITED

    return -1

<|endoftext|>


<commit_before>import requests
import os
from functools import wraps
import inspect
import sys
# Pandas became an optional dependency, but we still want to track it
try:
    import pandas
    _PANDAS_FOUND = True
except ImportError:
    _PANDAS_FOUND = False
import csv

class AlphaVantage(object):
    """ Base class where the decorators and base function for the other
    classes of this python wrapper will inherit from.
    """
    _ALPHA_VANTAGE_API_URL = "http://www.alphavantage.co/query?"
    _ALPHA_VANTAGE_MATH_MAP = ['SMA', 'EMA', 'WMA', 'DEMA', 'TEMA', 'TRIMA',
                               'T3', 'KAMA', 'MAMA']
    _ALPHA_VANTAGE_DIGITAL_CURRENCY_LIST = \
        "https://www.alphavantage.co/digital_currency_list/"

    def __init__(self, key=None, output_format='json',
                 treat_info_as_error=True, indexing_type='date', proxy=None):
        """ Initialize the class

        Keyword Arguments:
            key:  Alpha Vantage api key
            retries:  Maximum amount of retries in case of faulty connection or
                server not able to answer the call.
            treat_info_as_error: Treat information from the api as errors
            output_format:  Either 'json', 'pandas' os 'csv'
            indexing_type: Either 'date' to use the default date string given
            by the alpha vantage api call or 'integer' if you just want an
            integer indexing on your dataframe. Only valid, when the
            output_format is 'pandas'
            proxy: Dictionary mapping protocol or protocol and hostname to
            the URL of the proxy.
        """
        if key is None:
            key = os.getenv('ALPHAVANTAGE_API_KEY')
        if not key or not isinstance(key, str):
            raise ValueError('The AlphaVantage API key must be provided '
                             'either through the key parameter or '
                             'through the environment variable '
                             'ALPHAVANTAGE_API_KEY. Get a free key '
                             'from the alphavantage website: '
                             'https://www.alphavantage.co/support/#api-key')
        self.key = key
        self.output_format = output_format
        if self.output_format is 'pandas' and not _PANDAS_FOUND:
            raise ValueError("The pandas library was not found, therefore can "
                             "not be used as an output format, please install "
                             "manually")
        self.treat_info_as_error = treat_info_as_error
        # Not all the calls accept a data type appended at the end, this
        # variable will be overridden by those functions not needing it.
        self._append_type = True
        self.indexing_type = indexing_type
        self.proxy = proxy or {}

    @classmethod
    def _call_api_on_func(cls, func):
        """ Decorator for forming the api call with the arguments of the
        function, it works by taking the arguments given to the function
        and building the url to call the api on it

        Keyword Arguments:
            func:  The function to be decorated
        """

        # Argument Handling
        if sys.version_info[0] < 3:
            # Deprecated since version 3.0
            argspec = inspect.getargspec(func)
        else:
            argspec = inspect.getfullargspec(func)
        try:
            # Asumme most of the cases have a mixed between args and named
            # args
            positional_count = len(argspec.args) - len(argspec.defaults)
            defaults = dict(
                zip(argspec.args[positional_count:], argspec.defaults))
        except TypeError:
            if argspec.args:
                # No defaults
                positional_count = len(argspec.args)
                defaults = {}
            elif argspec.defaults:
                # Only defaults
                positional_count = 0
                defaults = argspec.defaults
        # Actual decorating

        @wraps(func)
        def _call_wrapper(self, *args, **kwargs):
            used_kwargs = kwargs.copy()
            # Get the used positional arguments given to the function
            used_kwargs.update(zip(argspec.args[positional_count:],
                                   args[positional_count:]))
            # Update the dictionary to include the default parameters from the
            # function
            used_kwargs.update({k: used_kwargs.get(k, d)
                                for k, d in defaults.items()})
            # Form the base url, the original function called must return
            # the function name defined in the alpha vantage api and the data
            # key for it and for its meta data.
            function_name, data_key, meta_data_key = func(
                self, *args, **kwargs)
            url = "{}function={}".format(AlphaVantage._ALPHA_VANTAGE_API_URL,
                                         function_name)
            for idx, arg_name in enumerate(argspec.args[1:]):
                try:
                    arg_value = args[idx]
                except IndexError:
                    arg_value = used_kwargs[arg_name]
                if 'matype' in arg_name and arg_value:
                    # If the argument name has matype, we gotta map the string
                    # or the integer
                    arg_value = self.map_to_matype(arg_value)
                if arg_value:
                    # Discard argument in the url formation if it was set to
                    # None (in other words, this will call the api with its
                    # internal defined parameter)
                    if isinstance(arg_value, tuple) or isinstance(arg_value, list):
                        # If the argument is given as list, then we have to
                        # format it, you gotta format it nicely
                        arg_value = ','.join(arg_value)
                    url = '{}&{}={}'.format(url, arg_name, arg_value)
            # Allow the output format to be json or csv (supported by
            # alphavantage api). Pandas is simply json converted.
            if 'json' in self.output_format.lower() or 'csv' in self.output_format.lower():
                oformat = self.output_format.lower()
            elif 'pandas' in self.output_format.lower():
                oformat = 'json'
            else:
                raise ValueError("Output format: {} not recognized, only json,"
                                 "pandas and csv are supported".format(
                                     self.output_format.lower()))
            if self._append_type:
                url = '{}&apikey={}&datatype={}'.format(url, self.key, oformat)
            else:
                url = '{}&apikey={}'.format(url, self.key)
            return self._handle_api_call(url), data_key, meta_data_key
        return _call_wrapper

    @classmethod
    def _output_format(cls, func, override=None):
        """ Decorator in charge of giving the output its right format, either
        json or pandas

        Keyword Arguments:
            func:  The function to be decorated
            override:  Override the internal format of the call, default None
        """
        @wraps(func)
        def _format_wrapper(self, *args, **kwargs):
            call_response, data_key, meta_data_key = func(
                self, *args, **kwargs)
            if 'json' in self.output_format.lower() or 'pandas' \
                    in self.output_format.lower():
                data = call_response[data_key]
                if meta_data_key is not None:
                    meta_data = call_response[meta_data_key]
                else:
                    meta_data = None
                # Allow to override the output parameter in the call
                if override is None:
                    output_format = self.output_format.lower()
                elif 'json' or 'pandas' in override.lower():
                    output_format = override.lower()
                # Choose output format
                if output_format == 'json':
                    return data, meta_data
                elif output_format == 'pandas':
                    if isinstance(data, list):
                        # If the call returns a list, then we will append them
                        # in the resulting data frame. If in the future
                        # alphavantage decides to do more with returning arrays
                        # this might become buggy. For now will do the trick.
                        data_array = []
                        for val in data:
                            data_array.append([v for _, v in val.items()])
                        data_pandas = pandas.DataFrame(data_array, columns=[
                            k for k, _ in data[0].items()])
                    else:
                        data_pandas = pandas.DataFrame.from_dict(data,
                                                                 orient='index',
                                                                 dtype=float)
                    if 'integer' in self.indexing_type:
                        # Set Date as an actual column so a new numerical index
                        # will be created, but only when specified by the user.
                        data_pandas.reset_index(level=0, inplace=True)
                        data_pandas.index.name = 'index'
                    else:
                        data_pandas.index.name = 'date'
                        # convert to pandas._libs.tslibs.timestamps.Timestamp
                        data_pandas.index = pandas.to_datetime(data_pandas.index)
                    return data_pandas, meta_data
            elif 'csv' in self.output_format.lower():
                return call_response, None
            else:
                raise ValueError('Format: {} is not supported'.format(
                    self.output_format))
        return _format_wrapper

    def set_proxy(self, proxy=None):
        """ Set a new proxy configuration

        Keyword Arguments:
            proxy: Dictionary mapping protocol or protocol and hostname to
            the URL of the proxy.
        """
        self.proxy = proxy or {}

    def map_to_matype(self, matype):
        """ Convert to the alpha vantage math type integer. It returns an
        integer correspondent to the type of math to apply to a function. It
        raises ValueError if an integer greater than the supported math types
        is given.

        Keyword Arguments:
            matype:  The math type of the alpha vantage api. It accepts
            integers or a string representing the math type.

                * 0 = Simple Moving Average (SMA),
                * 1 = Exponential Moving Average (EMA),
                * 2 = Weighted Moving Average (WMA),
                * 3 = Double Exponential Moving Average (DEMA),
                * 4 = Triple Exponential Moving Average (TEMA),
                * 5 = Triangular Moving Average (TRIMA),
                * 6 = T3 Moving Average,
                * 7 = Kaufman Adaptive Moving Average (KAMA),
                * 8 = MESA Adaptive Moving Average (MAMA)
        """
        # Check if it is an integer or a string
        try:
            value = int(matype)
            if abs(value) > len(AlphaVantage._ALPHA_VANTAGE_MATH_MAP):
                raise ValueError("The value {} is not supported".format(value))
        except ValueError:
            value = AlphaVantage._ALPHA_VANTAGE_MATH_MAP.index(matype)
        return value

    def _handle_api_call(self, url):
        """ Handle the return call from the  api and return a data and meta_data
        object. It raises a ValueError on problems

        Keyword Arguments:
            url:  The url of the service
            data_key:  The key for getting the data from the jso object
            meta_data_key:  The key for getting the meta data information out
            of the json object
        """
        response = requests.get(url, proxies=self.proxy)
        if 'json' in self.output_format.lower() or 'pandas' in \
                self.output_format.lower():
            json_response = response.json()
            if "Error Message" in json_response:
                raise ValueError(json_response["Error Message"])
            elif "Information" in json_response and self.treat_info_as_error:
                raise ValueError(json_response["Information"])
            elif "Note" in json_response and self.treat_info_as_error:
                raise ValueError(json_response["Note"])
            return json_response
        else:
            csv_response = csv.reader(response.text.splitlines())
            if not csv_response:
                raise ValueError(
                    'Error getting data from the api, no return was given.')
            return csv_response
<commit_msg>fixing dtype float error on global quote handling<commit_after>import requests
import os
from functools import wraps
import inspect
import sys
# Pandas became an optional dependency, but we still want to track it
try:
    import pandas
    _PANDAS_FOUND = True
except ImportError:
    _PANDAS_FOUND = False
import csv

class AlphaVantage(object):
    """ Base class where the decorators and base function for the other
    classes of this python wrapper will inherit from.
    """
    _ALPHA_VANTAGE_API_URL = "http://www.alphavantage.co/query?"
    _ALPHA_VANTAGE_MATH_MAP = ['SMA', 'EMA', 'WMA', 'DEMA', 'TEMA', 'TRIMA',
                               'T3', 'KAMA', 'MAMA']
    _ALPHA_VANTAGE_DIGITAL_CURRENCY_LIST = \
        "https://www.alphavantage.co/digital_currency_list/"

    def __init__(self, key=None, output_format='json',
                 treat_info_as_error=True, indexing_type='date', proxy=None):
        """ Initialize the class

        Keyword Arguments:
            key:  Alpha Vantage api key
            retries:  Maximum amount of retries in case of faulty connection or
                server not able to answer the call.
            treat_info_as_error: Treat information from the api as errors
            output_format:  Either 'json', 'pandas' os 'csv'
            indexing_type: Either 'date' to use the default date string given
            by the alpha vantage api call or 'integer' if you just want an
            integer indexing on your dataframe. Only valid, when the
            output_format is 'pandas'
            proxy: Dictionary mapping protocol or protocol and hostname to
            the URL of the proxy.
        """
        if key is None:
            key = os.getenv('ALPHAVANTAGE_API_KEY')
        if not key or not isinstance(key, str):
            raise ValueError('The AlphaVantage API key must be provided '
                             'either through the key parameter or '
                             'through the environment variable '
                             'ALPHAVANTAGE_API_KEY. Get a free key '
                             'from the alphavantage website: '
                             'https://www.alphavantage.co/support/#api-key')
        self.key = key
        self.output_format = output_format
        if self.output_format is 'pandas' and not _PANDAS_FOUND:
            raise ValueError("The pandas library was not found, therefore can "
                             "not be used as an output format, please install "
                             "manually")
        self.treat_info_as_error = treat_info_as_error
        # Not all the calls accept a data type appended at the end, this
        # variable will be overridden by those functions not needing it.
        self._append_type = True
        self.indexing_type = indexing_type
        self.proxy = proxy or {}

    @classmethod
    def _call_api_on_func(cls, func):
        """ Decorator for forming the api call with the arguments of the
        function, it works by taking the arguments given to the function
        and building the url to call the api on it

        Keyword Arguments:
            func:  The function to be decorated
        """

        # Argument Handling
        if sys.version_info[0] < 3:
            # Deprecated since version 3.0
            argspec = inspect.getargspec(func)
        else:
            argspec = inspect.getfullargspec(func)
        try:
            # Asumme most of the cases have a mixed between args and named
            # args
            positional_count = len(argspec.args) - len(argspec.defaults)
            defaults = dict(
                zip(argspec.args[positional_count:], argspec.defaults))
        except TypeError:
            if argspec.args:
                # No defaults
                positional_count = len(argspec.args)
                defaults = {}
            elif argspec.defaults:
                # Only defaults
                positional_count = 0
                defaults = argspec.defaults
        # Actual decorating

        @wraps(func)
        def _call_wrapper(self, *args, **kwargs):
            used_kwargs = kwargs.copy()
            # Get the used positional arguments given to the function
            used_kwargs.update(zip(argspec.args[positional_count:],
                                   args[positional_count:]))
            # Update the dictionary to include the default parameters from the
            # function
            used_kwargs.update({k: used_kwargs.get(k, d)
                                for k, d in defaults.items()})
            # Form the base url, the original function called must return
            # the function name defined in the alpha vantage api and the data
            # key for it and for its meta data.
            function_name, data_key, meta_data_key = func(
                self, *args, **kwargs)
            url = "{}function={}".format(AlphaVantage._ALPHA_VANTAGE_API_URL,
                                         function_name)
            for idx, arg_name in enumerate(argspec.args[1:]):
                try:
                    arg_value = args[idx]
                except IndexError:
                    arg_value = used_kwargs[arg_name]
                if 'matype' in arg_name and arg_value:
                    # If the argument name has matype, we gotta map the string
                    # or the integer
                    arg_value = self.map_to_matype(arg_value)
                if arg_value:
                    # Discard argument in the url formation if it was set to
                    # None (in other words, this will call the api with its
                    # internal defined parameter)
                    if isinstance(arg_value, tuple) or isinstance(arg_value, list):
                        # If the argument is given as list, then we have to
                        # format it, you gotta format it nicely
                        arg_value = ','.join(arg_value)
                    url = '{}&{}={}'.format(url, arg_name, arg_value)
            # Allow the output format to be json or csv (supported by
            # alphavantage api). Pandas is simply json converted.
            if 'json' in self.output_format.lower() or 'csv' in self.output_format.lower():
                oformat = self.output_format.lower()
            elif 'pandas' in self.output_format.lower():
                oformat = 'json'
            else:
                raise ValueError("Output format: {} not recognized, only json,"
                                 "pandas and csv are supported".format(
                                     self.output_format.lower()))
            if self._append_type:
                url = '{}&apikey={}&datatype={}'.format(url, self.key, oformat)
            else:
                url = '{}&apikey={}'.format(url, self.key)
            return self._handle_api_call(url), data_key, meta_data_key
        return _call_wrapper

    @classmethod
    def _output_format(cls, func, override=None):
        """ Decorator in charge of giving the output its right format, either
        json or pandas

        Keyword Arguments:
            func:  The function to be decorated
            override:  Override the internal format of the call, default None
        """
        @wraps(func)
        def _format_wrapper(self, *args, **kwargs):
            call_response, data_key, meta_data_key = func(
                self, *args, **kwargs)
            if 'json' in self.output_format.lower() or 'pandas' \
                    in self.output_format.lower():
                data = call_response[data_key]
                if meta_data_key is not None:
                    meta_data = call_response[meta_data_key]
                else:
                    meta_data = None
                # Allow to override the output parameter in the call
                if override is None:
                    output_format = self.output_format.lower()
                elif 'json' or 'pandas' in override.lower():
                    output_format = override.lower()
                # Choose output format
                if output_format == 'json':
                    return data, meta_data
                elif output_format == 'pandas':
                    if isinstance(data, list):
                        # If the call returns a list, then we will append them
                        # in the resulting data frame. If in the future
                        # alphavantage decides to do more with returning arrays
                        # this might become buggy. For now will do the trick.
                        data_array = []
                        for val in data:
                            data_array.append([v for _, v in val.items()])
                        data_pandas = pandas.DataFrame(data_array, columns=[
                            k for k, _ in data[0].items()])
                    else:
                        try:
                            data_pandas = pandas.DataFrame.from_dict(data,
                                                                     orient='index',
                                                                     dtype='float')
                        # This is for Global quotes or any other new Alpha Vantage
                        # data that is added.
                        # It will have to be updated so that we can get exactly
                        # The dataframes we want moving forward
                        except ValueError:
                            data = {data_key: data}
                            data_pandas = pandas.DataFrame.from_dict(data,
                                                                     orient='index',
                                                                     dtype='object')
                            return data_pandas, meta_data

                    if 'integer' in self.indexing_type:
                        # Set Date as an actual column so a new numerical index
                        # will be created, but only when specified by the user.
                        data_pandas.reset_index(level=0, inplace=True)
                        data_pandas.index.name = 'index'
                    else:
                        data_pandas.index.name = 'date'
                        # convert to pandas._libs.tslibs.timestamps.Timestamp
                        data_pandas.index = pandas.to_datetime(
                            data_pandas.index)
                    return data_pandas, meta_data
            elif 'csv' in self.output_format.lower():
                return call_response, None
            else:
                raise ValueError('Format: {} is not supported'.format(
                    self.output_format))
        return _format_wrapper

    def set_proxy(self, proxy=None):
        """ Set a new proxy configuration

        Keyword Arguments:
            proxy: Dictionary mapping protocol or protocol and hostname to
            the URL of the proxy.
        """
        self.proxy = proxy or {}

    def map_to_matype(self, matype):
        """ Convert to the alpha vantage math type integer. It returns an
        integer correspondent to the type of math to apply to a function. It
        raises ValueError if an integer greater than the supported math types
        is given.

        Keyword Arguments:
            matype:  The math type of the alpha vantage api. It accepts
            integers or a string representing the math type.

                * 0 = Simple Moving Average (SMA),
                * 1 = Exponential Moving Average (EMA),
                * 2 = Weighted Moving Average (WMA),
                * 3 = Double Exponential Moving Average (DEMA),
                * 4 = Triple Exponential Moving Average (TEMA),
                * 5 = Triangular Moving Average (TRIMA),
                * 6 = T3 Moving Average,
                * 7 = Kaufman Adaptive Moving Average (KAMA),
                * 8 = MESA Adaptive Moving Average (MAMA)
        """
        # Check if it is an integer or a string
        try:
            value = int(matype)
            if abs(value) > len(AlphaVantage._ALPHA_VANTAGE_MATH_MAP):
                raise ValueError("The value {} is not supported".format(value))
        except ValueError:
            value = AlphaVantage._ALPHA_VANTAGE_MATH_MAP.index(matype)
        return value

    def _handle_api_call(self, url):
        """ Handle the return call from the  api and return a data and meta_data
        object. It raises a ValueError on problems

        Keyword Arguments:
            url:  The url of the service
            data_key:  The key for getting the data from the jso object
            meta_data_key:  The key for getting the meta data information out
            of the json object
        """
        response = requests.get(url, proxies=self.proxy)
        if 'json' in self.output_format.lower() or 'pandas' in \
                self.output_format.lower():
            json_response = response.json()
            if "Error Message" in json_response:
                raise ValueError(json_response["Error Message"])
            elif "Information" in json_response and self.treat_info_as_error:
                raise ValueError(json_response["Information"])
            elif "Note" in json_response and self.treat_info_as_error:
                raise ValueError(json_response["Note"])
            return json_response
        else:
            csv_response = csv.reader(response.text.splitlines())
            if not csv_response:
                raise ValueError(
                    'Error getting data from the api, no return was given.')
            return csv_response
<|endoftext|>


<commit_before>import os

from ...utils.mimebundle import spec_to_mimebundle
from ..display import Displayable
from ..display import default_renderer_base
from ..display import json_renderer_base
from ..display import RendererRegistry
from ..display import HTMLRenderer

from .schema import SCHEMA_VERSION
VEGALITE_VERSION = SCHEMA_VERSION.lstrip('v')
VEGA_VERSION = '5'
VEGAEMBED_VERSION = '4'

# ==============================================================================
# VegaLite v3 renderer logic
# ==============================================================================

# The MIME type for Vega-Lite 3.x releases.
VEGALITE_MIME_TYPE = 'application/vnd.vegalite.v3+json'  # type: str

# The entry point group that can be used by other packages to declare other
# renderers that will be auto-detected. Explicit registration is also
# allowed by the PluginRegistery API.
ENTRY_POINT_GROUP = 'altair.vegalite.v3.renderer'  # type: str

# The display message when rendering fails
DEFAULT_DISPLAY = """\
<VegaLite 3 object>

If you see this message, it means the renderer has not been properly enabled
for the frontend that you are using. For more information, see
https://altair-viz.github.io/user_guide/troubleshooting.html
"""

renderers = RendererRegistry(entry_point_group=ENTRY_POINT_GROUP)

here = os.path.dirname(os.path.realpath(__file__))

def default_renderer(spec, **metadata):
    return default_renderer_base(spec, VEGALITE_MIME_TYPE, DEFAULT_DISPLAY,
                                 **metadata)

def json_renderer(spec, **metadata):
    return json_renderer_base(spec, DEFAULT_DISPLAY, **metadata)

def png_renderer(spec, **metadata):
    return spec_to_mimebundle(spec, format='png',
                              mode='vega-lite',
                              vega_version=VEGA_VERSION,
                              vegaembed_version=VEGAEMBED_VERSION,
                              vegalite_version=VEGALITE_VERSION,
                              **metadata)

def svg_renderer(spec, **metadata):
    return spec_to_mimebundle(spec, format='svg',
                              mode='vega-lite',
                              vega_version=VEGA_VERSION,
                              vegaembed_version=VEGAEMBED_VERSION,
                              vegalite_version=VEGALITE_VERSION,
                              **metadata)

colab_renderer = HTMLRenderer(mode='vega-lite',
                              fullhtml=True, requirejs=False,
                              output_div='altair-viz',
                              vega_version=VEGA_VERSION,
                              vegaembed_version=VEGAEMBED_VERSION,
                              vegalite_version=VEGALITE_VERSION)

zeppelin_renderer = HTMLRenderer(mode='vega-lite',
                              fullhtml=True, requirejs=False,
                              vega_version=VEGA_VERSION,
                              vegaembed_version=VEGAEMBED_VERSION,
                              vegalite_version=VEGALITE_VERSION)

kaggle_renderer = HTMLRenderer(mode='vega-lite',
                               fullhtml=False, requirejs=True,
                               vega_version=VEGA_VERSION,
                               vegaembed_version=VEGAEMBED_VERSION,
                               vegalite_version=VEGALITE_VERSION)

renderers.register('default', default_renderer)
renderers.register('jupyterlab', default_renderer)
renderers.register('nteract', default_renderer)
renderers.register('colab', colab_renderer)
renderers.register('kaggle', kaggle_renderer)
renderers.register('json', json_renderer)
renderers.register('png', png_renderer)
renderers.register('svg', svg_renderer)
renderers.register('zeppelin', zeppelin_renderer)
renderers.enable('default')

class VegaLite(Displayable):
    """An IPython/Jupyter display class for rendering VegaLite 3."""

    renderers = renderers
    schema_path = (__name__, 'schema/vega-lite-schema.json')

def vegalite(spec, validate=True):
    """Render and optionally validate a VegaLite 3 spec.

    This will use the currently enabled renderer to render the spec.

    Parameters
    ==========
    spec: dict
        A fully compliant VegaLite 3 spec, with the data portion fully processed.
    validate: bool
        Should the spec be validated against the VegaLite 3 schema?
    """
    from IPython.display import display

    display(VegaLite(spec, validate=validate))
<commit_msg>address code style<commit_after>import os

from ...utils.mimebundle import spec_to_mimebundle
from ..display import Displayable
from ..display import default_renderer_base
from ..display import json_renderer_base
from ..display import RendererRegistry
from ..display import HTMLRenderer

from .schema import SCHEMA_VERSION
VEGALITE_VERSION = SCHEMA_VERSION.lstrip('v')
VEGA_VERSION = '5'
VEGAEMBED_VERSION = '4'

# ==============================================================================
# VegaLite v3 renderer logic
# ==============================================================================

# The MIME type for Vega-Lite 3.x releases.
VEGALITE_MIME_TYPE = 'application/vnd.vegalite.v3+json'  # type: str

# The entry point group that can be used by other packages to declare other
# renderers that will be auto-detected. Explicit registration is also
# allowed by the PluginRegistery API.
ENTRY_POINT_GROUP = 'altair.vegalite.v3.renderer'  # type: str

# The display message when rendering fails
DEFAULT_DISPLAY = """\
<VegaLite 3 object>

If you see this message, it means the renderer has not been properly enabled
for the frontend that you are using. For more information, see
https://altair-viz.github.io/user_guide/troubleshooting.html
"""

renderers = RendererRegistry(entry_point_group=ENTRY_POINT_GROUP)

here = os.path.dirname(os.path.realpath(__file__))

def default_renderer(spec, **metadata):
    return default_renderer_base(spec, VEGALITE_MIME_TYPE, DEFAULT_DISPLAY,
                                 **metadata)

def json_renderer(spec, **metadata):
    return json_renderer_base(spec, DEFAULT_DISPLAY, **metadata)

def png_renderer(spec, **metadata):
    return spec_to_mimebundle(spec, format='png',
                              mode='vega-lite',
                              vega_version=VEGA_VERSION,
                              vegaembed_version=VEGAEMBED_VERSION,
                              vegalite_version=VEGALITE_VERSION,
                              **metadata)

def svg_renderer(spec, **metadata):
    return spec_to_mimebundle(spec, format='svg',
                              mode='vega-lite',
                              vega_version=VEGA_VERSION,
                              vegaembed_version=VEGAEMBED_VERSION,
                              vegalite_version=VEGALITE_VERSION,
                              **metadata)

colab_renderer = HTMLRenderer(mode='vega-lite',
                              fullhtml=True, requirejs=False,
                              output_div='altair-viz',
                              vega_version=VEGA_VERSION,
                              vegaembed_version=VEGAEMBED_VERSION,
                              vegalite_version=VEGALITE_VERSION)

zeppelin_renderer = HTMLRenderer(mode='vega-lite',
                                fullhtml=True, requirejs=False,
                                vega_version=VEGA_VERSION,
                                vegaembed_version=VEGAEMBED_VERSION,
                                vegalite_version=VEGALITE_VERSION)

kaggle_renderer = HTMLRenderer(mode='vega-lite',
                               fullhtml=False, requirejs=True,
                               vega_version=VEGA_VERSION,
                               vegaembed_version=VEGAEMBED_VERSION,
                               vegalite_version=VEGALITE_VERSION)

renderers.register('default', default_renderer)
renderers.register('jupyterlab', default_renderer)
renderers.register('nteract', default_renderer)
renderers.register('colab', colab_renderer)
renderers.register('kaggle', kaggle_renderer)
renderers.register('json', json_renderer)
renderers.register('png', png_renderer)
renderers.register('svg', svg_renderer)
renderers.register('zeppelin', zeppelin_renderer)
renderers.enable('default')

class VegaLite(Displayable):
    """An IPython/Jupyter display class for rendering VegaLite 3."""

    renderers = renderers
    schema_path = (__name__, 'schema/vega-lite-schema.json')

def vegalite(spec, validate=True):
    """Render and optionally validate a VegaLite 3 spec.

    This will use the currently enabled renderer to render the spec.

    Parameters
    ==========
    spec: dict
        A fully compliant VegaLite 3 spec, with the data portion fully processed.
    validate: bool
        Should the spec be validated against the VegaLite 3 schema?
    """
    from IPython.display import display

    display(VegaLite(spec, validate=validate))
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

"""
Test transparent checkouts from SCCS files in an SCCS subdirectory.
"""

import os.path
import string

import TestSCons

test = TestSCons.TestSCons()

sccs = test.where_is('sccs')
if not sccs:
    test.skip_test("Could not find 'sccs'; skipping test(s).\n")

test.subdir('SCCS', 'sub', ['sub', 'SCCS'])

for f in ['aaa.in', 'bbb.in', 'ccc.in']:
    test.write(f, "%%F%% %s\n" % f)
    args = "create %s" % f
    test.run(program = sccs, arguments = args, stderr = None)
    test.unlink(f)
    test.unlink(','+f)

test.write(['sub', 'SConscript'], """\
# %%F%%
Import("env")
env.Cat('ddd.out', 'ddd.in')
env.Cat('eee.out', 'eee.in')
env.Cat('fff.out', 'fff.in')
env.Cat('all', ['ddd.out', 'eee.out', 'fff.out'])
""")
args = "create SConscript"
test.run(chdir = 'sub', program = sccs, arguments = args, stderr = None)
test.unlink(['sub', 'SConscript'])
test.unlink(['sub', ',SConscript'])

for f in ['ddd.in', 'eee.in', 'fff.in']:
    test.write(['sub', f], "%%F%% sub/%s\n" % f)
    args = "create %s" % f
    test.run(chdir = 'sub', program = sccs, arguments = args, stderr = None)
    test.unlink(['sub', f])
    test.unlink(['sub', ','+f])

test.write(['SConstruct'], """
DefaultEnvironment()['SCCSCOM'] = 'cd ${TARGET.dir} && $SCCS get ${TARGET.file}'
def cat(env, source, target):
    target = str(target[0])
    source = map(str, source)
    f = open(target, "wb")
    for src in source:
        f.write(open(src, "rb").read())
    f.close()
SetOption('diskcheck', ['match', 'rcs'])
env = Environment(BUILDERS={'Cat':Builder(action=cat)},
                  SCCSFLAGS='-k')
env.Cat('aaa.out', 'aaa.in')
env.Cat('bbb.out', 'bbb.in')
env.Cat('ccc.out', 'ccc.in')
env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out'])
SConscript('sub/SConscript', "env")
""")

test.write(['bbb.in'], "checked-out bbb.in\n")

test.write(['sub', 'eee.in'], "checked-out sub/eee.in\n")

sub_SConscript = os.path.join('sub', 'SConscript')
SConstruct_file_line = test.python_file_line(test.workpath('SConstruct'), 17)[:-1]

expect = """\

scons: warning: Ignoring missing SConscript '%(sub_SConscript)s'
%(SConstruct_file_line)s
scons: *** Source `aaa.in' not found, needed by target `aaa.out'.  Stop.
""" % locals()

test.run(status=2, stderr=expect)

test.run(arguments = '--diskcheck=sccs', stderr = None)

lines = string.split("""
sccs get SConscript
sccs get aaa.in
cat(["aaa.out"], ["aaa.in"])
cat(["bbb.out"], ["bbb.in"])
sccs get ccc.in
cat(["ccc.out"], ["ccc.in"])
cat(["all"], ["aaa.out", "bbb.out", "ccc.out"])
sccs get ddd.in
cat(["sub/ddd.out"], ["sub/ddd.in"])
cat(["sub/eee.out"], ["sub/eee.in"])
sccs get fff.in
cat(["sub/fff.out"], ["sub/fff.in"])
cat(["sub/all"], ["sub/ddd.out", "sub/eee.out", "sub/fff.out"])
""", '\n')

stdout = test.stdout()
missing = filter(lambda l, s=stdout: string.find(s, l) == -1, lines)
if missing:
    print "Missing the following output lines:"
    print string.join(missing, '\n')
    print "Actual STDOUT =========="
    print stdout
    test.fail_test(1)

test.must_match('all', """\
s.aaa.in aaa.in
checked-out bbb.in
s.ccc.in ccc.in
""")

test.must_not_be_writable(test.workpath('sub', 'SConscript'))
test.must_not_be_writable(test.workpath('aaa.in'))
test.must_not_be_writable(test.workpath('ccc.in'))
test.must_not_be_writable(test.workpath('sub', 'ddd.in'))
test.must_not_be_writable(test.workpath('sub', 'fff.in'))

test.pass_test()
<commit_msg>Update test/SCCS/diskcheck.py for recent GetBuildFailures() robustness change, which changed the exit message in this case.<commit_after>#!/usr/bin/env python
#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

"""
Test transparent checkouts from SCCS files in an SCCS subdirectory.
"""

import os.path
import string

import TestSCons

test = TestSCons.TestSCons()

sccs = test.where_is('sccs')
if not sccs:
    test.skip_test("Could not find 'sccs'; skipping test(s).\n")

test.subdir('SCCS', 'sub', ['sub', 'SCCS'])

for f in ['aaa.in', 'bbb.in', 'ccc.in']:
    test.write(f, "%%F%% %s\n" % f)
    args = "create %s" % f
    test.run(program = sccs, arguments = args, stderr = None)
    test.unlink(f)
    test.unlink(','+f)

test.write(['sub', 'SConscript'], """\
# %%F%%
Import("env")
env.Cat('ddd.out', 'ddd.in')
env.Cat('eee.out', 'eee.in')
env.Cat('fff.out', 'fff.in')
env.Cat('all', ['ddd.out', 'eee.out', 'fff.out'])
""")
args = "create SConscript"
test.run(chdir = 'sub', program = sccs, arguments = args, stderr = None)
test.unlink(['sub', 'SConscript'])
test.unlink(['sub', ',SConscript'])

for f in ['ddd.in', 'eee.in', 'fff.in']:
    test.write(['sub', f], "%%F%% sub/%s\n" % f)
    args = "create %s" % f
    test.run(chdir = 'sub', program = sccs, arguments = args, stderr = None)
    test.unlink(['sub', f])
    test.unlink(['sub', ','+f])

test.write(['SConstruct'], """
DefaultEnvironment()['SCCSCOM'] = 'cd ${TARGET.dir} && $SCCS get ${TARGET.file}'
def cat(env, source, target):
    target = str(target[0])
    source = map(str, source)
    f = open(target, "wb")
    for src in source:
        f.write(open(src, "rb").read())
    f.close()
SetOption('diskcheck', ['match', 'rcs'])
env = Environment(BUILDERS={'Cat':Builder(action=cat)},
                  SCCSFLAGS='-k')
env.Cat('aaa.out', 'aaa.in')
env.Cat('bbb.out', 'bbb.in')
env.Cat('ccc.out', 'ccc.in')
env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out'])
SConscript('sub/SConscript', "env")
""")

test.write(['bbb.in'], "checked-out bbb.in\n")

test.write(['sub', 'eee.in'], "checked-out sub/eee.in\n")

sub_SConscript = os.path.join('sub', 'SConscript')
SConstruct_file_line = test.python_file_line(test.workpath('SConstruct'), 17)[:-1]

expect = """\

scons: warning: Ignoring missing SConscript '%(sub_SConscript)s'
%(SConstruct_file_line)s
scons: *** [aaa.out] Source `aaa.in' not found, needed by target `aaa.out'.
""" % locals()

test.run(status=2, stderr=expect)

test.run(arguments = '--diskcheck=sccs', stderr = None)

lines = string.split("""
sccs get SConscript
sccs get aaa.in
cat(["aaa.out"], ["aaa.in"])
cat(["bbb.out"], ["bbb.in"])
sccs get ccc.in
cat(["ccc.out"], ["ccc.in"])
cat(["all"], ["aaa.out", "bbb.out", "ccc.out"])
sccs get ddd.in
cat(["sub/ddd.out"], ["sub/ddd.in"])
cat(["sub/eee.out"], ["sub/eee.in"])
sccs get fff.in
cat(["sub/fff.out"], ["sub/fff.in"])
cat(["sub/all"], ["sub/ddd.out", "sub/eee.out", "sub/fff.out"])
""", '\n')

stdout = test.stdout()
missing = filter(lambda l, s=stdout: string.find(s, l) == -1, lines)
if missing:
    print "Missing the following output lines:"
    print string.join(missing, '\n')
    print "Actual STDOUT =========="
    print stdout
    test.fail_test(1)

test.must_match('all', """\
s.aaa.in aaa.in
checked-out bbb.in
s.ccc.in ccc.in
""")

test.must_not_be_writable(test.workpath('sub', 'SConscript'))
test.must_not_be_writable(test.workpath('aaa.in'))
test.must_not_be_writable(test.workpath('ccc.in'))
test.must_not_be_writable(test.workpath('sub', 'ddd.in'))
test.must_not_be_writable(test.workpath('sub', 'fff.in'))

test.pass_test()
<|endoftext|>


<commit_before>from BaseTasksProvider import BaseTasksProvider
from datetime import datetime, timedelta
import threading
from settingscron import MAX_NUMBER_OF_ACTIVE_DB_SESSIONS, TIMEOUT_WHEN_DB_LOADED
import logging
import pandas as pd
import numpy as np
from django.core.cache import cache
from django.utils.six.moves import cPickle as pickle
import core.libs.CampaignPredictionHelper as cph
import humanize

class SQLAggregatorCampaign(BaseTasksProvider):

    lock = threading.RLock()
    logger = logging.getLogger(__name__ + ' SQLAggregatorCampaign')

    def __dictfetchall(self, cursor):
        "Returns all rows from a cursor as a dict"
        desc = cursor.description
        return [
            dict(zip([col[0] for col in desc], row))
            for row in cursor.fetchall()
        ]

    def processPayload(self):

        # Here we do preprocessing for the prediction module restricted to the data18_13TeV reprocessing campaign

        self.logger.info("SQLAggregatorCampaign started")
        campaign = {
            'campaign': 'data18_13TeV'
        }

        while (self.getNumberOfActiveDBSessions() > MAX_NUMBER_OF_ACTIVE_DB_SESSIONS):
            threading.sleep(TIMEOUT_WHEN_DB_LOADED)
        connection = self.pool.acquire()
        cursor = connection.cursor()
        cursor.execute("alter session set NLS_DATE_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")
        cursor.execute("alter session set NLS_TIMESTAMP_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")

        #Columns names are specific due to DEFT compartibility
        query = """
            select STARTTIME as START_TIME, ENDTIME, STEP_NAME, round((NEVENTSUSED) / (count(*) over (partition by tr.jeditaskid))) as DONEEV  from (
            (SELECT DISTINCT STARTTIME, endtime, pandaid, jeditaskid FROM ATLAS_PANDAARCH.JOBSARCHIVED WHERE JOBSTATUS='finished' UNION ALL
            SELECT DISTINCT STARTTIME, endtime, pandaid, jeditaskid FROM ATLAS_PANDA.JOBSARCHIVED4 WHERE JOBSTATUS='finished')tr
             join (
                select t1.JEDITASKID, t5.STEP_NAME, t1.CREATIONDATE, t1.STATUS, t2.CAMPAIGN,t2.SUBCAMPAIGN, t1.neventstobeused, t1.nevents, t1.NEVENTSUSED from ATLAS_PANDABIGMON.RUNNINGPRODTASKS t1
                join ATLAS_DEFT.T_PRODUCTION_TASK t2 on t2.TASKID=t1.JEDITASKID
                JOIN ATLAS_DEFT.T_PRODUCTION_STEP t4 ON t2.STEP_ID=t4.STEP_ID JOIN ATLAS_DEFT.T_STEP_TEMPLATE t5 ON t4.STEP_T_ID=t5.STEP_T_ID
                where t2.campaign='data18_13TeV' and t1.processingtype='reprocessing'
            )td on tr.jeditaskid=td.jeditaskid) order by endtime ASC
        """

        campaign_df = pd.read_sql(query, con=connection)
        index = pd.date_range(start=campaign_df.START_TIME.min(), end=campaign_df.ENDTIME.max())
        stepsname = campaign_df.STEP_NAME.unique().tolist()
        new_df = pd.DataFrame(index=index, columns=stepsname)

        new_df_Reco_ev = new_df.apply(lambda x: campaign_df[
            (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                    campaign_df.STEP_NAME == 'Reco')]['DONEEV'].sum(), axis=1)

        reindexedbyDay_ev = pd.concat(
            [new_df_Reco_ev], axis=1, keys=['Reco'])

        # At this step we have number of events processed per each day of a campaign

        query = "select sum(neventstobeused) as sumtouse, sum(nevents) as sumtot, sum(NEVENTSUSED) as sumused, sum(NRUNNINGEVENTS) as sumrun from ATLAS_PANDABIGMON.RUNNINGPRODTASKS where campaign='data18_13TeV' and processingtype='reprocessing'"
        cursor.execute(query)
        eventstotals = self.__dictfetchall(cursor)
        if eventstotals and len(eventstotals) > 0:
            eventstotals = eventstotals[0]

        numberOfRemainingEventsPerStep = {
            'Reco': eventstotals['SUMTOUSE']
        }
        numberOfRunningEventsPerStep = {
            'Reco': eventstotals['SUMRUN']
        }
        numberOfTotalEventsPerStep = {
            'Reco': eventstotals['SUMTOT']
        }

        progressOfEventsPerStep = {
            'Reco': round(eventstotals['SUMUSED']/eventstotals['SUMTOT'],1)
        }
        numberOfDoneEventsPerStep = {
            'Reco': eventstotals['SUMUSED']
        }

        remainingForSubmitting = {}
        remainingForMaxPossible = {}
        progressForSubmitted = {}
        progressForMax = {}
        stepWithMaxEvents = 'Reco'
        maxEvents = numberOfTotalEventsPerStep[stepWithMaxEvents]
        eventsPerDay = {}

        rollingRes = reindexedbyDay_ev['Reco'].rolling(5, win_type='triang').mean()
        if len(rollingRes) > 2 and rollingRes[-1] > 0 and 'Reco' in numberOfRemainingEventsPerStep:
            remainingForSubmitting['Reco'] = str(round(numberOfRemainingEventsPerStep['Reco'] / \
                                                     rollingRes[-1], 2)) + " d"
            remainingForMaxPossible['Reco'] = str(round((maxEvents - numberOfDoneEventsPerStep['Reco']) / \
                                                      rollingRes[-1], 2)) + " d"
            progressForSubmitted['Reco'] = round(numberOfDoneEventsPerStep['Reco'] * 100.0 / (
                        numberOfDoneEventsPerStep['Reco'] + numberOfRemainingEventsPerStep['Reco']), 1)
            progressForMax['Reco'] = int(numberOfDoneEventsPerStep['Reco'] * 100.0 / maxEvents)
            eventsPerDay['Reco'] = humanize.intcomma(int(rollingRes[-1]))

        data = {
                "numberOfRemainingEventsPerStep":numberOfRemainingEventsPerStep,
                "numberOfRunningEventsPerStep":numberOfRunningEventsPerStep,
                "numberOfTotalEventsPerStep": numberOfTotalEventsPerStep,
                "progressOfEventsPerStep":progressOfEventsPerStep,
                "numberOfDoneEventsPerStep":numberOfDoneEventsPerStep,
                "remainingForSubmitting":remainingForSubmitting,
                "remainingForMaxPossible": remainingForMaxPossible,
                "progressForSubmitted":progressForSubmitted,
                "progressForMax":progressForMax,
                "stepWithMaxEvents":stepWithMaxEvents,
                "maxEvents":maxEvents,
                "eventsPerDay":eventsPerDay
                }

        cache.set("concatenate_ev_" + str(campaign['campaign']) + "_" + str(None) + "_v1",
                  pickle.dumps(data, pickle.HIGHEST_PROTOCOL), 60 * 60 * 72)
        self.logger.info("concatenate_ev_" + str(campaign['campaign']) + "_" + "  pushed into the cache")

        self.logger.info("SQLAggregatorCampaign finished")
        cursor.close()
        return 0

    def processPayload_wider_selection_backlog(self):

        self.logger.info("SQLAggregatorCampaign started")

        while (self.getNumberOfActiveDBSessions() > MAX_NUMBER_OF_ACTIVE_DB_SESSIONS):
            threading.sleep(TIMEOUT_WHEN_DB_LOADED)
        connection = self.pool.acquire()

        # we get recently alive campaigns
        campaigns = cph.getActiveCampaigns(connection)

        for campaign in campaigns:
            query = ""
            try:
                cursor = connection.cursor()
                cursor.execute("alter session set NLS_DATE_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")
                cursor.execute("alter session set NLS_TIMESTAMP_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")
                if campaign["subcampaign"]:
                    query = """
                        SELECT TASKID, START_TIME, ENDTIME, t1.STATUS, CAMPAIGN, SUBCAMPAIGN, TOTEVREM, TOTEV, t5.STEP_NAME FROM ATLAS_DEFT.T_PRODUCTION_TASK t1 JOIN ATLAS_PANDABIGMON.geteventsfortask t2
                        ON t1.TASKID=t2.JEDITASKID AND t1.campaign='%s' and t1.subcampaign='%s'
                        JOIN ATLAS_DEFT.T_PRODUCTION_STEP t4 ON t1.STEP_ID=t4.STEP_ID JOIN ATLAS_DEFT.T_STEP_TEMPLATE t5 ON t4.STEP_T_ID=t5.STEP_T_ID
                    """ % (campaign['campaign'], campaign['subcampaign'])
                else:
                    query = """
                        SELECT TASKID, START_TIME, ENDTIME, t1.STATUS, CAMPAIGN, SUBCAMPAIGN, TOTEVREM, TOTEV, t5.STEP_NAME FROM ATLAS_DEFT.T_PRODUCTION_TASK t1 JOIN ATLAS_PANDABIGMON.geteventsfortask t2
                        ON t1.TASKID=t2.JEDITASKID AND t1.campaign='%s' and t1.subcampaign is NULL
                        JOIN ATLAS_DEFT.T_PRODUCTION_STEP t4 ON t1.STEP_ID=t4.STEP_ID JOIN ATLAS_DEFT.T_STEP_TEMPLATE t5 ON t4.STEP_T_ID=t5.STEP_T_ID
                    """ % campaign['campaign']
            except Exception as e:
                self.logger.error(e)
                return -1

            #We retrieve data from the DB
            campaign_df = pd.read_sql(query, con=connection)
            campaign_df['START_TIME'] = pd.to_datetime(campaign_df['START_TIME'], format="%m-%d-%Y %H:%M:%S")
            campaign_df['ENDTIME'] = pd.to_datetime(campaign_df['ENDTIME'], format="%m-%d-%Y %H:%M:%S")
            campaign_df['DONEEV'] = campaign_df['TOTEV'] - campaign_df['TOTEVREM']
            stepsname = campaign_df.STEP_NAME.unique().tolist()

            #We do reindexing to have number of events deliveres per day
            index = pd.date_range(start=campaign_df.START_TIME.min(), end=campaign_df.ENDTIME.max())
            new_df = pd.DataFrame(index=index, columns=stepsname)
            new_df_total_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D'))][
                'DONEEV'].sum(), axis=1)
            new_df_Evgen_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Evgen')]['DONEEV'].sum(), axis=1)
            new_df_Simul_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Simul')]['DONEEV'].sum(), axis=1)
            new_df_Reco_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Reco')]['DONEEV'].sum(), axis=1)
            new_df_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Merge')]['DONEEV'].sum(), axis=1)
            new_df_Deriv_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Deriv')]['DONEEV'].sum(), axis=1)
            new_df_Evgen_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Evgen Merge')]['DONEEV'].sum(), axis=1)
            new_df_Rec_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Rec Merge')]['DONEEV'].sum(), axis=1)
            new_df_Deriv_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Deriv Merge')]['DONEEV'].sum(), axis=1)
            new_df_Rec_TAG_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Rec TAG')]['DONEEV'].sum(), axis=1)
            new_df_Digi_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Digi')]['DONEEV'].sum(), axis=1)

            reindexedbyDay_ev = pd.concat(
                [new_df_total_ev, new_df_Evgen_ev, new_df_Simul_ev, new_df_Reco_ev, new_df_Merge_ev,
                 new_df_Deriv_ev, new_df_Evgen_Merge_ev, new_df_Rec_Merge_ev, new_df_Deriv_Merge_ev, new_df_Rec_TAG_ev,
                 new_df_Digi_ev], axis=1, keys=['total', 'Evgen', 'Simul', 'Reco', 'Merge',
                                                'Deriv', 'Evgen Merge', 'Rec Merge', 'Deriv Merge', 'Rec TAG', 'Digi'])

            #Push data into the cache
            data = {"campaign_df":campaign_df,"reindexedbyDay_ev":reindexedbyDay_ev}
            cache.set( "concatenate_ev_"+str(campaign['campaign'])+"_"+str(campaign['subcampaign']), pickle.dumps(data, pickle.HIGHEST_PROTOCOL) , 60 * 60 * 72)
            self.logger.info("concatenate_ev_"+str(campaign['campaign'])+"_"+str(campaign['subcampaign']) + "  pushed into the cache")

        self.logger.info("SQLAggregatorCampaign finished")
        cursor.close()
        return 0

<commit_msg>campaing calculator item: - added debug information<commit_after>from BaseTasksProvider import BaseTasksProvider
from datetime import datetime, timedelta
import threading
from settingscron import MAX_NUMBER_OF_ACTIVE_DB_SESSIONS, TIMEOUT_WHEN_DB_LOADED
import logging
import pandas as pd
import numpy as np
from django.core.cache import cache
from django.utils.six.moves import cPickle as pickle
import core.libs.CampaignPredictionHelper as cph
import humanize

class SQLAggregatorCampaign(BaseTasksProvider):

    lock = threading.RLock()
    logger = logging.getLogger(__name__ + ' SQLAggregatorCampaign')

    def __dictfetchall(self, cursor):
        "Returns all rows from a cursor as a dict"
        desc = cursor.description
        return [
            dict(zip([col[0] for col in desc], row))
            for row in cursor.fetchall()
        ]

    def processPayload(self):

        # Here we do preprocessing for the prediction module restricted to the data18_13TeV reprocessing campaign

        self.logger.info("SQLAggregatorCampaign started")
        campaign = {
            'campaign': 'data18_13TeV'
        }

        while (self.getNumberOfActiveDBSessions() > MAX_NUMBER_OF_ACTIVE_DB_SESSIONS):
            threading.sleep(TIMEOUT_WHEN_DB_LOADED)
        connection = self.pool.acquire()
        cursor = connection.cursor()
        cursor.execute("alter session set NLS_DATE_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")
        cursor.execute("alter session set NLS_TIMESTAMP_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")

        #Columns names are specific due to DEFT compartibility
        query = """
            select STARTTIME as START_TIME, ENDTIME, STEP_NAME, round((NEVENTSUSED) / (count(*) over (partition by tr.jeditaskid))) as DONEEV  from (
            (SELECT DISTINCT STARTTIME, endtime, pandaid, jeditaskid FROM ATLAS_PANDAARCH.JOBSARCHIVED WHERE JOBSTATUS='finished' UNION ALL
            SELECT DISTINCT STARTTIME, endtime, pandaid, jeditaskid FROM ATLAS_PANDA.JOBSARCHIVED4 WHERE JOBSTATUS='finished')tr
             join (
                select t1.JEDITASKID, t5.STEP_NAME, t1.CREATIONDATE, t1.STATUS, t2.CAMPAIGN,t2.SUBCAMPAIGN, t1.neventstobeused, t1.nevents, t1.NEVENTSUSED from ATLAS_PANDABIGMON.RUNNINGPRODTASKS t1
                join ATLAS_DEFT.T_PRODUCTION_TASK t2 on t2.TASKID=t1.JEDITASKID
                JOIN ATLAS_DEFT.T_PRODUCTION_STEP t4 ON t2.STEP_ID=t4.STEP_ID JOIN ATLAS_DEFT.T_STEP_TEMPLATE t5 ON t4.STEP_T_ID=t5.STEP_T_ID
                where t2.campaign='data18_13TeV' and t1.processingtype='reprocessing'
            )td on tr.jeditaskid=td.jeditaskid) order by endtime ASC
        """

        self.logger.debug("SQLAggregatorCampaign starting filling pandas df")
        campaign_df = pd.read_sql(query, con=connection)
        index = pd.date_range(start=campaign_df.START_TIME.min(), end=campaign_df.ENDTIME.max())
        stepsname = campaign_df.STEP_NAME.unique().tolist()
        new_df = pd.DataFrame(index=index, columns=stepsname)

        new_df_Reco_ev = new_df.apply(lambda x: campaign_df[
            (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                    campaign_df.STEP_NAME == 'Reco')]['DONEEV'].sum(), axis=1)

        reindexedbyDay_ev = pd.concat(
            [new_df_Reco_ev], axis=1, keys=['Reco'])

        self.logger.debug("SQLAggregatorCampaign filled pandas df")

        # At this step we have number of events processed per each day of a campaign

        query = "select sum(neventstobeused) as sumtouse, sum(nevents) as sumtot, sum(NEVENTSUSED) as sumused, sum(NRUNNINGEVENTS) as sumrun from ATLAS_PANDABIGMON.RUNNINGPRODTASKS where campaign='data18_13TeV' and processingtype='reprocessing'"
        cursor.execute(query)
        eventstotals = self.__dictfetchall(cursor)
        if eventstotals and len(eventstotals) > 0:
            eventstotals = eventstotals[0]

        self.logger.debug("SQLAggregatorCampaign tasks filled")

        numberOfRemainingEventsPerStep = {
            'Reco': eventstotals['SUMTOUSE']
        }
        numberOfRunningEventsPerStep = {
            'Reco': eventstotals['SUMRUN']
        }
        numberOfTotalEventsPerStep = {
            'Reco': eventstotals['SUMTOT']
        }

        progressOfEventsPerStep = {
            'Reco': round(eventstotals['SUMUSED']/eventstotals['SUMTOT'],1)
        }
        numberOfDoneEventsPerStep = {
            'Reco': eventstotals['SUMUSED']
        }

        remainingForSubmitting = {}
        remainingForMaxPossible = {}
        progressForSubmitted = {}
        progressForMax = {}
        stepWithMaxEvents = 'Reco'
        maxEvents = numberOfTotalEventsPerStep[stepWithMaxEvents]
        eventsPerDay = {}

        rollingRes = reindexedbyDay_ev['Reco'].rolling(5, win_type='triang').mean()
        if len(rollingRes) > 2 and rollingRes[-1] > 0 and 'Reco' in numberOfRemainingEventsPerStep:
            remainingForSubmitting['Reco'] = str(round(numberOfRemainingEventsPerStep['Reco'] / \
                                                     rollingRes[-1], 2)) + " d"
            remainingForMaxPossible['Reco'] = str(round((maxEvents - numberOfDoneEventsPerStep['Reco']) / \
                                                      rollingRes[-1], 2)) + " d"
            progressForSubmitted['Reco'] = round(numberOfDoneEventsPerStep['Reco'] * 100.0 / (
                        numberOfDoneEventsPerStep['Reco'] + numberOfRemainingEventsPerStep['Reco']), 1)
            progressForMax['Reco'] = int(numberOfDoneEventsPerStep['Reco'] * 100.0 / maxEvents)
            eventsPerDay['Reco'] = humanize.intcomma(int(rollingRes[-1]))

        self.logger.debug("SQLAggregatorCampaign data prepared")

        data = {
                "numberOfRemainingEventsPerStep":numberOfRemainingEventsPerStep,
                "numberOfRunningEventsPerStep":numberOfRunningEventsPerStep,
                "numberOfTotalEventsPerStep": numberOfTotalEventsPerStep,
                "progressOfEventsPerStep":progressOfEventsPerStep,
                "numberOfDoneEventsPerStep":numberOfDoneEventsPerStep,
                "remainingForSubmitting":remainingForSubmitting,
                "remainingForMaxPossible": remainingForMaxPossible,
                "progressForSubmitted":progressForSubmitted,
                "progressForMax":progressForMax,
                "stepWithMaxEvents":stepWithMaxEvents,
                "maxEvents":maxEvents,
                "eventsPerDay":eventsPerDay
                }

        cache.set("concatenate_ev_" + str(campaign['campaign']) + "_" + str(None) + "_v1",
                  pickle.dumps(data, pickle.HIGHEST_PROTOCOL), 60 * 60 * 72)
        self.logger.info("concatenate_ev_" + str(campaign['campaign']) + "_" + "  pushed into the cache")

        self.logger.info("SQLAggregatorCampaign finished")
        cursor.close()
        return 0

    def processPayload_wider_selection_backlog(self):

        self.logger.info("SQLAggregatorCampaign started")

        while (self.getNumberOfActiveDBSessions() > MAX_NUMBER_OF_ACTIVE_DB_SESSIONS):
            threading.sleep(TIMEOUT_WHEN_DB_LOADED)
        connection = self.pool.acquire()

        # we get recently alive campaigns
        campaigns = cph.getActiveCampaigns(connection)

        for campaign in campaigns:
            query = ""
            try:
                cursor = connection.cursor()
                cursor.execute("alter session set NLS_DATE_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")
                cursor.execute("alter session set NLS_TIMESTAMP_FORMAT = 'mm-dd-yyyy HH24:mi:ss'")
                if campaign["subcampaign"]:
                    query = """
                        SELECT TASKID, START_TIME, ENDTIME, t1.STATUS, CAMPAIGN, SUBCAMPAIGN, TOTEVREM, TOTEV, t5.STEP_NAME FROM ATLAS_DEFT.T_PRODUCTION_TASK t1 JOIN ATLAS_PANDABIGMON.geteventsfortask t2
                        ON t1.TASKID=t2.JEDITASKID AND t1.campaign='%s' and t1.subcampaign='%s'
                        JOIN ATLAS_DEFT.T_PRODUCTION_STEP t4 ON t1.STEP_ID=t4.STEP_ID JOIN ATLAS_DEFT.T_STEP_TEMPLATE t5 ON t4.STEP_T_ID=t5.STEP_T_ID
                    """ % (campaign['campaign'], campaign['subcampaign'])
                else:
                    query = """
                        SELECT TASKID, START_TIME, ENDTIME, t1.STATUS, CAMPAIGN, SUBCAMPAIGN, TOTEVREM, TOTEV, t5.STEP_NAME FROM ATLAS_DEFT.T_PRODUCTION_TASK t1 JOIN ATLAS_PANDABIGMON.geteventsfortask t2
                        ON t1.TASKID=t2.JEDITASKID AND t1.campaign='%s' and t1.subcampaign is NULL
                        JOIN ATLAS_DEFT.T_PRODUCTION_STEP t4 ON t1.STEP_ID=t4.STEP_ID JOIN ATLAS_DEFT.T_STEP_TEMPLATE t5 ON t4.STEP_T_ID=t5.STEP_T_ID
                    """ % campaign['campaign']
            except Exception as e:
                self.logger.error(e)
                return -1

            #We retrieve data from the DB
            campaign_df = pd.read_sql(query, con=connection)
            campaign_df['START_TIME'] = pd.to_datetime(campaign_df['START_TIME'], format="%m-%d-%Y %H:%M:%S")
            campaign_df['ENDTIME'] = pd.to_datetime(campaign_df['ENDTIME'], format="%m-%d-%Y %H:%M:%S")
            campaign_df['DONEEV'] = campaign_df['TOTEV'] - campaign_df['TOTEVREM']
            stepsname = campaign_df.STEP_NAME.unique().tolist()

            #We do reindexing to have number of events deliveres per day
            index = pd.date_range(start=campaign_df.START_TIME.min(), end=campaign_df.ENDTIME.max())
            new_df = pd.DataFrame(index=index, columns=stepsname)
            new_df_total_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D'))][
                'DONEEV'].sum(), axis=1)
            new_df_Evgen_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Evgen')]['DONEEV'].sum(), axis=1)
            new_df_Simul_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Simul')]['DONEEV'].sum(), axis=1)
            new_df_Reco_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Reco')]['DONEEV'].sum(), axis=1)
            new_df_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Merge')]['DONEEV'].sum(), axis=1)
            new_df_Deriv_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Deriv')]['DONEEV'].sum(), axis=1)
            new_df_Evgen_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Evgen Merge')]['DONEEV'].sum(), axis=1)
            new_df_Rec_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Rec Merge')]['DONEEV'].sum(), axis=1)
            new_df_Deriv_Merge_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Deriv Merge')]['DONEEV'].sum(), axis=1)
            new_df_Rec_TAG_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Rec TAG')]['DONEEV'].sum(), axis=1)
            new_df_Digi_ev = new_df.apply(lambda x: campaign_df[
                (campaign_df.ENDTIME >= x.name) & (campaign_df.ENDTIME <= x.name + np.timedelta64(1, 'D')) & (
                            campaign_df.STEP_NAME == 'Digi')]['DONEEV'].sum(), axis=1)

            reindexedbyDay_ev = pd.concat(
                [new_df_total_ev, new_df_Evgen_ev, new_df_Simul_ev, new_df_Reco_ev, new_df_Merge_ev,
                 new_df_Deriv_ev, new_df_Evgen_Merge_ev, new_df_Rec_Merge_ev, new_df_Deriv_Merge_ev, new_df_Rec_TAG_ev,
                 new_df_Digi_ev], axis=1, keys=['total', 'Evgen', 'Simul', 'Reco', 'Merge',
                                                'Deriv', 'Evgen Merge', 'Rec Merge', 'Deriv Merge', 'Rec TAG', 'Digi'])

            #Push data into the cache
            data = {"campaign_df":campaign_df,"reindexedbyDay_ev":reindexedbyDay_ev}
            cache.set( "concatenate_ev_"+str(campaign['campaign'])+"_"+str(campaign['subcampaign']), pickle.dumps(data, pickle.HIGHEST_PROTOCOL) , 60 * 60 * 72)
            self.logger.info("concatenate_ev_"+str(campaign['campaign'])+"_"+str(campaign['subcampaign']) + "  pushed into the cache")

        self.logger.info("SQLAggregatorCampaign finished")
        cursor.close()
        return 0

<|endoftext|>