<commit_before>import datetime
import logging
import re

import pytz
import requests
from twisted.internet import error, reactor

from cardinal.bot import user_info
from cardinal.decorators import regex

# Alpha Vantage API key
AV_API_URL = "https://www.alphavantage.co/query"

CHECK_REGEX = r'^(?:<(.+?)>\s+)?!check (.+)'

def colorize(percentage):
    if percentage > 0:
        return '\x0309{:.2f}%\x03'.format(percentage)
    else:
        return '\x0304{:.2f}%\x03'.format(percentage)

class TickerPlugin(object):
    def __init__(self, cardinal, config):
        self.logger = logging.getLogger(__name__)
        self.cardinal = cardinal

        self.config = config or {}
        self.config.setdefault('channels', [])
        self.config.setdefault('stocks', [])
        self.config.setdefault('relay_bots', [])

        if not self.config["channels"]:
            self.logger.warning("No channels for ticker defined in config --"
                                "ticker will be disabled")
        if not self.config["stocks"]:
            self.logger.warning("No stocks for ticker defined in config -- "
                                "ticker will be disabled")

        if not self.config["api_key"]:
            raise KeyError("Missing required api_key in ticker config")
        if len(self.config["stocks"]) > 5:
            raise ValueError("No more than 5 stocks may be present in ticker "
                             "config")

        self.relay_bots = []
        for relay_bot in self.config['relay_bots']:
            user = user_info(
                relay_bot['nick'],
                relay_bot['user'],
                relay_bot['vhost'])
            self.relay_bots.append(user)

        self.call_id = None
        self.wait()

    def is_relay_bot(self, user):
        """Compares a user against the registered relay bots."""
        for bot in self.relay_bots:
            if (bot.nick is None or bot.nick == user.nick) and \
                    (bot.user is None or bot.user == user.user) and \
                    (bot.vhost is None or bot.vhost == user.vhost):
                return True

        return False

    def wait(self):
        """Tell the reactor to call tick() at the next 15 minute interval"""
        now = datetime.datetime.now()
        minutes_to_sleep = 15 - now.minute % 15
        seconds_to_sleep = minutes_to_sleep * 60
        seconds_to_sleep = seconds_to_sleep - now.second

        self.call_id = reactor.callLater(minutes_to_sleep * 60, self.tick)

    def tick(self):
        """Send a message with daily stock movements"""
        # If it's after 4pm ET or before 9:30am ET on a weekday, or if it's
        # a weekend (Saturday or Sunday), don't tick, just wait.
        tz = pytz.timezone('US/Eastern')
        now = datetime.datetime.now(tz)
        if (now.weekday() >= 5) or \
                (now.hour < 9 or now.hour >= 17) or \
                (now.hour == 9 and now.minute < 30) or \
                (now.hour == 16 and now.minute > 0):
            self.wait()
            return

        # If there are no stocks to send in the ticker, or no channels to send
        # them to, don't tick, just wait.
        if not self.config["channels"] or not self.config["stocks"]:
            self.wait()
            return

        try:
            results = {}
            for symbol, name in self.config["stocks"].items():
                results[symbol] = colorize(self.get_daily_change(symbol))

            messages = []
            for symbol, result in results.items():
                messages.append("{name} (\x02{symbol}\x02): {result}".format(
                    symbol=symbol,
                    name=self.config["stocks"][symbol],
                    result=result
                ))

            message = ' | '.join(messages)
            for channel in self.config["channels"]:
                self.cardinal.sendMsg(channel, message)
        except Exception as e:
            self.logger.exception("Error during tick: {}".format(e))

        self.wait()

    @regex(CHECK_REGEX)
    def check(self, cardinal, user, channel, msg):
        """Check a specific stock for current value and daily change"""
        nick = user.nick

        match = re.match(CHECK_REGEX, msg)
        if match.group(1):
            # this group should only be present when a relay bot is relaying a
            # message for another user
            if not self.is_relay_bot(user):
                return

            nick = match.group(1)
        symbol = match.group(2)
        try:
            data = self.get_daily(symbol)
        except Exception:
            cardinal.sendMsg(
                channel, "{}: Is your symbol correct?".format(nick))
            return

        cardinal.sendMsg(
            channel,
            "Symbol: \x02{}\x02 | Current: {} | Daily Change: {}".format(
                symbol,
                data['current'],
                colorize(data['percentage'])))

    def close(self, cardinal):
        if self.call_id:
            try:
                self.call_id.cancel()
            except error.AlreadyCancelled as e:
                self.logger.debug(e)

    def make_av_request(self, function, params=None):
        if params is None:
            params = {}
        params['function'] = function
        params['apikey'] = self.config["api_key"]
        params['datatype'] = 'json'

        r = requests.get(AV_API_URL, params=params)
        return r.json()

    def get_time_series_daily(self, symbol, outputsize='compact'):
        data = self.make_av_request('TIME_SERIES_DAILY',
                                    {'symbol': symbol,
                                     'outputsize': outputsize,
                                     })
        try:
            data = data['Time Series (Daily)']
        except KeyError:
            raise Exception("Error with data: {}".format(data))

        for date, values in data.items():
            values = {k[3:]: float(v) for k, v in values.items()}
            data[date] = values

        return data

    def get_daily(self, symbol):
        data = self.get_time_series_daily(symbol)

        today = datetime.date.today()
        last_day = today - datetime.timedelta(days=1)

        while data.get(last_day.strftime('%Y-%m-%d'), None) is None:
            last_day = last_day - datetime.timedelta(days=1)

        current_value = data[today.strftime('%Y-%m-%d')]
        last_day_value = data[last_day.strftime('%Y-%m-%d')]

        percentage = (((current_value['close'] / last_day_value['close']) - 1)
                      * 100)
        return {'current': current_value['close'],
                'percentage': percentage,
                }

    def get_daily_change(self, symbol):
        return self.get_daily(symbol)['percentage']

def setup(cardinal, config):
    return TickerPlugin(cardinal, config)
<commit_msg>Defer Alpha Vantage calls to thread in ticker plugin<commit_after>import datetime
import logging
import re

import pytz
import requests
from twisted.internet import defer, error, reactor
from twisted.internet.threads import deferToThread

from cardinal.bot import user_info
from cardinal.decorators import regex

# Alpha Vantage API key
AV_API_URL = "https://www.alphavantage.co/query"

CHECK_REGEX = r'^(?:<(.+?)>\s+)?!check (.+)'

def colorize(percentage):
    if percentage > 0:
        return '\x0309{:.2f}%\x03'.format(percentage)
    else:
        return '\x0304{:.2f}%\x03'.format(percentage)

class TickerPlugin(object):
    def __init__(self, cardinal, config):
        self.logger = logging.getLogger(__name__)
        self.cardinal = cardinal

        self.config = config or {}
        self.config.setdefault('channels', [])
        self.config.setdefault('stocks', [])
        self.config.setdefault('relay_bots', [])

        if not self.config["channels"]:
            self.logger.warning("No channels for ticker defined in config --"
                                "ticker will be disabled")
        if not self.config["stocks"]:
            self.logger.warning("No stocks for ticker defined in config -- "
                                "ticker will be disabled")

        if not self.config["api_key"]:
            raise KeyError("Missing required api_key in ticker config")
        if len(self.config["stocks"]) > 5:
            raise ValueError("No more than 5 stocks may be present in ticker "
                             "config")

        self.relay_bots = []
        for relay_bot in self.config['relay_bots']:
            user = user_info(
                relay_bot['nick'],
                relay_bot['user'],
                relay_bot['vhost'])
            self.relay_bots.append(user)

        self.call_id = None
        self.wait()

    def is_relay_bot(self, user):
        """Compares a user against the registered relay bots."""
        for bot in self.relay_bots:
            if (bot.nick is None or bot.nick == user.nick) and \
                    (bot.user is None or bot.user == user.user) and \
                    (bot.vhost is None or bot.vhost == user.vhost):
                return True

        return False

    def wait(self):
        """Tell the reactor to call tick() at the next 15 minute interval"""
        now = datetime.datetime.now()
        minutes_to_sleep = 15 - now.minute % 15
        seconds_to_sleep = minutes_to_sleep * 60
        seconds_to_sleep = seconds_to_sleep - now.second

        self.call_id = reactor.callLater(minutes_to_sleep * 60, self.tick)

    @defer.inlineCallbacks
    def tick(self):
        """Send a message with daily stock movements"""
        # If it's after 4pm ET or before 9:30am ET on a weekday, or if it's
        # a weekend (Saturday or Sunday), don't tick, just wait.
        tz = pytz.timezone('US/Eastern')
        now = datetime.datetime.now(tz)
        if (now.weekday() >= 5) or \
                (now.hour < 9 or now.hour >= 17) or \
                (now.hour == 9 and now.minute < 30) or \
                (now.hour == 16 and now.minute > 0):
            self.wait()
            return

        # If there are no stocks to send in the ticker, or no channels to send
        # them to, don't tick, just wait.
        if not self.config["channels"] or not self.config["stocks"]:
            self.wait()
            return

        try:
            results = {}
            for symbol, name in self.config["stocks"].items():
                change = yield self.get_daily_change(symbol)
                results[symbol] = colorize(change)

            messages = []
            for symbol, result in results.items():
                messages.append("{name} (\x02{symbol}\x02): {result}".format(
                    symbol=symbol,
                    name=self.config["stocks"][symbol],
                    result=result
                ))

            message = ' | '.join(messages)
            for channel in self.config["channels"]:
                self.cardinal.sendMsg(channel, message)
        except Exception as e:
            self.logger.exception("Error during tick: {}".format(e))

        self.wait()

    @regex(CHECK_REGEX)
    @defer.inlineCallbacks
    def check(self, cardinal, user, channel, msg):
        """Check a specific stock for current value and daily change"""
        nick = user.nick

        match = re.match(CHECK_REGEX, msg)
        if match.group(1):
            # this group should only be present when a relay bot is relaying a
            # message for another user
            if not self.is_relay_bot(user):
                return

            nick = match.group(1)
        symbol = match.group(2)
        try:
            data = yield self.get_daily(symbol)
        except Exception as exc:
            self.logger.warning("Error trying to look up symbol {}: {}".format(
                symbol, exc))
            cardinal.sendMsg(
                channel, "{}: Is your symbol correct?".format(nick))
            return

        cardinal.sendMsg(
            channel,
            "Symbol: \x02{}\x02 | Current: {} | Daily Change: {}".format(
                symbol,
                data['current'],
                colorize(data['percentage'])))

    def close(self, cardinal):
        if self.call_id:
            try:
                self.call_id.cancel()
            except error.AlreadyCancelled as e:
                self.logger.debug(e)

    @defer.inlineCallbacks
    def make_av_request(self, function, params=None):
        if params is None:
            params = {}
        params['function'] = function
        params['apikey'] = self.config["api_key"]
        params['datatype'] = 'json'

        r = yield deferToThread(requests.get, AV_API_URL, params=params)
        defer.returnValue(r.json())

    @defer.inlineCallbacks
    def get_time_series_daily(self, symbol, outputsize='compact'):
        data = yield self.make_av_request('TIME_SERIES_DAILY',
                                          {'symbol': symbol,
                                           'outputsize': outputsize,
                                           })
        try:
            data = data['Time Series (Daily)']
        except KeyError:
            raise Exception("Error with data: {}".format(data))

        for date, values in data.items():
            values = {k[3:]: float(v) for k, v in values.items()}
            data[date] = values

        defer.returnValue(data)

    @defer.inlineCallbacks
    def get_daily(self, symbol):
        data = yield self.get_time_series_daily(symbol)

        today = datetime.date.today()
        last_day = today - datetime.timedelta(days=1)

        while data.get(last_day.strftime('%Y-%m-%d'), None) is None:
            last_day = last_day - datetime.timedelta(days=1)

        current_value = data[today.strftime('%Y-%m-%d')]
        last_day_value = data[last_day.strftime('%Y-%m-%d')]

        percentage = (((current_value['close'] / last_day_value['close']) - 1)
                      * 100)
        defer.returnValue({'current': current_value['close'],
                           'percentage': percentage,
                           })

    @defer.inlineCallbacks
    def get_daily_change(self, symbol):
        res = yield self.get_daily(symbol)
        defer.returnValue(res['percentage'])

def setup(cardinal, config):
    return TickerPlugin(cardinal, config)
<|endoftext|>


<commit_before>VERSION = (2, 22, 4)

def get_version():
    return '.'.join(map(str, VERSION))
<commit_msg>2.22.5<commit_after>VERSION = (2, 22, 5)

def get_version():
    return '.'.join(map(str, VERSION))
<|endoftext|>


<commit_before>
import os
from os.path import join

from praatio import pitch_and_intensity

from promo import f0_morph
from promo.morph_utils import utils

# Define the arguments for the code
# Windows paths
path = r"C:\Users\Tim\Dropbox\workspace\prosodyMorph\examples\files"
praatEXE = r"C:\Praat.exe"

# Mac paths
path = "/Users/tmahrt/Dropbox/workspace/prosodyMorph/examples/files"
praatEXE = "/Applications/Praat.app/Contents/MacOS/Praat"

minPitch = 50
maxPitch = 350
stepList = utils.generateStepList(3)

fromName = "mary1"
toName = "mary2"
fromWavFN = fromName + ".wav"
toWavFN = toName + ".wav"

fromPitchFN = fromName + ".txt"
toPitchFN = toName + ".txt"

fromTGFN = join(path, os.path.splitext(fromWavFN)[0] + ".TextGrid")
toTGFN = join(path, os.path.splitext(toWavFN)[0] + ".TextGrid")

# Prepare the data for morphing
# 1st load it into memory
fromPitch = pitch_and_intensity.audioToPI(path, fromWavFN, path,
                                          fromPitchFN, praatEXE, minPitch,
                                          maxPitch, forceRegenerate=False)
toPitch = pitch_and_intensity.audioToPI(path, toWavFN, path,
                                        toPitchFN, praatEXE, minPitch,
                                        maxPitch, forceRegenerate=False)

# 2nd remove intensity values
fromPitch = [(time, pitch) for time, pitch, _ in fromPitch]
toPitch = [(time, pitch) for time, pitch, _ in toPitch]

# 3rd select which sections to align.
# We'll use textgrids for this purpose.
tierName = "PhonAlign"
fromPitch = f0_morph.getPitchForIntervals(fromPitch, fromTGFN, tierName)
toPitch = f0_morph.getPitchForIntervals(toPitch, toTGFN, tierName)

# Run the morph process
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE)

# Or for more control over the steps:
stepList = [0.10, ]  # 10% morph
# Run the morph process
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE)

# And we can reset the speaker's pitch range and mean pitch back to their own
stepList = [1.0, ]

f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_average" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=False,
                 keepAveragePitch=True)

f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_range" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=True,
                 keepAveragePitch=False)

f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_range_and_average" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=True,
                 keepAveragePitch=True)

f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_regular" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=False,
                 keepAveragePitch=False)<commit_msg>DOCUMENTATION: Expanded comments for clarity<commit_after>'''

An example of morphing the pitch in one file to that in another
'''
import os
from os.path import join

from praatio import pitch_and_intensity

from promo import f0_morph
from promo.morph_utils import utils

# Define the arguments for the code
# Windows paths
path = r"C:\Users\Tim\Dropbox\workspace\prosodyMorph\examples\files"
praatEXE = r"C:\Praat.exe"

# Mac paths
path = "/Users/tmahrt/Dropbox/workspace/prosodyMorph/examples/files"
praatEXE = "/Applications/Praat.app/Contents/MacOS/Praat"

minPitch = 50
maxPitch = 350
stepList = utils.generateStepList(3)

fromName = "mary1"
toName = "mary2"
fromWavFN = fromName + ".wav"
toWavFN = toName + ".wav"

fromPitchFN = fromName + ".txt"
toPitchFN = toName + ".txt"

fromTGFN = join(path, os.path.splitext(fromWavFN)[0] + ".TextGrid")
toTGFN = join(path, os.path.splitext(toWavFN)[0] + ".TextGrid")

# Prepare the data for morphing
# 1st load it into memory
fromPitch = pitch_and_intensity.audioToPI(path, fromWavFN, path,
                                          fromPitchFN, praatEXE, minPitch,
                                          maxPitch, forceRegenerate=False)
toPitch = pitch_and_intensity.audioToPI(path, toWavFN, path,
                                        toPitchFN, praatEXE, minPitch,
                                        maxPitch, forceRegenerate=False)

# 2nd remove intensity values
fromPitch = [(time, pitch) for time, pitch, _ in fromPitch]
toPitch = [(time, pitch) for time, pitch, _ in toPitch]

# 3rd select which sections to align.
# We'll use textgrids for this purpose.
tierName = "PhonAlign"
fromPitch = f0_morph.getPitchForIntervals(fromPitch, fromTGFN, tierName)
toPitch = f0_morph.getPitchForIntervals(toPitch, toTGFN, tierName)

# Run the morph process
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE)

# Or for more control over the steps:
stepList = [0.10, ]  # 10% morph
# Run the morph process
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE)

# And, as shown in the next four examples,
# we can reset the speaker's pitch range and mean pitch back to their own
stepList = [1.0, ]

# Source's mean pitch, target's pitch range
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_average" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=False,
                 keepAveragePitch=True)

# Target's mean pitch, source's pitch range
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_range" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=True,
                 keepAveragePitch=False)

# Source's mean pitch, source's pitch range
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_range_and_average" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=True,
                 keepAveragePitch=True)

# Target's mean pitch, target's pitch range (default behavior)
f0_morph.f0Morph(fromWavFN=join(path, fromWavFN),
                 pitchPath=path,
                 stepList=stepList,
                 outputName="%s_%s_f0_morph_w_regular" % (fromName, toName),
                 doPlotPitchSteps=True,
                 fromPitchData=fromPitch,
                 toPitchData=toPitch,
                 outputMinPitch=minPitch,
                 outputMaxPitch=maxPitch,
                 praatEXE=praatEXE,
                 keepPitchRange=False,
                 keepAveragePitch=False)
<|endoftext|>


<commit_before># Image windowing class

import numpy as np
import sidpy
from scipy import fftpack
from scipy.ndimage import zoom
from scipy.signal import hanning, blackman
from skimage.transform import rescale

class ImageWindowing:
    """
    This class will generate windows from sidpy dataset objects. At present only 2D windowing is allowed.
    """
    def __init__(self, parms_dict, verbose = False):
        '''Sliding Window Class.

        Parameters
        ----------
        - parms_dict : dictionary
            Dictionary with parameters of the windowing process, see below.

            Keys:
            - 'window_size_x' (integer) (required): size of the window across the x-axis
            - 'window_size_y' (integer) (required): size of the window across the y-axis
            - 'window_step_x' (integer) (required): step size of the window across the x-axis. Sometimes referred to as 'strides'
            - 'window_step_y' (integer) (required): step size of the window across the y-axis. Sometimes referred to as 'strides'
            - 'mode' (string) (Optional, default is 'image'): One of 'image' or 'fft' which defines the processing to be performed for each window.
                The choice of 'fft' will perform 2D fast Fourier transforms on each image whereas 'image' will not perform any operation on the window
            - 'fft_mode' (string) (Optional, default is 'abs'): If mode is 'fft', choose whether to look at amplitude or phase. Options are 'abs', 'phase'.
            - 'interpol_factor' (float) (Optional, default is 1.0): Interpolation factor for windows to increase or decrease size of the windows.
            - 'zoom_factor' (float) (Optional, default is 1.0): Zoom the window by this factor, typically done for 'fft' mode to observe higher frequencies clearly
            - 'filter' (string) (Optional, default is None): Filtering to use for the image window. Options are 'blackman', 'hanning'.
            The filter is applied to each window before 'mode'.
        - verbose : (Optional) Boolean
            Verbose flag. Default is False.

        Returns
        --------
        Instance of ImageWindowing object setup with parameters defined by the parms_dict above.
       '''

        self.window_step_x = parms_dict['window_step_x']
        self.window_step_y = parms_dict['window_step_y']
        self.window_size_x = parms_dict['window_size_x']
        self.window_size_y = parms_dict['window_size_y']
        self.fft_mode = 'abs'
        self.verbose = verbose

        if 'mode' in parms_dict.keys():
            if parms_dict['mode'] not in ['image', 'fft']:
                raise ValueError("Parameters dictionary field 'mode' must be one of 'image' or 'fft'."
                                 "Try again with one of these two options.")
            else:
                self.mode = parms_dict['mode']
        else:
            self.mode = 'image'
            parms_dict['mode'] = 'image'

        if 'interpol_factor' in parms_dict.keys(): self.interpol_factor = parms_dict['interpol_factor']
        else:
            self.interpol_factor = 1
            parms_dict['interpol_facor']=1

        if 'zoom_factor' in parms_dict.keys(): self.zoom_factor = parms_dict['zoom_factor']
        else:
            self.zoom_factor = 1
            parms_dict['zoom_facor'] = 1

        # Based on the zoom and interpolation factors we need to figure out the final size of the window
        self.window_size_final_x, self.window_size_final_y = self._get_window_size()
        #Setup the filter for the window
        self.filter = 'None'
        self.filter_mat = np.ones((self.window_size_final_x, self.window_size_final_y))

        if self.mode=='fft':
            #load FFT options
            if 'filter' in parms_dict.keys():
                if parms_dict['filter'] not in ['blackman', 'hanning']:
                    raise ValueError("Parameter 'filter' must be one of 'hanning', 'blackman'")
                else:
                    self.filter = parms_dict['filter']
                    if self.filter=='hanning':
                        filter_x = hanning(self.window_size_final_x)
                        filter_y = hanning(self.window_size_final_y)
                        self.filter_mat = np.sqrt(np.outer(filter_x,filter_y))
                    elif self.filter=='blackman':
                        filter_x = blackman(self.window_size_final_x)
                        filter_y = blackman(self.window_size_final_y)
                        self.filter_mat = np.sqrt(np.outer(filter_x,filter_y))
            if 'fft_mode' in parms_dict.keys():
                if parms_dict['fft_mode'] not in ['abs', 'phase']:
                    raise ValueError("Parameter 'fft_mode' must be one of 'abs', 'phase'")
                else:
                    self.fft_mode = parms_dict['fft_mode']
            else:
                self.fft_mode = 'abs' #default to absolute value in case fft mode is not provided
                parms_dict['fft_mode'] = 'abs'
        if self.verbose:
            print('ImageWindowing Object created with parameters {}'.format(parms_dict))

        self.window_parms = parms_dict

        return

    def _get_window_size(self):
        '''
        Computes window size based on zoom and interpolation factors
        '''

        image_test = np.random.uniform(size=(self.window_size_x, self.window_size_y))

        image_zoomed = zoom(image_test, self.zoom_factor)

        #interpolate it
        zoomed_interpolated = rescale(image_zoomed, self.interpol_factor)

        return zoomed_interpolated.shape[0],zoomed_interpolated.shape[1]

    def MakeWindows(self, dataset, dim_slice=None):
        '''
            Image should be a sidpy dataset object
            We will take the image to be the first two spatial dimensions,
            unless dimensions are specified

            Inputs:
                - dataset (sidpy.Dataset object of the image to be windowed)
                - dim_slice (List) (Optional). list of integers of the slices over which the
                image windowing should take place. This should be of length number of dimensions of
                the dataset minus two.

            Returns:
                - windowed_dataset (sidpy.Dataset) object with windows created as per
                the parameters passed to the ImageWindowing class.

            '''

        # This is the windowing function. Will generate the windows (but not the FFT)
        num_dimensions = dataset.ndim
        if dim_slice is None:
            if num_dimensions > 2:
                raise ValueError('You have specified windowing on a sidpy dataset '
                                 'with more than 2 dimensions without specifying slices')
            else:
                image_source = dataset[:]
                image_dims = [0,1]
        elif dim_slice is not None:
            """Get all spatial dimensions"""
            image_dims = []
            for dim, axis in dataset._axes.items():
                if axis.dimension_type == sidpy.DimensionType.SPATIAL:
                    image_dims.append(dim)
            all_dims = np.arange(0, num_dimensions)
            slice_list = []
            for k in range(num_dimensions):
                if k in image_dims:
                    slice_list.append(slice(None, dataset.shape[k], 1))
                else:
                    slice_list.append(dim_slice)
            image_source = dataset[tuple(slice_list)]

        image_shape = image_source.shape
        if self.verbose:
            print('Full image shape is {}'.format(image_shape))

        window_step = [self.window_step_x, self.window_step_y]
        window_size = [self.window_size_x, self.window_size_y]
        window_size_final = [self.window_size_final_x, self.window_size_final_y]

        dim_vec = []
        for i in range(2):
            dim_vec.append(np.append(np.arange(0, image_shape[i] - window_size[i], window_step[i]), image_shape[i] - window_size[i]))
        print("dim vec is {}".format(dim_vec))

        _, pos_vec = self.build_ind_val_matrices(dim_vec)
        if self.verbose:
            print("Pos vec is {}".format(pos_vec))

        pca_mat = np.zeros(shape=(pos_vec.shape[0], np.prod(window_size_final)), dtype=np.complex64)
        pos_vec = np.int32(pos_vec)

        for ind, pos in enumerate(pos_vec):
            start_stop = [slice(x, x + y, 1) for x, y in zip(pos, window_size)]
            full_slice = image_source[tuple(start_stop)]
            full_slice = self._return_win_image_processed(full_slice)
            pca_mat[ind] = full_slice.flatten()

        self.pos_vec = pos_vec

        # Get the positions and make them dimensions
        new_x_vals = np.linspace(dataset._axes[image_dims[0]].values.min(),
                                 dataset._axes[image_dims[0]].values.max(), len(np.unique(pos_vec[:, 0])))

        new_y_vals = np.linspace(dataset._axes[image_dims[1]].values.min(),
                                 dataset._axes[image_dims[1]].values.max(), len(np.unique(pos_vec[:, 1])))
        if self.verbose:
            print("position values x {} and y {}".format(new_x_vals, new_y_vals))
        windows_reshaped = pca_mat.reshape(len(new_x_vals), len(new_y_vals),
                                           self.window_size_final_x, self.window_size_final_y)
        if self.verbose:
            print('Reshaped windows size is {}'.format(windows_reshaped.shape))

        # Make a sidpy dataset
        #if the data is complex, then convert it to absolute
        #this needs to be changed..depending on user preferences.
        if np.iscomplexobj(windows_reshaped):
            if self.fft_mode == 'abs':
                windows_reshaped = np.array(np.abs(windows_reshaped), dtype = np.float64)
            elif self.fft_mode == 'phase':
                windows_reshaped = np.array(np.angle(windows_reshaped), dtype=np.float64)

        data_set = sidpy.Dataset.from_array(windows_reshaped,
                                            name='Image_Windowed')

        # Set the data type
        data_set.data_type = 'Image_4d'

        # Add quantity and units
        data_set.units = dataset.units
        data_set.quantity = dataset.quantity

        # Add dimension info

        window_size_fraction_x = window_size[0]/image_shape[0]
        window_size_fraction_y = window_size[1] / image_shape[1]

        window_extent_x = (dataset._axes[image_dims[0]].values.max() -
                           dataset._axes[image_dims[0]].values.min())*window_size_fraction_x

        window_extent_y = (dataset._axes[image_dims[1]].values.max() -
                           dataset._axes[image_dims[1]].values.min()) * window_size_fraction_y

        if self.mode =='fft':
            #to check if this is correct
            z_dimx = np.linspace(0, 1.0/(window_extent_x / self.zoom_factor), data_set.shape[2])
            z_dimy = np.linspace(0, 1.0/(window_extent_y / self.zoom_factor), data_set.shape[3])
        else:
            z_dimx = np.linspace(0, window_extent_x/self.zoom_factor, data_set.shape[2])
            z_dimy = np.linspace(0, window_extent_y/self.zoom_factor, data_set.shape[3])

        data_set.set_dimension(0, sidpy.Dimension(new_x_vals,
                                                  name=dataset._axes[image_dims[0]].name,
                                                  units=dataset._axes[image_dims[0]].units,
                                                  quantity=dataset._axes[image_dims[0]].quantity,
                                                  dimension_type='spatial'))

        data_set.set_dimension(1, sidpy.Dimension(new_y_vals,
                                                  name=dataset._axes[image_dims[1]].name,
                                                  units=dataset._axes[image_dims[1]].units,
                                                  quantity=dataset._axes[image_dims[1]].quantity,
                                                  dimension_type='spatial'))

        data_set.set_dimension(2, sidpy.Dimension(z_dimx,
                                                  name='WindowX',
                                                  units='m', quantity='kx',
                                                  dimension_type='spectral'))

        data_set.set_dimension(3, sidpy.Dimension(z_dimy,
                                                  name='WindowY',
                                                  units='m', quantity='ky',
                                                  dimension_type='spectral'))

        # append metadata
        data_set.metadata = self._merge_dictionaries(dataset.metadata, self.window_parms)

        return data_set
    #TODO: After next release of sidpy, remove this method and use sidpy.base.num_utils copy
    def build_ind_val_matrices(self, unit_values):
        """
        Builds indices and values matrices using given unit values for each dimension.
        This function is originally from pyUSID.io
        Unit values must be arranged from fastest varying to slowest varying

        Parameters
        ----------
        unit_values : list / tuple
            Sequence of values vectors for each dimension

        Returns
        -------
        ind_mat : 2D numpy array
            Indices matrix
        val_mat : 2D numpy array
            Values matrix
        """
        if not isinstance(unit_values, (list, tuple)):
            raise TypeError('unit_values should be a list or tuple')
        if not np.all([np.array(x).ndim == 1 for x in unit_values]):
            raise ValueError('unit_values should only contain 1D array')
        lengths = [len(x) for x in unit_values]
        tile_size = [np.prod(lengths[x:]) for x in range(1, len(lengths))] + [1]
        rep_size = [1] + [np.prod(lengths[:x]) for x in range(1, len(lengths))]
        val_mat = np.zeros(shape=(len(lengths), np.prod(lengths)))
        ind_mat = np.zeros(shape=val_mat.shape, dtype=np.uint32)
        for ind, ts, rs, vec in zip(range(len(lengths)), tile_size, rep_size, unit_values):
            val_mat[ind] = np.tile(np.repeat(vec, rs), ts)
            ind_mat[ind] = np.tile(np.repeat(np.arange(len(vec)), rs), ts)

        val_mat = val_mat.T
        ind_mat = ind_mat.T
        return ind_mat, val_mat

    def _return_win_image_processed(self, img_window):
        #Real image slice, returns it back with image processed
        if self.zoom_factor==1 and self.interpol_factor==1 and self.filter == 'None':
            #simply skip this function if there is no zooming, interpolation to be done.
            return img_window
        else:
            img_window = zoom(img_window, self.zoom_factor) #Zoom it
            img_window = rescale(img_window, self.interpol_factor) #Rescale
            img_window *= self.filter_mat  # Apply filter
            if self.mode == 'fft': img_window = np.fft.fftshift(np.fft.fft2(img_window))
        return img_window

    def _merge_dictionaries(self, dict1, dict2):
        #given two dictionaries, merge them into one
        merged_dict = {**dict1, **dict2}
        return merged_dict

<commit_msg>Added reconstruct image method<commit_after># Image windowing class

import numpy as np
import sidpy
from scipy import fftpack
from scipy.ndimage import zoom
from scipy.signal import hanning, blackman
from skimage.transform import rescale

class ImageWindowing:
    """
    This class will generate windows from sidpy dataset objects. At present only 2D windowing is allowed.
    """
    def __init__(self, parms_dict, verbose = False):
        '''Sliding Window Class.

        Parameters
        ----------
        - parms_dict : dictionary
            Dictionary with parameters of the windowing process, see below.

            Keys:
            - 'window_size_x' (integer) (required): size of the window across the x-axis
            - 'window_size_y' (integer) (required): size of the window across the y-axis
            - 'window_step_x' (integer) (required): step size of the window across the x-axis. Sometimes referred to as 'strides'
            - 'window_step_y' (integer) (required): step size of the window across the y-axis. Sometimes referred to as 'strides'
            - 'mode' (string) (Optional, default is 'image'): One of 'image' or 'fft' which defines the processing to be performed for each window.
                The choice of 'fft' will perform 2D fast Fourier transforms on each image whereas 'image' will not perform any operation on the window
            - 'fft_mode' (string) (Optional, default is 'abs'): If mode is 'fft', choose whether to look at amplitude or phase. Options are 'abs', 'phase'.
            - 'interpol_factor' (float) (Optional, default is 1.0): Interpolation factor for windows to increase or decrease size of the windows.
            - 'zoom_factor' (float) (Optional, default is 1.0): Zoom the window by this factor, typically done for 'fft' mode to observe higher frequencies clearly
            - 'filter' (string) (Optional, default is None): Filtering to use for the image window. Options are 'blackman', 'hanning'.
            The filter is applied to each window before 'mode'.
        - verbose : (Optional) Boolean
            Verbose flag. Default is False.

        Returns
        --------
        Instance of ImageWindowing object setup with parameters defined by the parms_dict above.
       '''

        self.window_step_x = parms_dict['window_step_x']
        self.window_step_y = parms_dict['window_step_y']
        self.window_size_x = parms_dict['window_size_x']
        self.window_size_y = parms_dict['window_size_y']
        self.fft_mode = 'abs'
        self.verbose = verbose

        if 'mode' in parms_dict.keys():
            if parms_dict['mode'] not in ['image', 'fft']:
                raise ValueError("Parameters dictionary field 'mode' must be one of 'image' or 'fft'."
                                 "Try again with one of these two options.")
            else:
                self.mode = parms_dict['mode']
        else:
            self.mode = 'image'
            parms_dict['mode'] = 'image'

        if 'interpol_factor' in parms_dict.keys(): self.interpol_factor = parms_dict['interpol_factor']
        else:
            self.interpol_factor = 1
            parms_dict['interpol_facor']=1

        if 'zoom_factor' in parms_dict.keys(): self.zoom_factor = parms_dict['zoom_factor']
        else:
            self.zoom_factor = 1
            parms_dict['zoom_facor'] = 1

        # Based on the zoom and interpolation factors we need to figure out the final size of the window
        self.window_size_final_x, self.window_size_final_y = self._get_window_size()
        #Setup the filter for the window
        self.filter = 'None'
        self.filter_mat = np.ones((self.window_size_final_x, self.window_size_final_y))

        if self.mode=='fft':
            #load FFT options
            if 'filter' in parms_dict.keys():
                if parms_dict['filter'] not in ['blackman', 'hanning']:
                    raise ValueError("Parameter 'filter' must be one of 'hanning', 'blackman'")
                else:
                    self.filter = parms_dict['filter']
                    if self.filter=='hanning':
                        filter_x = hanning(self.window_size_final_x)
                        filter_y = hanning(self.window_size_final_y)
                        self.filter_mat = np.sqrt(np.outer(filter_x,filter_y))
                    elif self.filter=='blackman':
                        filter_x = blackman(self.window_size_final_x)
                        filter_y = blackman(self.window_size_final_y)
                        self.filter_mat = np.sqrt(np.outer(filter_x,filter_y))
            if 'fft_mode' in parms_dict.keys():
                if parms_dict['fft_mode'] not in ['abs', 'phase']:
                    raise ValueError("Parameter 'fft_mode' must be one of 'abs', 'phase'")
                else:
                    self.fft_mode = parms_dict['fft_mode']
            else:
                self.fft_mode = 'abs' #default to absolute value in case fft mode is not provided
                parms_dict['fft_mode'] = 'abs'
        if self.verbose:
            print('ImageWindowing Object created with parameters {}'.format(parms_dict))

        self.window_parms = parms_dict

        return

    def _get_window_size(self):
        '''
        Computes window size based on zoom and interpolation factors
        '''

        image_test = np.random.uniform(size=(self.window_size_x, self.window_size_y))

        image_zoomed = zoom(image_test, self.zoom_factor)

        #interpolate it
        zoomed_interpolated = rescale(image_zoomed, self.interpol_factor)

        return zoomed_interpolated.shape[0],zoomed_interpolated.shape[1]

    def MakeWindows(self, dataset, dim_slice=None):
        '''
            Image should be a sidpy dataset object
            We will take the image to be the first two spatial dimensions,
            unless dimensions are specified

            Inputs:
                - dataset (sidpy.Dataset object of the image to be windowed)
                - dim_slice (List) (Optional). list of integers of the slices over which the
                image windowing should take place. This should be of length number of dimensions of
                the dataset minus two.

            Returns:
                - windowed_dataset (sidpy.Dataset) object with windows created as per
                the parameters passed to the ImageWindowing class.

            '''

        # This is the windowing function. Will generate the windows (but not the FFT)
        num_dimensions = dataset.ndim
        if dim_slice is None:
            if num_dimensions > 2:
                raise ValueError('You have specified windowing on a sidpy dataset '
                                 'with more than 2 dimensions without specifying slices')
            else:
                image_source = dataset[:]
                image_dims = [0,1]
        elif dim_slice is not None:
            """Get all spatial dimensions"""
            image_dims = []
            for dim, axis in dataset._axes.items():
                if axis.dimension_type == sidpy.DimensionType.SPATIAL:
                    image_dims.append(dim)
            all_dims = np.arange(0, num_dimensions)
            slice_list = []
            for k in range(num_dimensions):
                if k in image_dims:
                    slice_list.append(slice(None, dataset.shape[k], 1))
                else:
                    slice_list.append(dim_slice)
            image_source = dataset[tuple(slice_list)]

        self.image_shape = image_source.shape

        if self.verbose:
            print('Full image shape is {}'.format(self.image_shape))

        window_step = [self.window_step_x, self.window_step_y]
        window_size = [self.window_size_x, self.window_size_y]
        window_size_final = [self.window_size_final_x, self.window_size_final_y]

        dim_vec = []
        for i in range(2):
            dim_vec.append(np.append(np.arange(0, self.image_shape[i] - window_size[i], window_step[i]), self.image_shape[i] - window_size[i]))
        print("dim vec is {}".format(dim_vec))

        _, pos_vec = self.build_ind_val_matrices(dim_vec)
        if self.verbose:
            print("Pos vec is {}".format(pos_vec))

        pca_mat = np.zeros(shape=(pos_vec.shape[0], np.prod(window_size_final)), dtype=np.complex64)
        pos_vec = np.int32(pos_vec)

        for ind, pos in enumerate(pos_vec):
            start_stop = [slice(x, x + y, 1) for x, y in zip(pos, window_size)]
            full_slice = image_source[tuple(start_stop)]
            full_slice = self._return_win_image_processed(full_slice)
            pca_mat[ind] = full_slice.flatten()

        self.pos_vec = pos_vec

        # Get the positions and make them dimensions
        new_x_vals = np.linspace(dataset._axes[image_dims[0]].values.min(),
                                 dataset._axes[image_dims[0]].values.max(), len(np.unique(pos_vec[:, 0])))

        new_y_vals = np.linspace(dataset._axes[image_dims[1]].values.min(),
                                 dataset._axes[image_dims[1]].values.max(), len(np.unique(pos_vec[:, 1])))
        if self.verbose:
            print("position values x {} and y {}".format(new_x_vals, new_y_vals))
        windows_reshaped = pca_mat.reshape(len(new_x_vals), len(new_y_vals),
                                           self.window_size_final_x, self.window_size_final_y)
        if self.verbose:
            print('Reshaped windows size is {}'.format(windows_reshaped.shape))

        # Make a sidpy dataset
        #if the data is complex, then convert it to absolute
        #this needs to be changed..depending on user preferences.
        if np.iscomplexobj(windows_reshaped):
            if self.fft_mode == 'abs':
                windows_reshaped = np.array(np.abs(windows_reshaped), dtype = np.float64)
            elif self.fft_mode == 'phase':
                windows_reshaped = np.array(np.angle(windows_reshaped), dtype=np.float64)

        data_set = sidpy.Dataset.from_array(windows_reshaped,
                                            name='Image_Windowed')

        # Set the data type
        data_set.data_type = 'Image_4d'

        # Add quantity and units
        data_set.units = dataset.units
        data_set.quantity = dataset.quantity

        # Add dimension info

        window_size_fraction_x = window_size[0]/self.image_shape[0]
        window_size_fraction_y = window_size[1] / self.image_shape[1]

        window_extent_x = (dataset._axes[image_dims[0]].values.max() -
                           dataset._axes[image_dims[0]].values.min())*window_size_fraction_x

        window_extent_y = (dataset._axes[image_dims[1]].values.max() -
                           dataset._axes[image_dims[1]].values.min()) * window_size_fraction_y

        if self.mode =='fft':
            #to check if this is correct
            z_dimx = np.linspace(0, 1.0/(window_extent_x / self.zoom_factor), data_set.shape[2])
            z_dimy = np.linspace(0, 1.0/(window_extent_y / self.zoom_factor), data_set.shape[3])
        else:
            z_dimx = np.linspace(0, window_extent_x/self.zoom_factor, data_set.shape[2])
            z_dimy = np.linspace(0, window_extent_y/self.zoom_factor, data_set.shape[3])

        data_set.set_dimension(0, sidpy.Dimension(new_x_vals,
                                                  name=dataset._axes[image_dims[0]].name,
                                                  units=dataset._axes[image_dims[0]].units,
                                                  quantity=dataset._axes[image_dims[0]].quantity,
                                                  dimension_type='spatial'))

        data_set.set_dimension(1, sidpy.Dimension(new_y_vals,
                                                  name=dataset._axes[image_dims[1]].name,
                                                  units=dataset._axes[image_dims[1]].units,
                                                  quantity=dataset._axes[image_dims[1]].quantity,
                                                  dimension_type='spatial'))

        data_set.set_dimension(2, sidpy.Dimension(z_dimx,
                                                  name='WindowX',
                                                  units='m', quantity='kx',
                                                  dimension_type='spectral'))

        data_set.set_dimension(3, sidpy.Dimension(z_dimy,
                                                  name='WindowY',
                                                  units='m', quantity='ky',
                                                  dimension_type='spectral'))

        # append metadata
        data_set.metadata = self._merge_dictionaries(dataset.metadata, self.window_parms)

        return data_set
    #TODO: After next release of sidpy, remove this method and use sidpy.base.num_utils copy
    def build_ind_val_matrices(self, unit_values):
        """
        Builds indices and values matrices using given unit values for each dimension.
        This function is originally from pyUSID.io
        Unit values must be arranged from fastest varying to slowest varying

        Parameters
        ----------
        unit_values : list / tuple
            Sequence of values vectors for each dimension

        Returns
        -------
        ind_mat : 2D numpy array
            Indices matrix
        val_mat : 2D numpy array
            Values matrix
        """
        if not isinstance(unit_values, (list, tuple)):
            raise TypeError('unit_values should be a list or tuple')
        if not np.all([np.array(x).ndim == 1 for x in unit_values]):
            raise ValueError('unit_values should only contain 1D array')
        lengths = [len(x) for x in unit_values]
        tile_size = [np.prod(lengths[x:]) for x in range(1, len(lengths))] + [1]
        rep_size = [1] + [np.prod(lengths[:x]) for x in range(1, len(lengths))]
        val_mat = np.zeros(shape=(len(lengths), np.prod(lengths)))
        ind_mat = np.zeros(shape=val_mat.shape, dtype=np.uint32)
        for ind, ts, rs, vec in zip(range(len(lengths)), tile_size, rep_size, unit_values):
            val_mat[ind] = np.tile(np.repeat(vec, rs), ts)
            ind_mat[ind] = np.tile(np.repeat(np.arange(len(vec)), rs), ts)

        val_mat = val_mat.T
        ind_mat = ind_mat.T
        return ind_mat, val_mat

    def _return_win_image_processed(self, img_window):
        #Real image slice, returns it back with image processed
        if self.zoom_factor==1 and self.interpol_factor==1 and self.filter == 'None':
            #simply skip this function if there is no zooming, interpolation to be done.
            return img_window
        else:
            img_window = zoom(img_window, self.zoom_factor) #Zoom it
            img_window = rescale(img_window, self.interpol_factor) #Rescale
            img_window *= self.filter_mat  # Apply filter
            if self.mode == 'fft': img_window = np.fft.fftshift(np.fft.fft2(img_window))
        return img_window

    def _merge_dictionaries(self, dict1, dict2):
        #given two dictionaries, merge them into one
        merged_dict = {**dict1, **dict2}
        return merged_dict

<|endoftext|>


<commit_before>import urllib2,urlparse,pycurl,json,time,re,sys,time,os,socket,base64
from datetime import datetime
from celery.task.base import PeriodicTask,Task
from celery.signals import worker_ready
from django.conf import settings
from models import *
from kral.models import *
from tasks import *
from kral.tasks import *
from celery.registry import tasks
from celery.execute import send_task

class Twitter(Task):
    def run(self, querys, **kwargs):
        self.query_post = str("track="+",".join([q.text for q in querys]))
        print self.query_post
        self.request = urllib2.Request('http://stream.twitter.com/1/statuses/filter.json',self.query_post)
        self.auth = base64.b64encode('%s:%s' % (settings.TWITTER_USER, settings.TWITTER_PASS))
        self.request.add_header('Authorization', "basic %s" % self.auth)
        self.stream = urllib2.urlopen(self.request)
        for tweet in self.stream:
            ProcessTweet.delay(tweet, querys)

class ProcessTweet(Task):
    def run(self, data, querys, **kwargs):
        logger = self.get_logger(**kwargs)
        content = json.loads(data)
        user_id = content["user"].get('id_str', None)
        urls = content['entities']['urls']
        time_format = "%a %b %d %H:%M:%S +0000 %Y"
        if user_id is not None:
            twitter_user, created = TwitterUser.objects.get_or_create (
                user_id = user_id,
                user_name = content["user"]["screen_name"],
                real_name = content["user"]["name"],
                #location = content["user"]["location"],
                avatar = content["user"]["profile_image_url"],
                date = datetime.fromtimestamp(time.mktime(time.strptime(content["user"]["created_at"],time_format))),
                language = content["user"]["lang"],
                total_tweets = content["user"]["statuses_count"],
                #time_zone = content["user"]["time_zone"],
                listed = content["user"]["listed_count"],
                following = content["user"]["friends_count"],
                followers = content["user"]["followers_count"],
                geo_enabled = content["user"]["geo_enabled"],
                contributors_enabled = content["user"]["contributors_enabled"],
                #utc_offset = content["user"]["utc_offset"],
            )
            twitter_user.save()
            logger.debug("Saved/Updated profile for Twitter user %s" % (content["user"]["screen_name"]))
            try:
                twitter_tweet, created = TwitterTweet.objects.get_or_create(
                    date = datetime.fromtimestamp(time.mktime(time.strptime(content["created_at"],time_format))),
                    tweet_id = content["id_str"],
                    user_id = TwitterUser.objects.get(user_id=content["user"]["id_str"]),
                    text = content["text"],
                    #place = content["user"]["place"],
                    truncated = content['truncated'],
                    geo = content["user"]["location"],
                    contributors = content["contributors"],
                    #retweeted = content['retweeted'],
                    #irt_status_id = content['in_reply_to_status_id'],
                    #irt_status_name = content['in_reply_to_status_name'],
                    #retweet_count = content['retweet_count'],
                    #geo = content['geo'],
                )
                twitter_tweet.save()
                for query in [query.text.lower() for query in querys]:
                    if query in twitter_tweet.text.lower():
                        qobj = Query.objects.get(text__iexact=query)
                        twitter_tweet.querys.add(qobj)
                        for url in urls:
                            if url['expanded_url']:
                                send_task("kral.tasks.ExpandURL", [url['expanded_url'],query])
                            else:
                                send_task("kral.tasks.ExpandURL", [url['url'],query])
                        logger.debug("Added relation for tweet %s to %s" % (content['id_str'], qobj))
            except Exception, e:
                logger.warning("ERROR  - Unable to save tweet %s - %s" % (content["id_str"],e))

#vim: ai ts=5 sts=4 et sw=4
<commit_msg>More specific twitter error handeling<commit_after>import urllib2,urlparse,pycurl,json,time,re,sys,time,os,socket,base64
from datetime import datetime
from celery.task.base import PeriodicTask,Task
from celery.signals import worker_ready
from django.conf import settings
from models import *
from kral.models import *
from tasks import *
from kral.tasks import *
from celery.registry import tasks
from celery.execute import send_task

class Twitter(Task):
    def run(self, querys, **kwargs):
        logger = self.get_logger(**kwargs)
        self.query_post = str("track="+",".join([q.text for q in querys]))
        print self.query_post
        self.request = urllib2.Request('http://stream.twitter.com/1/statuses/filter.json',self.query_post)
        self.auth = base64.b64encode('%s:%s' % (settings.TWITTER_USER, settings.TWITTER_PASS))
        self.request.add_header('Authorization', "basic %s" % self.auth)
        try:
            self.stream = urllib2.urlopen(self.request)
        except Exception,e:
            if e.code == 420:
                logger.error("Maximum API connections exceeded")
            else:
                logger.error("Invalid/null response from server: %s" % (e))
            return False
        for tweet in self.stream:
            ProcessTweet.delay(tweet, querys)

class ProcessTweet(Task):
    def run(self, data, querys, **kwargs):
        logger = self.get_logger(**kwargs)
        content = json.loads(data)
        user_id = content["user"].get('id_str', None)
        urls = content['entities']['urls']
        time_format = "%a %b %d %H:%M:%S +0000 %Y"
        if user_id is not None:
            twitter_user, created = TwitterUser.objects.get_or_create (
                user_id = user_id,
                user_name = content["user"]["screen_name"],
                real_name = content["user"]["name"],
                #location = content["user"]["location"],
                avatar = content["user"]["profile_image_url"],
                date = datetime.fromtimestamp(time.mktime(time.strptime(content["user"]["created_at"],time_format))),
                language = content["user"]["lang"],
                total_tweets = content["user"]["statuses_count"],
                #time_zone = content["user"]["time_zone"],
                listed = content["user"]["listed_count"],
                following = content["user"]["friends_count"],
                followers = content["user"]["followers_count"],
                geo_enabled = content["user"]["geo_enabled"],
                contributors_enabled = content["user"]["contributors_enabled"],
                #utc_offset = content["user"]["utc_offset"],
            )
            twitter_user.save()
            logger.debug("Saved/Updated profile for Twitter user %s" % (content["user"]["screen_name"]))
            try:
                twitter_tweet, created = TwitterTweet.objects.get_or_create(
                    date = datetime.fromtimestamp(time.mktime(time.strptime(content["created_at"],time_format))),
                    tweet_id = content["id_str"],
                    user_id = TwitterUser.objects.get(user_id=content["user"]["id_str"]),
                    text = content["text"],
                    #place = content["user"]["place"],
                    truncated = content['truncated'],
                    geo = content["user"]["location"],
                    contributors = content["contributors"],
                    #retweeted = content['retweeted'],
                    #irt_status_id = content['in_reply_to_status_id'],
                    #irt_status_name = content['in_reply_to_status_name'],
                    #retweet_count = content['retweet_count'],
                    #geo = content['geo'],
                )
                twitter_tweet.save()
                for query in [query.text.lower() for query in querys]:
                    if query in twitter_tweet.text.lower():
                        qobj = Query.objects.get(text__iexact=query)
                        twitter_tweet.querys.add(qobj)
                        for url in urls:
                            if url['expanded_url']:
                                send_task("kral.tasks.ExpandURL", [url['expanded_url'],query])
                            else:
                                send_task("kral.tasks.ExpandURL", [url['url'],query])
                        logger.debug("Added relation for tweet %s to %s" % (content['id_str'], qobj))
            except Exception, e:
                logger.error("Unable to save tweet %s - %s" % (content["id_str"],e))

#vim: ai ts=5 sts=4 et sw=4
<|endoftext|>


<commit_before>#! /usr/bin/python3

import timeit

def filter_using_for_loop(numbers):
    even_numbers = []
    for number in numbers:
        if number % 2 == 0:
            even_numbers.append(number)

    return list(even_numbers)

def filter_function(numbers):
    def is_even(number):
        return number % 2 == 0

    return list(filter(is_even, numbers))

def filter_function_with_lambda(numbers):
    return list(filter(lambda number: number % 2 == 0, numbers))

def list_comprehension(numbers):
    return list([number for number in numbers if number % 2 == 0])

def truth_testing_with_filter(numbers):
    return list(filter(None, numbers))

def truth_testing_with_filter_and_builtin(numbers):
    return list(filter(bool, numbers))

def truth_testing_with_lambda_and_filter(numbers):
    return list(filter(lambda x: x, numbers))

def truth_testing_list_comprehension(numbers):
    return list([number for number in numbers if number])

def unique_items_with_set(elements):
    return list(set(elements))

def unique_items_with_for_loop(elements):
    new = []
    for element in elements:
        if element not in new:
            new.append(element)
    return list(new)

if __name__ == '__main__':
    elements = list(range(26)) * 200

    assert filter_using_for_loop(elements) == filter_function(elements)
    assert filter_function(elements) == filter_function_with_lambda(elements)
    assert filter_function_with_lambda(elements) == list_comprehension(elements)
    assert truth_testing_with_filter(elements) == truth_testing_with_lambda_and_filter(elements)
    assert truth_testing_with_filter(elements) == truth_testing_with_filter_and_builtin(elements)
    assert truth_testing_with_lambda_and_filter(elements) == truth_testing_list_comprehension(elements)
    assert unique_items_with_set(elements) == unique_items_with_for_loop(elements)

    repeat_kwargs = {"number": 1000000, "repeat": 3}
    print("Filtering a list of {0} elements - lowest result of {number} calls repeated {repeat} times".format(len(elements), **repeat_kwargs))
    print("for loop: {0:.2f}s".format(min(timeit.repeat("filter_using_for_loop(elements)", setup="from __main__ import filter_using_for_loop, elements ")), **repeat_kwargs))
    print("filter(): {0:.2f}s".format(min(timeit.repeat("filter_function(elements)", setup="from __main__ import filter_function, elements ")), **repeat_kwargs))
    print("filter() and lambda: {0:.2f}s".format(min(timeit.repeat("filter_function_with_lambda(elements)", setup="from __main__ import filter_function_with_lambda, elements ")), **repeat_kwargs))
    print("list comprehension: {0:.2f}s".format(min(timeit.repeat("list_comprehension(elements)", setup="from __main__ import list_comprehension, elements")), **repeat_kwargs))

    print("Truth testing with a list of {0} elements - lowest result of 1000000 calls repeated 3 times".format(len(elements), **repeat_kwargs))
    print("filter(): {0:.2f}s".format(min(timeit.repeat("truth_testing_with_filter(elements)", setup="from __main__ import truth_testing_with_filter, elements ")), **repeat_kwargs))
    print("filter() and lambda: {0:.2f}s".format(min(timeit.repeat("truth_testing_with_lambda_and_filter(elements)", setup="from __main__ import truth_testing_with_lambda_and_filter, elements ")), **repeat_kwargs))
    print("filter() and builtin: {0:.2f}s".format(min(timeit.repeat("truth_testing_with_filter_and_builtin(elements)", setup="from __main__ import truth_testing_with_filter_and_builtin, elements ")), **repeat_kwargs))
    print("list comprehension: {0:.2f}s".format(min(timeit.repeat("truth_testing_list_comprehension(elements)", setup="from __main__ import truth_testing_list_comprehension, elements")), **repeat_kwargs))

    print("Creating a list with of unique elements with a list of {0} elements - lowest result of 1000000 calls repeated 3 times".format(len(elements), **repeat_kwargs))
    print("set(): {0:.2f}s".format(min(timeit.repeat("unique_items_with_set(elements)", setup="from __main__ import unique_items_with_set, elements")), **repeat_kwargs))
    print("for-loop: {0:.2f}s".format(min(timeit.repeat("unique_items_with_for_loop(elements)", setup="from __main__ import unique_items_with_for_loop, elements")), **repeat_kwargs))
<commit_msg>Adding test for set comprehension.<commit_after>#! /usr/bin/python3

import timeit
import sys

RUNS_PYTHON_3 = bool(sys.version_info >= (3, ))

def filter_using_for_loop(numbers):
    even_numbers = []
    for number in numbers:
        if number % 2 == 0:
            even_numbers.append(number)

    return list(even_numbers)

def filter_function(numbers):
    def is_even(number):
        return number % 2 == 0

    return list(filter(is_even, numbers))

def filter_function_with_lambda(numbers):
    return list(filter(lambda number: number % 2 == 0, numbers))

def list_comprehension(numbers):
    return list([number for number in numbers if number % 2 == 0])

def truth_testing_with_filter(numbers):
    return list(filter(None, numbers))

def truth_testing_with_filter_and_builtin(numbers):
    return list(filter(bool, numbers))

def truth_testing_with_lambda_and_filter(numbers):
    return list(filter(lambda x: x, numbers))

def truth_testing_list_comprehension(numbers):
    return list([number for number in numbers if number])

def unique_items_with_set(elements):
    return list(set(elements))

if RUNS_PYTHON_3:
    def unique_items_with_set_comprehension(elements):
        return list({element for element in elements})

def unique_items_with_for_loop(elements):
    new = []
    for element in elements:
        if element not in new:
            new.append(element)
    return list(new)

if __name__ == '__main__':
    elements = list(range(26)) * 200

    assert filter_using_for_loop(elements) == filter_function(elements)
    assert filter_function(elements) == filter_function_with_lambda(elements)
    assert filter_function_with_lambda(elements) == list_comprehension(elements)
    assert truth_testing_with_filter(elements) == truth_testing_with_lambda_and_filter(elements)
    assert truth_testing_with_filter(elements) == truth_testing_with_filter_and_builtin(elements)
    assert truth_testing_with_lambda_and_filter(elements) == truth_testing_list_comprehension(elements)
    assert unique_items_with_set(elements) == unique_items_with_for_loop(elements)
    if RUNS_PYTHON_3:
        assert unique_items_with_set_comprehension(elements) == unique_items_with_for_loop(elements)

    repeat_kwargs = {"number": 1000000, "repeat": 3}
    print("Filtering a list of {0} elements - lowest result of {number} calls repeated {repeat} times".format(len(elements), **repeat_kwargs))
    print("for loop: {0:.2f}s".format(min(timeit.repeat("filter_using_for_loop(elements)", setup="from __main__ import filter_using_for_loop, elements ")), **repeat_kwargs))
    print("filter(): {0:.2f}s".format(min(timeit.repeat("filter_function(elements)", setup="from __main__ import filter_function, elements ")), **repeat_kwargs))
    print("filter() and lambda: {0:.2f}s".format(min(timeit.repeat("filter_function_with_lambda(elements)", setup="from __main__ import filter_function_with_lambda, elements ")), **repeat_kwargs))
    print("list comprehension: {0:.2f}s".format(min(timeit.repeat("list_comprehension(elements)", setup="from __main__ import list_comprehension, elements")), **repeat_kwargs))

    print("Truth testing with a list of {0} elements - lowest result of 1000000 calls repeated 3 times".format(len(elements), **repeat_kwargs))
    print("filter(): {0:.2f}s".format(min(timeit.repeat("truth_testing_with_filter(elements)", setup="from __main__ import truth_testing_with_filter, elements ")), **repeat_kwargs))
    print("filter() and lambda: {0:.2f}s".format(min(timeit.repeat("truth_testing_with_lambda_and_filter(elements)", setup="from __main__ import truth_testing_with_lambda_and_filter, elements ")), **repeat_kwargs))
    print("filter() and builtin: {0:.2f}s".format(min(timeit.repeat("truth_testing_with_filter_and_builtin(elements)", setup="from __main__ import truth_testing_with_filter_and_builtin, elements ")), **repeat_kwargs))
    print("list comprehension: {0:.2f}s".format(min(timeit.repeat("truth_testing_list_comprehension(elements)", setup="from __main__ import truth_testing_list_comprehension, elements")), **repeat_kwargs))

    print("Creating a list with of unique elements with a list of {0} elements - lowest result of 1000000 calls repeated 3 times".format(len(elements), **repeat_kwargs))
    print("set(): {0:.2f}s".format(min(timeit.repeat("unique_items_with_set(elements)", setup="from __main__ import unique_items_with_set, elements")), **repeat_kwargs))
    if RUNS_PYTHON_3:
        print("set comprehension: {0:.2f}s".format(min(timeit.repeat("unique_items_with_set_comprehension(elements)", setup="from __main__ import unique_items_with_set_comprehension, elements")), **repeat_kwargs))
    print("for-loop: {0:.2f}s".format(min(timeit.repeat("unique_items_with_for_loop(elements)", setup="from __main__ import unique_items_with_for_loop, elements")), **repeat_kwargs))
<|endoftext|>