<commit_before>
import pyglet
import array
import logging

from unlock.view.view import UnlockView
from math import cos, sin, radians
from unlock.util import switch

class SpritePositionComputer(object):
    North = 0
    NorthEast = 1
    East = 2
    SouthEast = 3
    South = 4
    SouthWest = 5
    West = 6
    NorthWest = 7
    Center = 8

    def __init__(self, canvas, image_width, image_height, rotation):
        self.width = canvas.width
        self.height = canvas.height
        self.x0 = canvas.x
        self.y0 = canvas.y
        self.angle = abs(radians(rotation))
        self.box_width = (image_width * cos(self.angle) + image_height * sin(self.angle))
        self.box_height = (image_width * sin(self.angle) + image_height * cos(self.angle))
        self.center()

    def compute(self, position):
        {
            SpritePositionComputer.North: self.north,
            SpritePositionComputer.NorthEast: self.northeast,
            SpritePositionComputer.East: self.east,
            SpritePositionComputer.SouthEast: self.southeast,
            SpritePositionComputer.South: self.south,
            SpritePositionComputer.SouthWest: self.southwest,
            SpritePositionComputer.West: self.west,
            SpritePositionComputer.NorthWest: self.northwest,
            SpritePositionComputer.Center: self.center,
        }.get(position, self.center)()
#        print("X, y", self.x, self.y)
#        print("X, y", self.x0, self.y0)
#        print("width, height", self.width, self.height)
#        print("box_width, box_height", self.box_width, self.box_height)
        self.x += self.x0
        self.y += self.y0

    def north(self):
        self.x = self.width / 2
        self.y = self.height - self.box_height / 2

    def northeast(self):
        self.x = self.width - self.box_width / 2
        self.y = self.height - self.box_height / 2

    def east(self):
        self.x = self.width - self.box_width / 2
        self.y = self.height / 2

    def southeast(self):
        self.x = self.width - self.box_width / 2
        self.y = self.box_height / 2

    def south(self):
        self.x = self.width / 2
        self.y = self.box_height / 2

    def southwest(self):
        self.x = self.box_width / 2
        self.y = self.box_height / 2

    def west(self):
        self.x = self.box_width / 2
        self.y = self.height / 2

    def northwest(self):
        self.x = self.box_width / 2
        self.y = self.height  - self.box_height/2

    def center(self):
        self.x = self.width / 2
        self.y = self.height / 2

class PygletSprite(UnlockView):
    def __init__(self, model, canvas, image, x, y, rotation):
        self.model = model
        self.canvas = canvas

        image.anchor_x = int(image.width / 2)
        image.anchor_y = int(image.height / 2)

        self.sprite = pyglet.sprite.Sprite(image, batch=self.canvas.batch)
        self.sprite.rotation = rotation
        self.sprite.x = x
        self.sprite.y = y
        self.sprite.visible = False
        self.logger = logging.getLogger(__name__)

    def render(self):
        self.sprite.visible = self.model.get_state()

class FlickeringPygletSprite(PygletSprite):
    def __init__(self, sprite, reversed_sprite, batch):
        self.sprite = sprite
        self.reversed_sprite = reversed_sprite
        self.batch = batch
        self.logger = logging.getLogger(__name__)

    def render(self):
        state = self.sprite.model.get_state()
        if state == None:
            self.sprite.sprite.visible = False
            self.reversed_sprite.visible = False
        else:
            #print("FlickeringPygletSprite, x,y = ", self.sprite.sprite.x, self.sprite.sprite.y, " rotation ", self.sprite.sprite.rotation, " visible = ", state)
            self.sprite.sprite.visible = state
            self.reversed_sprite.sprite.visible = not state
        #print('flickeringsprit state = ', state)

class CheckerboardProperties(object):
    """
    An object that collects all the properties used to generate a
    checkerboard sprite.

    :param width: width of the checkerboard in pixels
    :param height: height of the checkerboard in pixels
    :param x_tiles: total number of horizontal checks
    :param y_tiles: total number of vertical checks
    :param x_ratio: width ratio between alternating checks
    :param y_ratio: height ratio between alternating checks
    :param color1: RGB color of one set of checks
    :param color2: RGB color of the other set of checks
    """
    def __init__(self, width=300, height=300, x_tiles=4, y_tiles=4,
                 x_ratio=1, y_ratio=1, color1=(0, 0, 0),
                 color2=(255, 255, 255)):
        self.width = width
        self.height = height
        self.x_tiles = x_tiles
        self.y_tiles = y_tiles
        self.x_ratio = x_ratio
        self.y_ratio = y_ratio
        self.color1 = color1
        self.color2 = color2

        # Computed properties
        pair_width = int(width / x_tiles * 2)
        self.tile1_width = int(pair_width * (x_ratio / (x_ratio + 1)))
        self.tile2_width = pair_width - self.tile1_width

        pair_height = int(height / y_tiles * 2)
        self.tile1_height = int(pair_height * (y_ratio / (y_ratio + 1)))
        self.tile2_height = pair_height - self.tile1_height
<commit_msg>bugfix for reversal image not turning off when state is none<commit_after>
import pyglet
import array
import logging

from unlock.view.view import UnlockView
from math import cos, sin, radians
from unlock.util import switch

class SpritePositionComputer(object):
    North = 0
    NorthEast = 1
    East = 2
    SouthEast = 3
    South = 4
    SouthWest = 5
    West = 6
    NorthWest = 7
    Center = 8

    def __init__(self, canvas, image_width, image_height, rotation):
        self.width = canvas.width
        self.height = canvas.height
        self.x0 = canvas.x
        self.y0 = canvas.y
        self.angle = abs(radians(rotation))
        self.box_width = (image_width * cos(self.angle) + image_height * sin(self.angle))
        self.box_height = (image_width * sin(self.angle) + image_height * cos(self.angle))
        self.center()

    def compute(self, position):
        {
            SpritePositionComputer.North: self.north,
            SpritePositionComputer.NorthEast: self.northeast,
            SpritePositionComputer.East: self.east,
            SpritePositionComputer.SouthEast: self.southeast,
            SpritePositionComputer.South: self.south,
            SpritePositionComputer.SouthWest: self.southwest,
            SpritePositionComputer.West: self.west,
            SpritePositionComputer.NorthWest: self.northwest,
            SpritePositionComputer.Center: self.center,
        }.get(position, self.center)()
#        print("X, y", self.x, self.y)
#        print("X, y", self.x0, self.y0)
#        print("width, height", self.width, self.height)
#        print("box_width, box_height", self.box_width, self.box_height)
        self.x += self.x0
        self.y += self.y0

    def north(self):
        self.x = self.width / 2
        self.y = self.height - self.box_height / 2

    def northeast(self):
        self.x = self.width - self.box_width / 2
        self.y = self.height - self.box_height / 2

    def east(self):
        self.x = self.width - self.box_width / 2
        self.y = self.height / 2

    def southeast(self):
        self.x = self.width - self.box_width / 2
        self.y = self.box_height / 2

    def south(self):
        self.x = self.width / 2
        self.y = self.box_height / 2

    def southwest(self):
        self.x = self.box_width / 2
        self.y = self.box_height / 2

    def west(self):
        self.x = self.box_width / 2
        self.y = self.height / 2

    def northwest(self):
        self.x = self.box_width / 2
        self.y = self.height  - self.box_height/2

    def center(self):
        self.x = self.width / 2
        self.y = self.height / 2

class PygletSprite(UnlockView):
    def __init__(self, model, canvas, image, x, y, rotation):
        self.model = model
        self.canvas = canvas

        image.anchor_x = int(image.width / 2)
        image.anchor_y = int(image.height / 2)

        self.sprite = pyglet.sprite.Sprite(image, batch=self.canvas.batch)
        self.sprite.rotation = rotation
        self.sprite.x = x
        self.sprite.y = y
        self.sprite.visible = False
        self.logger = logging.getLogger(__name__)

    def render(self):
        self.sprite.visible = self.model.get_state()

class FlickeringPygletSprite(PygletSprite):
    def __init__(self, sprite, reversed_sprite, batch):
        self.sprite = sprite
        self.reversed_sprite = reversed_sprite
        self.batch = batch
        self.logger = logging.getLogger(__name__)

    def render(self):
        state = self.sprite.model.get_state()
        if state is None:
            self.sprite.sprite.visible = False
            self.reversed_sprite.sprite.visible = False
        else:
            #print("FlickeringPygletSprite, x,y = ", self.sprite.sprite.x, self.sprite.sprite.y, " rotation ", self.sprite.sprite.rotation, " visible = ", state)
            self.sprite.sprite.visible = state
            self.reversed_sprite.sprite.visible = not state
        #print('flickeringsprit state = ', state)

class CheckerboardProperties(object):
    """
    An object that collects all the properties used to generate a
    checkerboard sprite.

    :param width: width of the checkerboard in pixels
    :param height: height of the checkerboard in pixels
    :param x_tiles: total number of horizontal checks
    :param y_tiles: total number of vertical checks
    :param x_ratio: width ratio between alternating checks
    :param y_ratio: height ratio between alternating checks
    :param color1: RGB color of one set of checks
    :param color2: RGB color of the other set of checks
    """
    def __init__(self, width=300, height=300, x_tiles=4, y_tiles=4,
                 x_ratio=1, y_ratio=1, color1=(0, 0, 0),
                 color2=(255, 255, 255)):
        self.width = width
        self.height = height
        self.x_tiles = x_tiles
        self.y_tiles = y_tiles
        self.x_ratio = x_ratio
        self.y_ratio = y_ratio
        self.color1 = color1
        self.color2 = color2

        # Computed properties
        pair_width = int(width / x_tiles * 2)
        self.tile1_width = int(pair_width * (x_ratio / (x_ratio + 1)))
        self.tile2_width = pair_width - self.tile1_width

        pair_height = int(height / y_tiles * 2)
        self.tile1_height = int(pair_height * (y_ratio / (y_ratio + 1)))
        self.tile2_height = pair_height - self.tile1_height
<|endoftext|>


<commit_before>import gc
import sys

import numpy as np

import nrrd

adjust_thresh = 0.0035

def AutoBalance(data, threshold=adjust_thresh, background=0):
    if np.max(data) == 0:
        data = np.uint8(data)
    else:
        data = np.uint8(np.round(255.0 * (np.double(data) / np.max(data))))  # scale to 8 bit
    bins = np.unique(data)
    binc = np.bincount(data.flat)
    histogram = binc[binc > 0]
    del binc
    gc.collect()
    temp = 1
    c = 0
    mid = int(np.round(np.shape(bins)[0] / 2.0))
    # low end threshold only uses data from low end
    if background in bins[:mid]:
        i = np.where(bins == background)
        v = bins[i][0]
        c = histogram[i][0]
        th = long((np.sum(histogram[:mid]) - c) * threshold)
    else:
        th = long((np.sum(histogram[:mid])) * threshold)
    print 'low end balancing:'
    print 'number of background voxels: ' + str(c)
    print 'number of data voxels: ' + str(np.sum(histogram[:mid]) - c)
    print 'threshold set to: ' + str(th)
    m = np.min(bins)
    M = np.max(bins)
    if M == m:
        m = np.uint8(0)
        M = np.uint8(255)
    else:
        for x in range(1, np.shape(bins)[0] - 1):
            if (np.sum(histogram[0:x]) - c) > th:
                m = bins[x - 1]
                temp = x - 1
                break
        print 'number of low end voxels cut: ' + str(np.sum(histogram[0:temp]) - c)
        # high end threshold only uses data from high end
        c = 0
        if background in bins[mid:]:
            i = np.where(bins == background)
            v = bins[i][0]
            c = histogram[i][0]
            th = long((np.sum(histogram[mid:]) - c) * threshold)
        else:
            th = long((np.sum(histogram[mid:])) * threshold)
        print 'high end balancing:'
        print 'number of background voxels: ' + str(c)
        print 'number of data voxels: ' + str(np.sum(histogram[mid:]) - c)
        print 'threshold set to: ' + str(th)
        for x in range(np.shape(bins)[0] - 1, 0, -1):
            if (np.sum(histogram[x:])) > th:
                M = bins[x]
                temp = x + 1
                break
        print 'number of high end voxels cut: ' + str(np.sum(histogram[temp:]))
    del temp
    gc.collect()
    data[data > M] = M
    data[data < m] = m
    dataA = np.round((data - m) * (255.0 / (M - m)))
    hist = np.zeros(255, dtype=long)
    for i in range(0, np.shape(bins)[0] - 1):
        hist[bins[i]] = histogram[i]
    return dataA, {'min': int(m), 'max': int(M)}, hist

if __name__ == "__main__":
    if (len(sys.argv) < 2):
        print 'Error: missing arguments!'
        print 'e.g. python autoBalance.py imageIn.nrrd [imageOut.nrrd] [0] [0.0035]'
        print '0 = Background level'
        print '0.0035 = adjustment threshold - max percentage of voxels that can be clipped'
    else:
        outFile = str(sys.argv[1])
        if (len(sys.argv) > 2):
            outFile = str(sys.argv[2])
        BG = 0
        if (len(sys.argv) > 3):
            BG = int(sys.argv[3])
        TH = adjust_thresh
        if (len(sys.argv) > 4):
            TH = float(sys.argv[4])
        print 'Loading input file %s...' % (str(sys.argv[1]))
        data, header = nrrd.read(str(sys.argv[1]))
        chan, Nbound, hist = AutoBalance(data, threshold=TH, background=BG)
        print 'New boundaries %s' % (str(Nbound))
        print 'Saving result to %s...' % (outFile)
        nrrd.write(outFile, np.uint8(chan), options=header)
<commit_msg>adding debug<commit_after>import gc
import sys

import numpy as np

import nrrd

adjust_thresh = 0.0035

def AutoBalance(data, threshold=adjust_thresh, background=0):
    try:
        if np.max(data) == 0:
            data = np.uint8(data)
        else:
            data = np.uint8(np.round(255.0 * (np.double(data) / np.max(data))))  # scale to 8 bit
        bins = np.unique(data)
        binc = np.bincount(data.flat)
        histogram = binc[binc > 0]
        del binc
        gc.collect()
        temp = 1
        c = 0
        mid = int(np.round(np.shape(bins)[0] / 2.0))
        # low end threshold only uses data from low end
        if background in bins[:mid]:
            i = np.where(bins == background)
            v = bins[i][0]
            c = histogram[i][0]
            th = long((np.sum(histogram[:mid]) - c) * threshold)
        else:
            th = long((np.sum(histogram[:mid])) * threshold)
        print 'low end balancing:'
        print 'number of background voxels: ' + str(c)
        print 'number of data voxels: ' + str(np.sum(histogram[:mid]) - c)
        print 'threshold set to: ' + str(th)
        m = np.min(bins)
        M = np.max(bins)
        if M == m:
            m = np.uint8(0)
            M = np.uint8(255)
        else:
            for x in range(1, np.shape(bins)[0] - 1):
                if (np.sum(histogram[0:x]) - c) > th:
                    m = bins[x - 1]
                    temp = x - 1
                    break
            print 'number of low end voxels cut: ' + str(np.sum(histogram[0:temp]) - c)
            # high end threshold only uses data from high end
            c = 0
            if background in bins[mid:]:
                i = np.where(bins == background)
                v = bins[i][0]
                c = histogram[i][0]
                th = long((np.sum(histogram[mid:]) - c) * threshold)
            else:
                th = long((np.sum(histogram[mid:])) * threshold)
            print 'high end balancing:'
            print 'number of background voxels: ' + str(c)
            print 'number of data voxels: ' + str(np.sum(histogram[mid:]) - c)
            print 'threshold set to: ' + str(th)
            for x in range(np.shape(bins)[0] - 1, 0, -1):
                if (np.sum(histogram[x:])) > th:
                    M = bins[x]
                    temp = x + 1
                    break
            print 'number of high end voxels cut: ' + str(np.sum(histogram[temp:]))
        del temp
        gc.collect()
        data[data > M] = M
        data[data < m] = m
        dataA = np.round((data - m) * (255.0 / (M - m)))
        hist = np.zeros(255, dtype=long)
        for i in range(0, np.shape(bins)[0] - 1):
            hist[bins[i]] = histogram[i]
        return dataA, {'min': int(m), 'max': int(M)}, hist
    except Exception, e:
        print 'Error during intensity auto balancing:'
        print (e)
        return data, {'min': int(0), 'max': int(255)}, np.zeros(255, dtype=long)

if __name__ == "__main__":
    if (len(sys.argv) < 2):
        print 'Error: missing arguments!'
        print 'e.g. python autoBalance.py imageIn.nrrd [imageOut.nrrd] [0] [0.0035]'
        print '0 = Background level'
        print '0.0035 = adjustment threshold - max percentage of voxels that can be clipped'
    else:
        outFile = str(sys.argv[1])
        if (len(sys.argv) > 2):
            outFile = str(sys.argv[2])
        BG = 0
        if (len(sys.argv) > 3):
            BG = int(sys.argv[3])
        TH = adjust_thresh
        if (len(sys.argv) > 4):
            TH = float(sys.argv[4])
        print 'Loading input file %s...' % (str(sys.argv[1]))
        data, header = nrrd.read(str(sys.argv[1]))
        chan, Nbound, hist = AutoBalance(data, threshold=TH, background=BG)
        print 'New boundaries %s' % (str(Nbound))
        print 'Saving result to %s...' % (outFile)
        nrrd.write(outFile, np.uint8(chan), options=header)
<|endoftext|>


<commit_before>from flask import render_template
from flask_mail import Message
from app import mail
from decorators import async
from config import ADMINS

@async
def send_async_email(msg):
    mail.send(msg)

def send_email(subject, sender, recipients, text_body, html_body):
    msg = Message(subject, sender = sender, recipients = recipients)
    msg.body = text_body
    msg.html = html_body
    send_async_email(msg)

def send_emails(subject, text_body, recipient):
    recipients_list = []
    recipients_list.append(recipient)
    msg = Message(subject, sender = ADMINS[0], recipients = recipients_list)
    msg.body = text_body
    send_async_email(msg)

def follower_notification(followed, follower):
    send_email("[microblog] %s is now following you!" % follower.nickname,
        ADMINS[0],
        [followed.email],
        render_template("follower_email.txt",
            user = followed, follower = follower),
        render_template("follower_email.html",
            user = followed, follower = follower))

<commit_msg>Fixed send_email out of app context issue<commit_after>from flask import render_template, current_app
from flask_mail import Message
from app import mail
from decorators import async
from config import ADMINS

@async
def send_async_email(app, msg):
    with app.app_context():
        mail.send(msg)

def send_email(subject, sender, recipients, text_body, html_body):
    app = current_app._get_current_object()
    msg = Message(subject, sender=sender, recipients=recipients)
    msg.body = text_body
    msg.html = html_body
    send_async_email(app, msg)

def send_emails(subject, text_body, recipient):
    app = current_app._get_current_object()
    recipients_list = list()
    recipients_list.append(recipient)
    msg = Message(subject, sender=ADMINS[0], recipients=recipients_list)
    msg.body = text_body
    send_async_email(app, msg)

def follower_notification(followed, follower):
    send_email(
        "[microblog] %s is now following you!" % follower.nickname,
        ADMINS[0],
        [followed.email],
        render_template("follower_email.txt", user=followed, follower=follower),
        render_template("follower_email.html", user=followed, follower=follower)
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright © 2009-2011 Alexander Kojevnikov <alexander@kojevnikov.com>
#
# muspy is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# muspy is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with muspy.  If not, see <http://www.gnu.org/licenses/>.

import logging
import re
import StringIO
from urllib2 import Request, urlopen

from PIL import Image

from app import lastfm
from app.cover import Cover
from app.models import *
import app.musicbrainz as mb
from app.tools import str_to_date
from daemon import tools

def process():
    """Work on pending jobs."""
    while True:
        try:
            job = Job.objects.select_related('user').order_by('id')[0]
        except IndexError:
            break

        if job.type == Job.ADD_ARTIST:
            if not add_artist(job.user, job.data):
                tools.sleep()
                continue
        elif job.type == Job.ADD_RELEASE_GROUPS:
            if not add_release_groups(job.data):
                tools.sleep()
                continue
        elif job.type == Job.GET_COVER:
            get_cover(job.data)
        elif job.type == Job.IMPORT_LASTFM:
            count, username = job.data.split(',', 1)
            import_lastfm(job.user, username, int(count))

        job.delete()

def add_artist(user, search):
    tools.sleep()
    logging.info('[JOB] Searching for artist [%s] for user %d' % (search, user.id))
    found_artists, count = mb.search_artists(search, limit=2, offset=0)
    if found_artists is None:
        logging.warning('[ERR] MusicBrainz error while searching, retrying')
        return False

    only_one = len(found_artists) == 1
    first_is_exact = (len(found_artists) > 1 and
                      found_artists[0]['name'].lower() == search.lower() and
                      found_artists[1]['name'].lower() != search.lower())
    if only_one or first_is_exact:
        artist_data = found_artists[0]
        mbid = artist_data['id']

        # get_by_mbid() queries MB, must sleep.
        tools.sleep()
        logging.info('[JOB] Adding artist %s' % mbid)
        try:
            artist = Artist.get_by_mbid(mbid)
        except Artist.Blacklisted:
            logging.warning('[ERR] Artist %s is blacklisted, skipping' % mbid)
            return True
        except Artist.Unknown:
            logging.warning('[ERR] Artist %s is unknown, skipping' % mbid)
            return True
        if not artist:
            logging.warning('[ERR] Could not fetch artist %s, retrying' % mbid)
            return False
        UserArtist.add(user, artist)
    else:
        logging.info('[JOB] Could not identify artist by name, saving for later')
        UserSearch(user=user, search=search).save()

    return True

def add_release_groups(mbid):
    logging.info('[JOB] Fetching release groups for artist %s' % mbid)
    try:
        artist = Artist.objects.get(mbid=mbid)
    except Artist.DoesNotExist:
        logging.warning('[ERR] Cannot find by mbid, skipping' % mbid)
        return True

    LIMIT = 100
    offset = 0
    while True:
        tools.sleep()
        logging.info('[JOB] Fetching release groups at offset %d' % offset)
        release_groups = mb.get_release_groups(mbid, limit=LIMIT, offset=offset)
        if release_groups:
            with transaction.commit_on_success():
                for rg_data in release_groups:
                    # Ignoring releases without a release date or a type.
                    if rg_data.get('first-release-date') and rg_data.get('type'):
                        if ReleaseGroup.objects.filter(mbid=rg_data['id']).exists():
                            continue
                        release_group = ReleaseGroup(
                            artist=artist,
                            mbid=rg_data['id'],
                            name=rg_data['title'],
                            type=rg_data['type'],
                            date=str_to_date(rg_data['first-release-date']),
                            is_deleted=False)
                        release_group.save()
        if release_groups is None:
            logging.warning('[ERR] MusicBrainz error, retrying')
            continue
        if len(release_groups) < LIMIT:
            break
        offset += LIMIT

    return True

def get_cover(mbid):
    logging.info('[JOB] Trying to find a cover for %s' % mbid)
    tools.sleep()
    logging.info('[JOB] Get releases')
    releases = mb.get_releases(mbid, limit=100, offset=0)
    if releases is None:
        logging.warning('[ERR] Could not get releases, skipping')
        return
    releases = [r for r in releases if r.get('date')]

    # Order releases by date.
    def by_date(a, b):
        # Convert 2011 to 2011-99-99 and 2011-01 to 2011-01-99.
        d1, d2 = a['date'], b['date']
        while len(d1) < 10: d1 += '-99'
        while len(d2) < 10: d2 += '-99'
        return cmp(d1, d2)
    releases = sorted(releases, cmp=by_date)

    # We don't want to check all 100 releases.
    releases = [r['id'] for r in releases][:10]

    url = None
    for release in releases:
        tools.sleep()
        logging.info('[JOB] Checking release %s' % release)
        try:
            request = Request(
                'http://musicbrainz.org/release/' + release,
                headers = {'User-Agent': 'muspy/2.0'})
            response = urlopen(request)
            html = response.read()
        except:
            logging.warning('[ERR] Could not fetch the release page, skipping')
            continue

        # Parsing the release page
        pattern = r'
\s*<img src="(?P<url>[^"]+)"'
        match = re.search(pattern, html)
        if not match:
            logging.info('[JOB] No cover art, skipping')
            continue
        url = match.group('url')
        if _fetch_cover(mbid, url):
            return

    logging.info('[JOB] Try to get cover from Last.fm')
    rg = ReleaseGroup.objects.filter(mbid=mbid).select_related('artist').get()
    urls = lastfm.get_cover_urls(rg.artist.name, rg.name) or []
    for url in urls:
        if _fetch_cover(mbid, url):
            return

    logging.warning('[ERR] Could not find a cover')

def _fetch_cover(mbid, url):
    logging.info('[JOB] Downloading the cover')
    image = None
    try:
        request = Request(url, headers = {'User-Agent': 'muspy/2.0'})
        response = urlopen(request)
        image = response.read()
    except:
        logging.warning('[ERR] Could not download, skipping')
        return False

    # Sometimes we get just a one-pixel image, avoid resizing it.
    if len(image) < 4096:
        logging.warning('[ERR] Bad image, skipping')
        return False

    logging.info('[JOB] Saving the cover')
    try:
        im = Image.open(StringIO.StringIO(image))
        im = im.resize((120, 120), Image.ANTIALIAS)
        f = StringIO.StringIO()
        im.save(f, 'JPEG', quality=95)
        image = f.getvalue()
        Cover(mbid, image)
        return True
    except:
        logging.warning('[ERR] Could not save the cover, skipping')
        return False

def import_lastfm(user, username, count):
    logging.info('[JOB] Importing %d artists from Last.fm for user %s' % (count, username))
    LIMIT = 50
    page, added = 0, 0
    while True:
        page += 1
        tools.sleep()
        logging.info('[JOB] Getting page %d' % page)
        artists = lastfm.get_artists(username, LIMIT, page)

        if artists is None:
            logging.warning('[ERR] Last.fm error, retrying')
            continue

        for artist_data in artists:
            mbid = artist_data.get('mbid', '')
            if mbid:
                while True:
                    # Artist.get_by_mbid will query MB if the artist is not yet
                    # in the database. Query first to avoid unnecessary sleep.
                    if not Artist.objects.filter(mbid=mbid).exists():
                        tools.sleep()
                    logging.info('[JOB] Getting artist %s' % mbid)
                    try:
                        artist = Artist.get_by_mbid(mbid)
                    except Artist.Blacklisted:
                        logging.info('[JOB] Blacklisted artist, skipping')
                        break
                    except Artist.Unknown:
                        logging.info('[JOB] Unknown artist, skipping')
                        break
                    if not artist:
                        logging.warning('[ERR] Cannot get the artist data, retrying')
                        continue
                    UserArtist.add(user, artist)
                    break
            else:
                add_artists(user, artist_data['name'])
            added += 1
            if added == count: break

        if added == count: break
<commit_msg>daemon: Fix page increment when importing<commit_after># -*- coding: utf-8 -*-
#
# Copyright © 2009-2011 Alexander Kojevnikov <alexander@kojevnikov.com>
#
# muspy is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# muspy is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with muspy.  If not, see <http://www.gnu.org/licenses/>.

import logging
import re
import StringIO
from urllib2 import Request, urlopen

from PIL import Image

from app import lastfm
from app.cover import Cover
from app.models import *
import app.musicbrainz as mb
from app.tools import str_to_date
from daemon import tools

def process():
    """Work on pending jobs."""
    while True:
        try:
            job = Job.objects.select_related('user').order_by('id')[0]
        except IndexError:
            break

        if job.type == Job.ADD_ARTIST:
            if not add_artist(job.user, job.data):
                tools.sleep()
                continue
        elif job.type == Job.ADD_RELEASE_GROUPS:
            if not add_release_groups(job.data):
                tools.sleep()
                continue
        elif job.type == Job.GET_COVER:
            get_cover(job.data)
        elif job.type == Job.IMPORT_LASTFM:
            count, username = job.data.split(',', 1)
            import_lastfm(job.user, username, int(count))

        job.delete()

def add_artist(user, search):
    tools.sleep()
    logging.info('[JOB] Searching for artist [%s] for user %d' % (search, user.id))
    found_artists, count = mb.search_artists(search, limit=2, offset=0)
    if found_artists is None:
        logging.warning('[ERR] MusicBrainz error while searching, retrying')
        return False

    only_one = len(found_artists) == 1
    first_is_exact = (len(found_artists) > 1 and
                      found_artists[0]['name'].lower() == search.lower() and
                      found_artists[1]['name'].lower() != search.lower())
    if only_one or first_is_exact:
        artist_data = found_artists[0]
        mbid = artist_data['id']

        # get_by_mbid() queries MB, must sleep.
        tools.sleep()
        logging.info('[JOB] Adding artist %s' % mbid)
        try:
            artist = Artist.get_by_mbid(mbid)
        except Artist.Blacklisted:
            logging.warning('[ERR] Artist %s is blacklisted, skipping' % mbid)
            return True
        except Artist.Unknown:
            logging.warning('[ERR] Artist %s is unknown, skipping' % mbid)
            return True
        if not artist:
            logging.warning('[ERR] Could not fetch artist %s, retrying' % mbid)
            return False
        UserArtist.add(user, artist)
    else:
        logging.info('[JOB] Could not identify artist by name, saving for later')
        UserSearch(user=user, search=search).save()

    return True

def add_release_groups(mbid):
    logging.info('[JOB] Fetching release groups for artist %s' % mbid)
    try:
        artist = Artist.objects.get(mbid=mbid)
    except Artist.DoesNotExist:
        logging.warning('[ERR] Cannot find by mbid, skipping' % mbid)
        return True

    LIMIT = 100
    offset = 0
    while True:
        tools.sleep()
        logging.info('[JOB] Fetching release groups at offset %d' % offset)
        release_groups = mb.get_release_groups(mbid, limit=LIMIT, offset=offset)
        if release_groups:
            with transaction.commit_on_success():
                for rg_data in release_groups:
                    # Ignoring releases without a release date or a type.
                    if rg_data.get('first-release-date') and rg_data.get('type'):
                        if ReleaseGroup.objects.filter(mbid=rg_data['id']).exists():
                            continue
                        release_group = ReleaseGroup(
                            artist=artist,
                            mbid=rg_data['id'],
                            name=rg_data['title'],
                            type=rg_data['type'],
                            date=str_to_date(rg_data['first-release-date']),
                            is_deleted=False)
                        release_group.save()
        if release_groups is None:
            logging.warning('[ERR] MusicBrainz error, retrying')
            continue
        if len(release_groups) < LIMIT:
            break
        offset += LIMIT

    return True

def get_cover(mbid):
    logging.info('[JOB] Trying to find a cover for %s' % mbid)
    tools.sleep()
    logging.info('[JOB] Get releases')
    releases = mb.get_releases(mbid, limit=100, offset=0)
    if releases is None:
        logging.warning('[ERR] Could not get releases, skipping')
        return
    releases = [r for r in releases if r.get('date')]

    # Order releases by date.
    def by_date(a, b):
        # Convert 2011 to 2011-99-99 and 2011-01 to 2011-01-99.
        d1, d2 = a['date'], b['date']
        while len(d1) < 10: d1 += '-99'
        while len(d2) < 10: d2 += '-99'
        return cmp(d1, d2)
    releases = sorted(releases, cmp=by_date)

    # We don't want to check all 100 releases.
    releases = [r['id'] for r in releases][:10]

    url = None
    for release in releases:
        tools.sleep()
        logging.info('[JOB] Checking release %s' % release)
        try:
            request = Request(
                'http://musicbrainz.org/release/' + release,
                headers = {'User-Agent': 'muspy/2.0'})
            response = urlopen(request)
            html = response.read()
        except:
            logging.warning('[ERR] Could not fetch the release page, skipping')
            continue

        # Parsing the release page
        pattern = r'
\s*<img src="(?P<url>[^"]+)"'
        match = re.search(pattern, html)
        if not match:
            logging.info('[JOB] No cover art, skipping')
            continue
        url = match.group('url')
        if _fetch_cover(mbid, url):
            return

    logging.info('[JOB] Try to get cover from Last.fm')
    rg = ReleaseGroup.objects.filter(mbid=mbid).select_related('artist').get()
    urls = lastfm.get_cover_urls(rg.artist.name, rg.name) or []
    for url in urls:
        if _fetch_cover(mbid, url):
            return

    logging.warning('[ERR] Could not find a cover')

def _fetch_cover(mbid, url):
    logging.info('[JOB] Downloading the cover')
    image = None
    try:
        request = Request(url, headers = {'User-Agent': 'muspy/2.0'})
        response = urlopen(request)
        image = response.read()
    except:
        logging.warning('[ERR] Could not download, skipping')
        return False

    # Sometimes we get just a one-pixel image, avoid resizing it.
    if len(image) < 4096:
        logging.warning('[ERR] Bad image, skipping')
        return False

    logging.info('[JOB] Saving the cover')
    try:
        im = Image.open(StringIO.StringIO(image))
        im = im.resize((120, 120), Image.ANTIALIAS)
        f = StringIO.StringIO()
        im.save(f, 'JPEG', quality=95)
        image = f.getvalue()
        Cover(mbid, image)
        return True
    except:
        logging.warning('[ERR] Could not save the cover, skipping')
        return False

def import_lastfm(user, username, count):
    logging.info('[JOB] Importing %d artists from Last.fm for user %s' % (count, username))
    LIMIT = 50
    page, added = 0, 0
    while True:
        page += 1
        tools.sleep()
        logging.info('[JOB] Getting page %d' % page)
        artists = lastfm.get_artists(username, LIMIT, page)

        if artists is None:
            logging.warning('[ERR] Last.fm error, retrying')
            page -= 1
            continue

        for artist_data in artists:
            mbid = artist_data.get('mbid', '')
            if mbid:
                while True:
                    # Artist.get_by_mbid will query MB if the artist is not yet
                    # in the database. Query first to avoid unnecessary sleep.
                    if not Artist.objects.filter(mbid=mbid).exists():
                        tools.sleep()
                    logging.info('[JOB] Getting artist %s' % mbid)
                    try:
                        artist = Artist.get_by_mbid(mbid)
                    except Artist.Blacklisted:
                        logging.info('[JOB] Blacklisted artist, skipping')
                        break
                    except Artist.Unknown:
                        logging.info('[JOB] Unknown artist, skipping')
                        break
                    if not artist:
                        logging.warning('[ERR] Cannot get the artist data, retrying')
                        continue
                    UserArtist.add(user, artist)
                    break
            else:
                add_artists(user, artist_data['name'])
            added += 1
            if added == count: break

        if added == count: break
<|endoftext|>


<commit_before>###################################################################################################
#
# time_discrepancy_finder.py
#   Attempts to find discrepancies between server-side and local timestamps
#
# References:
#   Lee Whitfield (http://forensic4cast.com/2011/07/flashpost-google-plus-artefacts-url-forwarding/)
#   Vincent Toubiana and Helen Nissenbaum (http://repository.cmu.edu/cgi/viewcontent.cgi?article=1058&context=jpc)
#
# Blog post explaining plugin:
#   http://www.obsidianforensics.com/blog/detecting-clock-changes-using-cookies/
#
# Plugin Author: Ryan Benson (ryan@obsidianforensics.com)
#
###################################################################################################

import re

# Config
friendlyName = "Time Discrepancy Finder"
description = "Attempts to find discrepancies between server-side and local timestamps"
artifactTypes = ("cookie (created)", "url")
remoteLookups = 0
browser = "any"
browserVersion = 1
version = "20141214"
parsedItems = 0

def plugin(target_browser):

    global parsedItems
    cookie_set = [
        # website                   # cookie name           # regex for timestamp
        {'.pubmatic.com/':          {'name': '_curtime',    'regex': r'(\d{10})'}},
        {'.invitemedia.com/':       {'name': 'dp_rec',      'regex': r':(\d{10})}'}},
        {'.atdmt.com/':             {'name': 'AA002',       'regex': r'^(\d{10})-'}},
        {'.mathtag.com/':           {'name': 'mt_mop',      'regex': r'4:(\d{10})'}},
        {'.33across.com/':          {'name': '33x_ps',      'regex': r'ts%3D(\d{13})'}},
        {'.twitter.com/':           {'name': 'guest_id',    'regex': r'v1%3A(\d{13})'}},
        {'.twitter.com/':           {'name': 'pid',         'regex': r'v3:(\d{13})'}},
        {'www.yahoo.com/':          {'name': 'FBJSC',       'regex': r'(\d{10})'}},
        {'.scorecardresearch.com/': {'name': 'UIDR',        'regex': r'(\d{10})'}},
        {'.rubiconproject.com/':    {'name': 'put_3076',    'regex': r'(\d{10})'}},
        {'.mookie1.com/':           {'name': 'mdata',       'regex': r'|(\d{10})$'}},
        {'*':                       {'name': 'PREF',        'regex': r'LM=(\d{10})'}},
        {'*':                       {'name': '__gads',      'regex': r'T=(1\d{9})'}},
        {'.cbsnews.com/':           {'name': 'apexLat',     'regex': r'^(1\d{12})$'}},
        {'.adblade.com/':           {'name': '__impt',      'regex': r'^(1\d{9})$'}},
        {'.wtp101.com/':            {'name': 'cookie_born', 'regex': r'^(1\d{9})$'}},
        {'.advertising.com/':       {'name': 'ACID',        'regex': r'^.{6}(1\d{9})'}},
        {'.bidswitch.net/':         {'name': 'c',           'regex': r'^(1\d{9})$'}},
        {'*':                       {'name': '__cfduid',    'regex': r'(1\d{9})$'}},
        {'.insightexpressai.com/':  {'name': 'DW_Time',     'regex': r'^(1\d{9})$'}},
        {'www.bose.com/':           {'name': 'bose_id',     'regex': r'^(1\d{12})'}},
        {'.liveperson.net/':        {'name': 'LivePersonID','regex': r'd=(1\d{9})'}}
    ]

    url_set = [
        # regex for url and timestamp
        r'google\..*&n=(\d{13})'
    ]

    for item in target_browser.parsed_artifacts:
        if item.row_type.startswith(artifactTypes):
            if item.row_type == u'cookie (created)':
                for site in cookie_set:
                    if item.url in site or site.keys()[0] == '*':
                        if site[site.keys()[0]]['name'] == item.name:
                            m = re.search(site[site.keys()[0]]['regex'], item.value)
                            if m:
                                server = target_browser.to_datetime(m.group(1))
                                local = item.timestamp
                                delta = abs(server - local)
                                item.interpretation = u'Server-side Timestamp: {} | Local Timestamp: {} | ' \
                                                      u'Difference: {} [Time Discrepancy]'.format(server, local, delta)
                                parsedItems += 1
            elif item.row_type == u'url' or item.row_type == u'url (archived)':
                for site in url_set:
                    m = re.search(site, item.url)
                    if m:
                        server = target_browser.to_datetime(m.group(1))
                        local = item.timestamp
                        delta = abs(server - local)
                        item.interpretation = u'Server-side Timestamp: {} | Local Timestamp: {} | ' \
                                              u'Difference: {} [Time Discrepancy]'.format(server, local, delta)
                        parsedItems += 1

    # Description of what the plugin did
    return u"{} differences parsed".format(parsedItems)
<commit_msg>Updated to add a few new cookies and modify the regex on some existing ones.<commit_after>###################################################################################################
#
# time_discrepancy_finder.py
#   Attempts to find discrepancies between server-side and local timestamps
#
# References:
#   Lee Whitfield (http://forensic4cast.com/2011/07/flashpost-google-plus-artefacts-url-forwarding/)
#   Vincent Toubiana and Helen Nissenbaum (http://repository.cmu.edu/cgi/viewcontent.cgi?article=1058&context=jpc)
#
# Blog post explaining plugin:
#   http://www.obsidianforensics.com/blog/detecting-clock-changes-using-cookies/
#
# Plugin Author: Ryan Benson (ryan@obsidianforensics.com)
#
###################################################################################################

import re

# Config
friendlyName = "Time Discrepancy Finder"
description = "Attempts to find discrepancies between server-side and local timestamps"
artifactTypes = ("cookie (created)", "url")
remoteLookups = 0
browser = "any"
browserVersion = 1
version = "20160707"
parsedItems = 0

def plugin(target_browser):

    global parsedItems
    cookie_set = [
        # website                   # cookie name            # regex for timestamp
        {'.pubmatic.com/':          {'name': '_curtime',     'regex': r'(\d{10})'}},
        {'.invitemedia.com/':       {'name': 'dp_rec',       'regex': r':(\d{10})}'}},
        {'.atdmt.com/':             {'name': 'AA002',        'regex': r'^(\d{10})-'}},
        {'.mathtag.com/':           {'name': 'mt_mop',       'regex': r'4:(\d{10})'}},
        {'.33across.com/':          {'name': '33x_ps',       'regex': r'ts%3D(\d{13})'}},
        {'.twitter.com/':           {'name': 'guest_id',     'regex': r'v1%3A(\d{13})'}},
        {'.twitter.com/':           {'name': 'pid',          'regex': r'v3:(\d{13})'}},
        {'www.yahoo.com/':          {'name': 'FBJSC',        'regex': r'(\d{10})'}},
        {'.scorecardresearch.com/': {'name': 'UIDR',         'regex': r'(\d{10})'}},
        {'.rubiconproject.com/':    {'name': 'put_3076',     'regex': r'(\d{10})'}},
        {'.mookie1.com/':           {'name': 'mdata',        'regex': r'\|(\d{10})$'}},
        {'*':                       {'name': 'PREF',         'regex': r'LM=(\d{10})'}},
        {'*':                       {'name': '__gads',       'regex': r'T=(1\d{9})'}},
        {'.cbsnews.com/':           {'name': 'apexLat',      'regex': r'^(1\d{12})$'}},
        {'.adblade.com/':           {'name': '__impt',       'regex': r'^(1\d{9})$'}},
        {'.wtp101.com/':            {'name': 'cookie_born',  'regex': r'^(1\d{9})$'}},
        {'.advertising.com/':       {'name': 'ACID',         'regex': r'^.{6}(1\d{9})'}},
        {'.bidswitch.net/':         {'name': 'c',            'regex': r'^(1\d{9})$'}},
        {'*':                       {'name': '__cfduid',     'regex': r'(1\d{9})$'}},
        {'.insightexpressai.com/':  {'name': 'DW_Time',      'regex': r'^(1\d{9})$'}},
        {'www.bose.com/':           {'name': 'bose_id',      'regex': r'^(1\d{12})'}},
        {'.liveperson.net/':        {'name': 'LivePersonID', 'regex': r'd=(1\d{9})'}},
        {'.baidu.com/':             {'name': 'PSTM',         'regex': r'^(1\d{9})$'}},
        {'.alexa.com/':             {'name': 'lv',           'regex': r'(1\d{9})$'}},
        {'.doubleclick.net/':       {'name': 'id',           'regex': r'\|t=(1\d{9})\|'}}
    ]

    url_set = [
        # regex for url and timestamp
        r'google\..*&n=(\d{13})'
    ]

    for item in target_browser.parsed_artifacts:
        if item.row_type.startswith(artifactTypes):
            if item.row_type == u'cookie (created)':
                for site in cookie_set:
                    if item.url in site or site.keys()[0] == '*':
                        if site[site.keys()[0]]['name'] == item.name:
                            m = re.search(site[site.keys()[0]]['regex'], item.value)
                            if m:
                                server = target_browser.to_datetime(m.group(1))
                                local = item.timestamp
                                delta = abs(server - local)
                                item.interpretation = u'Server-side Timestamp: {} | Local Timestamp: {} | ' \
                                                      u'Difference: {} [Time Discrepancy]'.format(server, local, delta)
                                parsedItems += 1
            elif item.row_type == u'url' or item.row_type == u'url (archived)':
                for site in url_set:
                    m = re.search(site, item.url)
                    if m:
                        server = target_browser.to_datetime(m.group(1))
                        local = item.timestamp
                        delta = abs(server - local)
                        item.interpretation = u'Server-side Timestamp: {} | Local Timestamp: {} | ' \
                                              u'Difference: {} [Time Discrepancy]'.format(server, local, delta)
                        parsedItems += 1

    # Description of what the plugin did
    return u"{} differences parsed".format(parsedItems)
<|endoftext|>


<commit_before>import pytest
from django_alexa.internal import validation

class TestVerifyCertUrl:
    def test_valid_url_1(self):
        url = "https://s3.amazonaws.com/echo.api/test"
        r = validation.verify_cert_url(url)
        assert r == True, 'Should be a valid url without port'

    def test_valid_url_2(self):
        url = "https://s3.amazonaws.com:443/echo.api/test"
        r = validation.verify_cert_url(url)
        assert r == True, 'Should be a valid url with port'

    def test_invalid_url_1(self):
        url = "this is the url"
        r = validation.verify_cert_url(url)
        assert r == False, 'Should be an invalid url'

    def test_invalid_url_2(self):
        url = None
        r = validation.verify_cert_url(url)
        assert r == False, 'Should be an invalid url, given None'<commit_msg>added some tests for internal/validation.py<commit_after>import pytest
from django_alexa.internal import validation

class TestValidateResponseLimit:
    def test_validate_response_limit_fail(self):
        with pytest.raises(Exception) as exc_info:
            value = 'x' * 1000 * 1000 * 24 + 'x'
            validation.validate_response_limit(value)
            msg = "Should raise exception, response > 24 kb"
        assert "InternalError" in str(exc_info), msg

    def test_validate_response_limit_pass(self):
        value = 'x' * 1000 * 1000 * 24
        validation.validate_response_limit(value)

class TestValidateAppIds:
    def test_validate_app_ids_fail(self):
        with pytest.raises(Exception) as exc_info:
            app_id = 'non_valid_id_example'
            validation.validate_app_ids(app_id)
            msg = "Should raise exception, app id not valid"
        assert "InternalError" in str(exc_info), msg

    def test_validate_app_ids_pass(self):
        pass

class TestVerifyCertUrl:
    def test_valid_url_1(self):
        url = "https://s3.amazonaws.com/echo.api/test"
        assert validation.verify_cert_url(url), 'Should be a valid url without port'

    def test_valid_url_2(self):
        url = "https://s3.amazonaws.com:443/echo.api/test"
        assert validation.verify_cert_url(url), 'Should be a valid url with port'

    def test_invalid_url_1(self):
        url = " "
        assert not validation.verify_cert_url(url), 'Should be an invalid url'

    def test_invalid_url_2(self):
        url = None
        assert not validation.verify_cert_url(url), 'Should be an invalid url, given None'<|endoftext|>


<commit_before>'''
Created on Oct 6, 2014

@author: Aaron
'''
import urllib2
from bs4 import BeautifulSoup
import sys
import re
from time import sleep

import markup
import retry_decorator

reload(sys)
sys.setdefaultencoding("utf-8") #@UndefinedVariable

team_abbrvs = ['ANA', 'ARI', 'BOS', 'BUF', 'CAR', 'CBJ', 'CGY', 'CHI', 'COL', 'DAL',
              'DET', 'EDM', 'FLA', 'LA', 'MIN', 'MTL', 'NSH', 'NJ', 'NYI', 'NYR',
              'OTT', 'PHI', 'PIT', 'SJ', 'STL', 'TB', 'TOR', 'VAN', 'WPG', 'WSH']

team_names = ['Ducks', 'Coyotes', 'Bruins', 'Sabres', 'Hurricanes', 'Blue Jackets',
              'Flames', 'Blackhawks', 'Avalanche', 'Stars','Red Wings', 'Oilers',
              'Panthers', 'Kings', 'Wild', 'Canadiens', 'Predators', 'Devils',
              'Islanders', 'Rangers', 'Senators', 'Flyers', 'Penguins', 'Sharks',
              'Blues', 'Lightning', 'Maple Leafs', 'Canucks', 'Jets', 'Capitals']

class GameData(object):
   def __init__(self, link, headline, date):
      self.link = link
      self.headline = headline
      self.date = date
      self.result = None

   def char_convert_link(self):
      self.link = re.sub('[&]', '&', self.link)

   def print_game_data(self):
      print self.headline, self.link, self.date, self.result

   def list_data(self):
      return [self.headline, self.link, self.date, self.result]

   def modify_headline(self):
      try:
         if self.date[4] == "0":
            self.headline = "[" + self.date[5:6] + "/" + self.date[6:] + "] " + self.result + " - " + self.headline
         else:
            self.headline = "[" + self.date[4:6] + "/" + self.date[6:] + "] " + self.result + " - " + self.headline
      except TypeError:
         self.headline = "[" + self.date[4:6] + "/" + self.date[6:] + "] " + self.result + "Error finding headline"

   def find_winner(self, team_name):
      soup = page_response(self.link)

      home_team = ""
      home_score = 0
      away_score = 0
      result = None

      for div in soup.find_all(attrs={"class" : "matchup"}):
         for home in div.find_all(attrs={"class" : "team home"}):
            for name in home.find('a'):
               home_team = str(name)
            for score in home.find_all(attrs={"class" : "gp-homeScore"}):
               for val in score:
                  home_score = int(val)

         for away in div.find_all(attrs={"class" : "team away"}):
            for score in away.find_all(attrs={"class" : "gp-awayScore"}):
               for val in score:
                  away_score = int(val)

         home = home_team == team_name

         if home_score > away_score and home:
            result = "W"
         elif home_score < away_score and home:
            result = "L"
         elif home_score > away_score:
            result = "L"
         elif home_score < away_score:
            result = "W"
         else:
            result = "error determining winner"

      self.result = result

def page_response(link):
   response = urlopen_with_retry(link)
   page_source = response.read()

   return BeautifulSoup(page_source)

def extract_game_data(team_ab, team_name):
   games = []
   link = "http://espn.go.com/nhl/team/schedule/_/name/" + team_ab
   soup = page_response(link)

   for div in soup.find_all(attrs={"class" : "score"}):
      complete_link = "http://espn.go.com" + str(div.find('a').get('href').encode('utf-8', 'ignore'))

      new_game = GameData(complete_link, get_game_headline(complete_link), get_game_date(complete_link))

      new_game.char_convert_link()
      new_game.find_winner(team_name)
      new_game.modify_headline()

      games.append(new_game)

   return games

@retry_decorator.retry(urllib2.URLError, tries=4, delay=5, backoff=2)
def urlopen_with_retry(link):
   return urllib2.urlopen(link)

def get_game_headline(link):
   try:
      soup = page_response(link)

      for meta in soup.findAll('meta', {"property":'og:title'}):
         return meta.get('content')
   except urllib2.HTTPError:
      print 'There was an error with the request'

def get_game_date(link):
   soup = page_response(link)

   for div in soup.find_all(attrs={"class" : "scoreboard-strip-wrapper"}):
      #print div
      date_string = str(div.find('a').get('href').encode('utf-8', 'ignore'))
      #print date_string
      return date_string[21:]

def main():
   for team_ab, team_name in zip(team_abbrvs, team_names):
      games = extract_game_data(team_ab, team_name)
      games.sort(key=lambda x: x.date, reverse=True)

      for game in games:
         print game.list_data()
      markup.xml_markup(games, team_ab, team_name)

      print "Completed " + team_name

main()<commit_msg>Added game resultt o headline<commit_after>'''
Created on Oct 6, 2014

@author: Aaron
'''
import urllib2
from bs4 import BeautifulSoup
import sys
import re

import markup
import retry_decorator

reload(sys)
sys.setdefaultencoding("utf-8") #@UndefinedVariable

team_abbrvs = ['ANA', 'ARI', 'BOS', 'BUF', 'CAR', 'CBJ', 'CGY', 'CHI', 'COL', 'DAL',
              'DET', 'EDM', 'FLA', 'LA', 'MIN', 'MTL', 'NSH', 'NJ', 'NYI', 'NYR',
              'OTT', 'PHI', 'PIT', 'SJ', 'STL', 'TB', 'TOR', 'VAN', 'WPG', 'WSH']

team_names = ['Ducks', 'Coyotes', 'Bruins', 'Sabres', 'Hurricanes', 'Blue Jackets',
              'Flames', 'Blackhawks', 'Avalanche', 'Stars','Red Wings', 'Oilers',
              'Panthers', 'Kings', 'Wild', 'Canadiens', 'Predators', 'Devils',
              'Islanders', 'Rangers', 'Senators', 'Flyers', 'Penguins', 'Sharks',
              'Blues', 'Lightning', 'Maple Leafs', 'Canucks', 'Jets', 'Capitals']

class GameData(object):
   def __init__(self, link, headline, date):
      self.link = link
      self.headline = headline
      self.date = date
      self.result = None

   def char_convert_link(self):
      self.link = re.sub('[&]', '&', self.link)

   def print_game_data(self):
      print self.headline, self.link, self.date, self.result

   def list_data(self):
      return [self.headline, self.link, self.date, self.result]

   def modify_headline(self):
      try:
         if self.date[4] == "0":
            self.headline = "[" + self.date[5:6] + "/" + self.date[6:] + "] " + self.result + " - " + self.headline
         else:
            self.headline = "[" + self.date[4:6] + "/" + self.date[6:] + "] " + self.result + " - " + self.headline
      except TypeError:
         self.headline = "[" + self.date[4:6] + "/" + self.date[6:] + "] " + self.result + "Error finding headline"

   def find_winner(self, team_name):
      soup = page_response(self.link)

      home_team = ""
      home_score = 0
      away_score = 0
      result = None

      for div in soup.find_all(attrs={"class" : "matchup"}):
         for home in div.find_all(attrs={"class" : "team home"}):
            for name in home.find('a'):
               home_team = str(name)
            for score in home.find_all(attrs={"class" : "gp-homeScore"}):
               for val in score:
                  home_score = int(val)

         for away in div.find_all(attrs={"class" : "team away"}):
            for score in away.find_all(attrs={"class" : "gp-awayScore"}):
               for val in score:
                  away_score = int(val)

         home = home_team == team_name

         if home_score > away_score and home:
            result = "W"
         elif home_score < away_score and home:
            result = "L"
         elif home_score > away_score:
            result = "L"
         elif home_score < away_score:
            result = "W"
         else:
            result = "error determining winner"

      self.result = result

def page_response(link):
   response = urlopen_with_retry(link)
   page_source = response.read()

   return BeautifulSoup(page_source)

def extract_game_data(team_ab, team_name):
   games = []
   link = "http://espn.go.com/nhl/team/schedule/_/name/" + team_ab
   soup = page_response(link)

   for div in soup.find_all(attrs={"class" : "score"}):
      complete_link = "http://espn.go.com" + str(div.find('a').get('href').encode('utf-8', 'ignore'))

      new_game = GameData(complete_link, get_game_headline(complete_link), get_game_date(complete_link))

      new_game.char_convert_link()
      new_game.find_winner(team_name)
      new_game.modify_headline()

      games.append(new_game)

   return games

@retry_decorator.retry(urllib2.URLError, tries=4, delay=5, backoff=2)
def urlopen_with_retry(link):
   return urllib2.urlopen(link)

def get_game_headline(link):
   try:
      soup = page_response(link)

      for meta in soup.findAll('meta', {"property":'og:title'}):
         return meta.get('content')
   except urllib2.HTTPError:
      print 'There was an error with the request'

def get_game_date(link):
   soup = page_response(link)

   for div in soup.find_all(attrs={"class" : "scoreboard-strip-wrapper"}):
      #print div
      date_string = str(div.find('a').get('href').encode('utf-8', 'ignore'))
      #print date_string
      return date_string[21:]

def main():
   for team_ab, team_name in zip(team_abbrvs, team_names):
      games = extract_game_data(team_ab, team_name)
      games.sort(key=lambda x: x.date, reverse=True)

      for game in games:
         print game.list_data()
      markup.xml_markup(games, team_ab, team_name)

      print "Completed " + team_name

main()<|endoftext|>


<commit_before>"""
logan.runner
~~~~~~~~~~~~

:copyright: (c) 2012 David Cramer.
:license: Apache License 2.0, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

from django.core import management
from optparse import OptionParser
import os
import re
import sys

from logan import importer
from logan.settings import create_default_settings

def sanitize_name(project):
    project = project.replace(' ', '-')
    return re.sub('[^A-Z0-9a-z_-]', '-', project)

def parse_args(args):
    """
    This parses the arguments and returns a tuple containing:

    (args, command, command_args)

    For example, "--config=bar start --with=baz" would return:

    (['--config=bar'], 'start', ['--with=baz'])
    """
    index = None
    for arg_i, arg in enumerate(args):
        if not arg.startswith('-'):
            index = arg_i
            break

    # Unable to parse any arguments
    if index is None:
        return (args, None, [])

    return (args[:index], args[index], args[(index + 1):])

def configure_app(config_path=None, project=None, default_config_path=None,
                  default_settings=None, settings_initializer=None,
                  settings_envvar=None, initializer=None, allow_extras=True,
                  config_module_name=None, runner_name=None):
    """
    :param project: should represent the canonical name for the project, generally
        the same name it assigned in distutils.
    :param default_config_path: the default location for the configuration file.
    :param default_settings: default settings to load (think inheritence).
    :param settings_initializer: a callback function which should return a string
        representing the default settings template to generate.
    :param initializer: a callback function which will be executed before the command
        is executed. It is passed a dictionary of various configuration attributes.
    """

    project_filename = sanitize_name(project)

    if default_config_path is None:
        default_config_path = '~/%s/%s.conf.py' % (project_filename, project_filename)

    if settings_envvar is None:
        settings_envvar = project_filename.upper() + '_CONF'

    if config_module_name is None:
        config_module_name = project_filename + '_config'

    # normalize path
    if settings_envvar in os.environ:
        default_config_path = os.environ.get(settings_envvar)
    else:
        default_config_path = os.path.normpath(os.path.abspath(os.path.expanduser(default_config_path)))

    if not config_path:
        config_path = default_config_path

    config_path = os.path.expanduser(config_path)

    if not os.path.exists(config_path):
        if runner_name:
            raise ValueError("Configuration file does not exist. Use '%s init' to initialize the file." % (runner_name,))
        raise ValueError("Configuration file does not exist at %r" % (config_path,))

    os.environ['DJANGO_SETTINGS_MODULE'] = config_module_name

    def settings_callback(settings):
        if initializer is None:
            return

        try:
            initializer({
                'project': project,
                'config_path': config_path,
                'settings': settings,
            })
        except Exception:
            # XXX: Django doesn't like various errors in this path
            import sys
            import traceback
            traceback.print_exc()
            sys.exit(1)

    importer.install(
        config_module_name, config_path, default_settings,
        allow_extras=allow_extras, callback=settings_callback)

    # HACK(dcramer): we need to force access of django.conf.settings to
    # ensure we don't hit any import-driven recursive behavior
    from django.conf import settings
    hasattr(settings, 'INSTALLED_APPS')

def run_app(**kwargs):
    sys_args = sys.argv

    # The established command for running this program
    runner_name = os.path.basename(sys_args[0])

    args, command, command_args = parse_args(sys_args[1:])

    if not command:
        print("usage: %s [--config=/path/to/settings.py] [command] [options]" % runner_name)
        sys.exit(1)

    default_config_path = kwargs.get('default_config_path')

    parser = OptionParser()

    # The ``init`` command is reserved for initializing configuration
    if command == 'init':
        (options, opt_args) = parser.parse_args()

        settings_initializer = kwargs.get('settings_initializer')

        config_path = os.path.expanduser(' '.join(opt_args[1:]) or default_config_path)

        if os.path.exists(config_path):
            resp = None
            while resp not in ('Y', 'n'):
                resp = raw_input('File already exists at %r, overwrite? [nY] ' % config_path)
                if resp == 'n':
                    print("Aborted!")
                    return

        try:
            create_default_settings(config_path, settings_initializer)
        except OSError as e:
            raise e.__class__('Unable to write default settings file to %r' % config_path)

        print("Configuration file created at %r" % config_path)

        return

    parser.add_option('--config', metavar='CONFIG')

    (options, logan_args) = parser.parse_args(args)

    config_path = options.config

    configure_app(config_path=config_path, **kwargs)

    management.execute_from_command_line([runner_name, command] + command_args)

    sys.exit(0)

if __name__ == '__main__':
    run_app()
<commit_msg>fix flake8 error<commit_after>"""
logan.runner
~~~~~~~~~~~~

:copyright: (c) 2012 David Cramer.
:license: Apache License 2.0, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

from django.core import management
from optparse import OptionParser
import os
import re
import sys

from logan import importer
from logan.settings import create_default_settings

try:
    raw_input
except NameError:  # PYthon 3
    raw_input = input

def sanitize_name(project):
    project = project.replace(' ', '-')
    return re.sub('[^A-Z0-9a-z_-]', '-', project)

def parse_args(args):
    """
    This parses the arguments and returns a tuple containing:

    (args, command, command_args)

    For example, "--config=bar start --with=baz" would return:

    (['--config=bar'], 'start', ['--with=baz'])
    """
    index = None
    for arg_i, arg in enumerate(args):
        if not arg.startswith('-'):
            index = arg_i
            break

    # Unable to parse any arguments
    if index is None:
        return (args, None, [])

    return (args[:index], args[index], args[(index + 1):])

def configure_app(config_path=None, project=None, default_config_path=None,
                  default_settings=None, settings_initializer=None,
                  settings_envvar=None, initializer=None, allow_extras=True,
                  config_module_name=None, runner_name=None):
    """
    :param project: should represent the canonical name for the project, generally
        the same name it assigned in distutils.
    :param default_config_path: the default location for the configuration file.
    :param default_settings: default settings to load (think inheritence).
    :param settings_initializer: a callback function which should return a string
        representing the default settings template to generate.
    :param initializer: a callback function which will be executed before the command
        is executed. It is passed a dictionary of various configuration attributes.
    """

    project_filename = sanitize_name(project)

    if default_config_path is None:
        default_config_path = '~/%s/%s.conf.py' % (project_filename, project_filename)

    if settings_envvar is None:
        settings_envvar = project_filename.upper() + '_CONF'

    if config_module_name is None:
        config_module_name = project_filename + '_config'

    # normalize path
    if settings_envvar in os.environ:
        default_config_path = os.environ.get(settings_envvar)
    else:
        default_config_path = os.path.normpath(os.path.abspath(os.path.expanduser(default_config_path)))

    if not config_path:
        config_path = default_config_path

    config_path = os.path.expanduser(config_path)

    if not os.path.exists(config_path):
        if runner_name:
            raise ValueError("Configuration file does not exist. Use '%s init' to initialize the file." % (runner_name,))
        raise ValueError("Configuration file does not exist at %r" % (config_path,))

    os.environ['DJANGO_SETTINGS_MODULE'] = config_module_name

    def settings_callback(settings):
        if initializer is None:
            return

        try:
            initializer({
                'project': project,
                'config_path': config_path,
                'settings': settings,
            })
        except Exception:
            # XXX: Django doesn't like various errors in this path
            import sys
            import traceback
            traceback.print_exc()
            sys.exit(1)

    importer.install(
        config_module_name, config_path, default_settings,
        allow_extras=allow_extras, callback=settings_callback)

    # HACK(dcramer): we need to force access of django.conf.settings to
    # ensure we don't hit any import-driven recursive behavior
    from django.conf import settings
    hasattr(settings, 'INSTALLED_APPS')

def run_app(**kwargs):
    sys_args = sys.argv

    # The established command for running this program
    runner_name = os.path.basename(sys_args[0])

    args, command, command_args = parse_args(sys_args[1:])

    if not command:
        print("usage: %s [--config=/path/to/settings.py] [command] [options]" % runner_name)
        sys.exit(1)

    default_config_path = kwargs.get('default_config_path')

    parser = OptionParser()

    # The ``init`` command is reserved for initializing configuration
    if command == 'init':
        (options, opt_args) = parser.parse_args()

        settings_initializer = kwargs.get('settings_initializer')

        config_path = os.path.expanduser(' '.join(opt_args[1:]) or default_config_path)

        if os.path.exists(config_path):
            resp = None
            while resp not in ('Y', 'n'):
                resp = raw_input('File already exists at %r, overwrite? [nY] ' % config_path)
                if resp == 'n':
                    print("Aborted!")
                    return

        try:
            create_default_settings(config_path, settings_initializer)
        except OSError as e:
            raise e.__class__('Unable to write default settings file to %r' % config_path)

        print("Configuration file created at %r" % config_path)

        return

    parser.add_option('--config', metavar='CONFIG')

    (options, logan_args) = parser.parse_args(args)

    config_path = options.config

    configure_app(config_path=config_path, **kwargs)

    management.execute_from_command_line([runner_name, command] + command_args)

    sys.exit(0)

if __name__ == '__main__':
    run_app()
<|endoftext|>


<commit_before>#    Copyright 2013, 2014 Joshua Charles Campbell
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import print_function
import os, zmq, signal, os.path, subprocess, fcntl, time
import errno
from unnaturalcode.unnaturalCode import *
from logging import debug, info, warning, error, getLogger
from multiprocessing import Process
from functools import wraps

allWhitespace = re.compile('^\s+$')

ucParanoid = os.getenv("PARANOID", False)

mitlmLogger = getLogger('MITLM')

CROSS_ENTROPY_PREFIX = 'x'
PREDICTION_PREFIX = 'p'

class mitlmCorpus(object):

    def __init__(self, readCorpus=None, writeCorpus=None, estimateNgramPath=None, uc=unnaturalCode(), order=10):
        self.readCorpus = (readCorpus or os.getenv("ucCorpus", "/tmp/ucCorpus"))
        self.writeCorpus = (writeCorpus or os.getenv("ucWriteCorpus", self.readCorpus))
        self.mitlmSocketPath = "ipc://%s/%s-%i-%i" % (os.path.dirname(self.readCorpus), "ucMitlmSocket", os.getpid(), id(self))
        self.estimateNgramPath = (estimateNgramPath or os.getenv("ESTIMATENGRAM", os.popen('which estimate-ngram').read()))
        self.corpusFile = False
        self.mitlmSocket = None
        self.mitlmProc = None
        self.order = order
        self.zctx = uc.zctx

    def checkMitlm(self):
        while self.mitlmProc:
            try:
                msg = self.mitlmProc.stdout.readline().rstrip('\n')
                if not allWhitespace.match(msg):
                  mitlmLogger.info()
            except:
                break

    def startMitlm(self):
        """Start MITLM estimate-ngram in 0MQ entropy query mode, unless already running."""
        if not self.mitlmSocket == None :
            assert not self.mitlmSocket.closed
            assert self.mitlmProc.poll() == None
            # Already running
            self.checkMitlm()
            return
        assert os.path.exists(self.readCorpus), "No such corpus."
        assert not allWhitespace.match(slurp(self.readCorpus)), "Corpus is full of whitespace!"
        assert os.path.exists(self.estimateNgramPath), "No such estimate-ngram."
        self.mitlmProc = subprocess.Popen([self.estimateNgramPath,
            "-t", self.readCorpus,
            "-o", str(self.order),
            "-s", "ModKN",
            "-u",
            "-live-prob", self.mitlmSocketPath], stdout=subprocess.PIPE)
        debug("Started MITLM as PID %i." % self.mitlmProc.pid)

        fd = self.mitlmProc.stdout.fileno()
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

        # Test the ZMQ connection.
        time.sleep(1)
        self.checkMitlm()
        self.mitlmSocket = self.zctx.socket(zmq.REQ)
        self.mitlmSocket.connect(self.mitlmSocketPath)
        self.checkMitlm()
        self.sendEntropyRequest(['for', '(', 'i', '='])
        self.checkMitlm()
        r = float(self.mitlmSocket.recv())
        debug("MITLM said %f" % r)
        self.checkMitlm()

    def stopMitlm(self):
        """Stop MITLM estimate-ngram, unless not running."""
        self.checkMitlm()
        if self.mitlmSocket:
            self.mitlmSocket.setsockopt(zmq.LINGER, 0)
            self.mitlmSocket.close()
            assert self.mitlmSocket.closed
            chill_rm(normalize_path(self.mitlmSocketPath))
            self.mitlmSocket = None
            self.mitlmSocketPath = None
        if self.mitlmProc:
            rc = None
            debug("Waiting for MITLM to shut down...")
            while self.mitlmProc and (rc == None):
                self.mitlmProc.terminate()
                time.sleep(0.1)
                rc = self.mitlmProc.poll()
            self.mitlmProc = None
            debug("MITLM exited with status %r" % (rc))

    def corpify(self, lexemes):
        """Stringify lexed source: produce space-seperated sequence of lexemes"""
        assert isinstance(lexemes, list)
        assert len(lexemes)
        return " ".join(lexemes)

    def openCorpus(self):
        """Opens the corpus (if necessary)"""
        if (self.corpusFile):
            assert not self.corpusFile.closed
            return
        self.corpusFile = open(self.writeCorpus, 'a')

    def closeCorpus(self):
        """Closes the corpus (if necessary)"""
        if (self.corpusFile):
            self.corpusFile.close()
            assert self.corpusFile.closed
            self.corpusFile = None

    def addToCorpus(self, lexemes):
        """Adds a string of lexemes to the corpus"""
        assert isinstance(lexemes, list)
        assert len(lexemes)
        self.openCorpus()
        cl = self.corpify(lexemes)
        #debug(cl)
        assert(len(cl))
        assert (not allWhitespace.match(cl)), "Adding blank line to corpus!"
        print(cl, file=self.corpusFile)
        self.corpusFile.flush()
        self.stopMitlm()

    def _corpified(wrappedMethod):
        "Decorator. Automatically deals with lexemes for the wrapped method."
        @wraps(wrappedMethod)
        def method(self, lexemes, *args, **kwargs):
            qString = self.corpify(lexemes)
            return wrappedMethod(self, qString, *args, **kwargs)
        return method

    def _waitForZMQResponse(self):
        assert self.mitlmSocket
        while True:
          self.checkMitlm()
          try:
            self.mitlmSocket.poll(timeout=1000)
            return self.mitlmSocket.recv(flags=zmq.NOBLOCK)
          except zmq.ZMQError:
              pass

    def queryCorpus(self, request):
        self.startMitlm()
        self.sendEntropyRequest(request)
        r = float(self._waitForZMQResponse())
        if r >= 70.0:
          warning("Infinity: %s" % qString)
          self.checkMitlm()
          assert False
        return r

    def predictCorpus(self, lexemes):
        self.startMitlm()
        self.sendPredictionRequest(lexemes)
        return self.parsePredictionResult(self._waitForZMQResponse(),
                remove_prefix=len(lexemes))

    @_corpified
    def sendEntropyRequest(self, request):
        # Coerce into bytes, if required.
        return self._send(CROSS_ENTROPY_PREFIX + request)

    @_corpified
    def sendPredictionRequest(self, request):
        return self._send(PREDICTION_PREFIX + request)

    def _send(self, string):
        "Sends a string to zmq. Ensures MITLM is initialized."
        assert self.mitlmSocket
        if isinstance(string, unicode):
            string = string.encode('utf-8')
        return self.mitlmSocket.send(string)

    @staticmethod
    def parsePredictionResult(resultString, remove_prefix):
        lines = resultString.split('\n')

        def split_tail(text):
            return text.split()[remove_prefix:]

        def cleanLine(line):
            components = line.split('\t', 1)
            assert len(components) == 2
            entropy_str, text = components
            return float(entropy_str), split_tail(text)

        return [cleanLine(line) for line in lines if line.strip() != ""]

    def release(self):
        """Close files and stop MITLM"""
        self.closeCorpus()
        self.stopMitlm()

    def __del__(self):
        """I am a destructor, but release should be called explictly."""
        assert not self.mitlmProc, "Destructor called before release()"
        assert not self.mitlmSocket, "Destructor called before release()"
        assert not self.corpusFile, "Destructor called before release()"

def chill_rm(filename):
    """
    rm, but silently ignore errors if the given file does not exist.
    """
    try:
        os.remove(filename)
    except OSError as error:
        # Ignore the error if the file does not exist.
        if error.errno == errno.ENOENT:
            pass
        else:
            raise error

def normalize_path(path):
    """
    Returns a path without a dangling ipc:// in front of it.

    >>> path1 = 'ipc:///home/ucuser/.unnaturalCode/socket'
    >>> path2 =       '/home/ucuser/.unnaturalCode/socket'
    >>> normalize_path(path1) == normalize_path(path2)
    True
    """

    if path.startswith('ipc://'):
        _head, _sep, tail = path.partition('ipc://')
        return tail
    return path

<commit_msg>I realize that I need the socket path...<commit_after>#    Copyright 2013, 2014 Joshua Charles Campbell
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import print_function
import os, zmq, signal, os.path, subprocess, fcntl, time
import errno
from unnaturalcode.unnaturalCode import *
from logging import debug, info, warning, error, getLogger
from multiprocessing import Process
from functools import wraps

allWhitespace = re.compile('^\s+$')

ucParanoid = os.getenv("PARANOID", False)

mitlmLogger = getLogger('MITLM')

CROSS_ENTROPY_PREFIX = 'x'
PREDICTION_PREFIX = 'p'

class mitlmCorpus(object):

    def __init__(self, readCorpus=None, writeCorpus=None, estimateNgramPath=None, uc=unnaturalCode(), order=10):
        self.readCorpus = (readCorpus or os.getenv("ucCorpus", "/tmp/ucCorpus"))
        self.writeCorpus = (writeCorpus or os.getenv("ucWriteCorpus", self.readCorpus))
        self.mitlmSocketPath = "ipc://%s/%s-%i-%i" % (os.path.dirname(self.readCorpus), "ucMitlmSocket", os.getpid(), id(self))
        self.estimateNgramPath = (estimateNgramPath or os.getenv("ESTIMATENGRAM", os.popen('which estimate-ngram').read()))
        self.corpusFile = False
        self.mitlmSocket = None
        self.mitlmProc = None
        self.order = order
        self.zctx = uc.zctx

    def checkMitlm(self):
        while self.mitlmProc:
            try:
                msg = self.mitlmProc.stdout.readline().rstrip('\n')
                if not allWhitespace.match(msg):
                  mitlmLogger.info()
            except:
                break

    def startMitlm(self):
        """Start MITLM estimate-ngram in 0MQ entropy query mode, unless already running."""
        if not self.mitlmSocket == None :
            assert not self.mitlmSocket.closed
            assert self.mitlmProc.poll() == None
            # Already running
            self.checkMitlm()
            return
        assert os.path.exists(self.readCorpus), "No such corpus."
        assert not allWhitespace.match(slurp(self.readCorpus)), "Corpus is full of whitespace!"
        assert os.path.exists(self.estimateNgramPath), "No such estimate-ngram."
        self.mitlmProc = subprocess.Popen([self.estimateNgramPath,
            "-t", self.readCorpus,
            "-o", str(self.order),
            "-s", "ModKN",
            "-u",
            "-live-prob", self.mitlmSocketPath], stdout=subprocess.PIPE)
        debug("Started MITLM as PID %i." % self.mitlmProc.pid)

        fd = self.mitlmProc.stdout.fileno()
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

        # Test the ZMQ connection.
        time.sleep(1)
        self.checkMitlm()
        self.mitlmSocket = self.zctx.socket(zmq.REQ)
        self.mitlmSocket.connect(self.mitlmSocketPath)
        self.checkMitlm()
        self.sendEntropyRequest(['for', '(', 'i', '='])
        self.checkMitlm()
        r = float(self.mitlmSocket.recv())
        debug("MITLM said %f" % r)
        self.checkMitlm()

    def stopMitlm(self):
        """Stop MITLM estimate-ngram, unless not running."""
        self.checkMitlm()
        if self.mitlmSocket:
            self.mitlmSocket.setsockopt(zmq.LINGER, 0)
            self.mitlmSocket.close()
            assert self.mitlmSocket.closed
            chill_rm(normalize_path(self.mitlmSocketPath))
            self.mitlmSocket = None
        if self.mitlmProc:
            rc = None
            debug("Waiting for MITLM to shut down...")
            while self.mitlmProc and (rc == None):
                self.mitlmProc.terminate()
                time.sleep(0.1)
                rc = self.mitlmProc.poll()
            self.mitlmProc = None
            debug("MITLM exited with status %r" % (rc))

    def corpify(self, lexemes):
        """Stringify lexed source: produce space-seperated sequence of lexemes"""
        assert isinstance(lexemes, list)
        assert len(lexemes)
        return " ".join(lexemes)

    def openCorpus(self):
        """Opens the corpus (if necessary)"""
        if (self.corpusFile):
            assert not self.corpusFile.closed
            return
        self.corpusFile = open(self.writeCorpus, 'a')

    def closeCorpus(self):
        """Closes the corpus (if necessary)"""
        if (self.corpusFile):
            self.corpusFile.close()
            assert self.corpusFile.closed
            self.corpusFile = None

    def addToCorpus(self, lexemes):
        """Adds a string of lexemes to the corpus"""
        assert isinstance(lexemes, list)
        assert len(lexemes)
        self.openCorpus()
        cl = self.corpify(lexemes)
        #debug(cl)
        assert(len(cl))
        assert (not allWhitespace.match(cl)), "Adding blank line to corpus!"
        print(cl, file=self.corpusFile)
        self.corpusFile.flush()
        self.stopMitlm()

    def _corpified(wrappedMethod):
        "Decorator. Automatically deals with lexemes for the wrapped method."
        @wraps(wrappedMethod)
        def method(self, lexemes, *args, **kwargs):
            qString = self.corpify(lexemes)
            return wrappedMethod(self, qString, *args, **kwargs)
        return method

    def _waitForZMQResponse(self):
        assert self.mitlmSocket
        while True:
          self.checkMitlm()
          try:
            self.mitlmSocket.poll(timeout=1000)
            return self.mitlmSocket.recv(flags=zmq.NOBLOCK)
          except zmq.ZMQError:
              pass

    def queryCorpus(self, request):
        self.startMitlm()
        self.sendEntropyRequest(request)
        r = float(self._waitForZMQResponse())
        if r >= 70.0:
          warning("Infinity: %s" % qString)
          self.checkMitlm()
          assert False
        return r

    def predictCorpus(self, lexemes):
        self.startMitlm()
        self.sendPredictionRequest(lexemes)
        return self.parsePredictionResult(self._waitForZMQResponse(),
                remove_prefix=len(lexemes))

    @_corpified
    def sendEntropyRequest(self, request):
        # Coerce into bytes, if required.
        return self._send(CROSS_ENTROPY_PREFIX + request)

    @_corpified
    def sendPredictionRequest(self, request):
        return self._send(PREDICTION_PREFIX + request)

    def _send(self, string):
        "Sends a string to zmq. Ensures MITLM is initialized."
        assert self.mitlmSocket
        if isinstance(string, unicode):
            string = string.encode('utf-8')
        return self.mitlmSocket.send(string)

    @staticmethod
    def parsePredictionResult(resultString, remove_prefix):
        lines = resultString.split('\n')

        def split_tail(text):
            return text.split()[remove_prefix:]

        def cleanLine(line):
            components = line.split('\t', 1)
            assert len(components) == 2
            entropy_str, text = components
            return float(entropy_str), split_tail(text)

        return [cleanLine(line) for line in lines if line.strip() != ""]

    def release(self):
        """Close files and stop MITLM"""
        self.closeCorpus()
        self.stopMitlm()

    def __del__(self):
        """I am a destructor, but release should be called explictly."""
        assert not self.mitlmProc, "Destructor called before release()"
        assert not self.mitlmSocket, "Destructor called before release()"
        assert not self.corpusFile, "Destructor called before release()"

def chill_rm(filename):
    """
    rm, but silently ignore errors if the given file does not exist.
    """
    try:
        os.remove(filename)
    except OSError as error:
        # Ignore the error if the file does not exist.
        if error.errno == errno.ENOENT:
            pass
        else:
            raise error

def normalize_path(path):
    """
    Returns a path without a dangling ipc:// in front of it.

    >>> path1 = 'ipc:///home/ucuser/.unnaturalCode/socket'
    >>> path2 =       '/home/ucuser/.unnaturalCode/socket'
    >>> normalize_path(path1) == normalize_path(path2)
    True
    """

    if path.startswith('ipc://'):
        _head, _sep, tail = path.partition('ipc://')
        return tail
    return path

<|endoftext|>