<commit_before>"""
2D rendering framework
"""
from __future__ import division
import os, sys
if "Apple" in sys.version:
    if 'DYLD_FALLBACK_LIBRARY_PATH' in os.environ:
        os.environ['DYLD_FALLBACK_LIBRARY_PATH'] += ':/usr/lib'
        # (JDS 2016/04/15): avoid bug on Anaconda 2.3.0 / Yosemite

from gym import error

try:
    import pyglet
except ImportError as e:
    raise error.DependencyNotInstalled("""{}

(HINT: you can install pyglet directly via 'pip install pyglet'. But if you really just want to install all Gym dependencies and not have to think about it, 'pip install -e .[all]' or 'pip install gym[all]' will do it.)""")

try:
    from pyglet.gl import *
except ImportError as e:
    raise error.DependencyNotInstalled("""{} (while running: from pyglet.gl import *).

(HINT: make sure you have OpenGL install. On Ubuntu, you can run 'apt-get install python-opengl'. If you're running on a server, you may need a virtual frame buffer; something like this should work: 'xvfb-run -s "-screen 0 1400x900x24" <your script here>')""".format(e))

import math
import numpy as np

RAD2DEG = 57.29577951308232

class Viewer(object):
    def __init__(self, width, height):
        self.width = width
        self.height = height
        self.window = pyglet.window.Window(width=width, height=height)
        self.geoms = []
        self.onetime_geoms = []
        self.transform = Transform()

        glEnable(GL_BLEND)
        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)

    def close(self):
        self.window.close()

    def set_bounds(self, left, right, bottom, top):
        assert right > left and top > bottom
        scalex = self.width/(right-left)
        scaley = self.height/(top-bottom)
        self.transform = Transform(
            translation=(-left*scalex, -bottom*scalex),
            scale=(scalex, scaley))

    def add_geom(self, geom):
        self.geoms.append(geom)

    def add_onetime(self, geom):
        self.onetime_geoms.append(geom)

    def render(self):
        glClearColor(1,1,1,1)
        self.window.clear()
        self.window.switch_to()
        self.window.dispatch_events()
        self.transform.enable()
        for geom in self.geoms:
            geom.render()
        for geom in self.onetime_geoms:
            geom.render()
        self.transform.disable()
        self.window.flip()
        self.onetime_geoms = []

    # Convenience
    def draw_circle(self, radius=10, res=30, filled=True, **attrs):
        geom = make_circle(radius=radius, res=res, filled=filled)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def draw_polygon(self, v, filled=True, **attrs):
        geom = make_polygon(v=v, filled=filled)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def draw_polyline(self, v, **attrs):
        geom = make_polyline(v=v)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def draw_line(self, start, end, **attrs):
        geom = Line(start, end)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def get_array(self):
        self.window.flip()
        image_data = pyglet.image.get_buffer_manager().get_color_buffer().get_image_data()
        self.window.flip()
        arr = np.fromstring(image_data.data, dtype=np.uint8, sep='')
        arr = arr.reshape(self.height, self.width, 4)
        return arr[::-1,:,0:3]

def _add_attrs(geom, attrs):
    if "color" in attrs:
        geom.set_color(attrs["color"])
    if "linewidth" in attrs:
        geom.set_linewidth(attrs["linewidth"])

class Geom(object):
    def __init__(self):
        self._color=Color((0, 0, 0, 1.0))
        self.attrs = [self._color]
    def render(self):
        for attr in reversed(self.attrs):
            attr.enable()
        self.render1()
        for attr in self.attrs:
            attr.disable()
    def render1(self):
        raise NotImplementedError
    def add_attr(self, attr):
        self.attrs.append(attr)
    def set_color(self, r, g, b):
        self._color.vec4 = (r, g, b, 1)

class Attr(object):
    def enable(self):
        raise NotImplementedError
    def disable(self):
        pass

class Transform(Attr):
    def __init__(self, translation=(0.0, 0.0), rotation=0.0, scale=(1,1)):
        self.set_translation(*translation)
        self.set_rotation(rotation)
        self.set_scale(*scale)
    def enable(self):
        glPushMatrix()
        glTranslatef(self.translation[0], self.translation[1], 0) # translate to GL loc ppint
        glRotatef(RAD2DEG * self.rotation, 0, 0, 1.0)
        glScalef(self.scale[0], self.scale[1], 1)
    def disable(self):
        glPopMatrix()
    def set_translation(self, newx, newy):
        self.translation = (float(newx), float(newy))
    def set_rotation(self, new):
        self.rotation = float(new)
    def set_scale(self, newx, newy):
        self.scale = (float(newx), float(newy))

class Color(Attr):
    def __init__(self, vec4):
        self.vec4 = vec4
    def enable(self):
        glColor4f(*self.vec4)

class LineStyle(Attr):
    def __init__(self, style):
        self.style = style
    def enable(self):
        glEnable(GL_LINE_STIPPLE)
        glLineStipple(1, self.style)
    def disable(self):
        glDisable(GL_LINE_STIPPLE)

class LineWidth(Attr):
    def __init__(self, stroke):
        self.stroke = stroke
    def enable(self):
        glLineWidth(self.stroke)

class Point(Geom):
    def __init__(self):
        Geom.__init__(self)
    def render1(self):
        glBegin(GL_POINTS) # draw point
        glVertex3f(0.0, 0.0, 0.0)
        glEnd()

class FilledPolygon(Geom):
    def __init__(self, v):
        Geom.__init__(self)
        self.v = v
    def render1(self):
        if   len(self.v) == 4 : glBegin(GL_QUADS)
        elif len(self.v)  > 4 : glBegin(GL_POLYGON)
        else: glBegin(GL_TRIANGLES)
        for p in self.v:
            glVertex3f(p[0], p[1],0)  # draw each vertex
        glEnd()

def make_circle(radius=10, res=30, filled=True):
    points = []
    for i in xrange(res):
        ang = 2*math.pi*i / res
        points.append((math.cos(ang)*radius, math.sin(ang)*radius))
    if filled:
        return FilledPolygon(points)
    else:
        return PolyLine(points, True)

def make_polygon(v, filled=True):
    if filled: return FilledPolygon(v)
    else: return PolyLine(v, True)

def make_polyline(v):
    return PolyLine(v, False)

def make_capsule(length, width):
    l, r, t, b = 0, length, width/2, -width/2
    box = make_polygon([(l,b), (l,t), (r,t), (r,b)])
    circ0 = make_circle(width/2)
    circ1 = make_circle(width/2)
    circ1.add_attr(Transform(translation=(length, 0)))
    geom = Compound([box, circ0, circ1])
    return geom

class Compound(Geom):
    def __init__(self, gs):
        Geom.__init__(self)
        self.gs = gs
        for g in self.gs:
            g.attrs = [a for a in g.attrs if not isinstance(a, Color)]
    def render1(self):
        for g in self.gs:
            g.render()

class PolyLine(Geom):
    def __init__(self, v, close):
        Geom.__init__(self)
        self.v = v
        self.close = close
        self.linewidth = LineWidth(1)
        self.add_attr(self.linewidth)
    def render1(self):
        glBegin(GL_LINE_LOOP if self.close else GL_LINE_STRIP)
        for p in self.v:
            glVertex3f(p[0], p[1],0)  # draw each vertex
        glEnd()
    def set_linewidth(self, x):
        self.linewidth.stroke = x

class Line(Geom):
    def __init__(self, start=(0.0, 0.0), end=(0.0, 0.0)):
        Geom.__init__(self)
        self.start = start
        self.end = end
        self.linewidth = LineWidth(1)
        self.add_attr(self.linewidth)

    def render1(self):
        glBegin(GL_LINES)
        glVertex2f(*self.start)
        glVertex2f(*self.end)
        glEnd()

class Image(Geom):
    def __init__(self, fname, width, height):
        Geom.__init__(self)
        self.width = width
        self.height = height
        img = pyglet.image.load(fname)
        self.img = img
        self.flip = False
    def render1(self):
        self.img.blit(-self.width/2, -self.height/2, width=self.width, height=self.height)

# ================================================================

class SimpleImageViewer(object):
    def __init__(self):
        self.window = None
        self.isopen = False
    def imshow(self, arr):
        if self.window is None:
            height, width, channels = arr.shape
            self.window = pyglet.window.Window(width=width, height=height)
            self.width = width
            self.height = height
            self.isopen = True
        assert arr.shape == (self.height, self.width, 3), "You passed in an image with the wrong number shape"
        image = pyglet.image.ImageData(self.width, self.height, 'RGB', arr.tobytes(), pitch=self.width * -3)
        self.window.clear()
        self.window.switch_to()
        self.window.dispatch_events()
        image.blit(0,0)
        self.window.flip()
    def close(self):
        if self.isopen:
            self.window.close()
            self.isopen = False
    def __del__(self):
        self.close()
<commit_msg>rendering.py: Actually interpolate error message<commit_after>"""
2D rendering framework
"""
from __future__ import division
import os, sys
if "Apple" in sys.version:
    if 'DYLD_FALLBACK_LIBRARY_PATH' in os.environ:
        os.environ['DYLD_FALLBACK_LIBRARY_PATH'] += ':/usr/lib'
        # (JDS 2016/04/15): avoid bug on Anaconda 2.3.0 / Yosemite

from gym import error

try:
    import pyglet
except ImportError as e:
    raise error.DependencyNotInstalled("""{}

(HINT: you can install pyglet directly via 'pip install pyglet'. But if you really just want to install all Gym dependencies and not have to think about it, 'pip install -e .[all]' or 'pip install gym[all]' will do it.)""".format(e))

try:
    from pyglet.gl import *
except ImportError as e:
    raise error.DependencyNotInstalled("""{} (while running: from pyglet.gl import *).

(HINT: make sure you have OpenGL install. On Ubuntu, you can run 'apt-get install python-opengl'. If you're running on a server, you may need a virtual frame buffer; something like this should work: 'xvfb-run -s "-screen 0 1400x900x24" <your script here>')""".format(e))

import math
import numpy as np

RAD2DEG = 57.29577951308232

class Viewer(object):
    def __init__(self, width, height):
        self.width = width
        self.height = height
        self.window = pyglet.window.Window(width=width, height=height)
        self.geoms = []
        self.onetime_geoms = []
        self.transform = Transform()

        glEnable(GL_BLEND)
        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)

    def close(self):
        self.window.close()

    def set_bounds(self, left, right, bottom, top):
        assert right > left and top > bottom
        scalex = self.width/(right-left)
        scaley = self.height/(top-bottom)
        self.transform = Transform(
            translation=(-left*scalex, -bottom*scalex),
            scale=(scalex, scaley))

    def add_geom(self, geom):
        self.geoms.append(geom)

    def add_onetime(self, geom):
        self.onetime_geoms.append(geom)

    def render(self):
        glClearColor(1,1,1,1)
        self.window.clear()
        self.window.switch_to()
        self.window.dispatch_events()
        self.transform.enable()
        for geom in self.geoms:
            geom.render()
        for geom in self.onetime_geoms:
            geom.render()
        self.transform.disable()
        self.window.flip()
        self.onetime_geoms = []

    # Convenience
    def draw_circle(self, radius=10, res=30, filled=True, **attrs):
        geom = make_circle(radius=radius, res=res, filled=filled)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def draw_polygon(self, v, filled=True, **attrs):
        geom = make_polygon(v=v, filled=filled)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def draw_polyline(self, v, **attrs):
        geom = make_polyline(v=v)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def draw_line(self, start, end, **attrs):
        geom = Line(start, end)
        _add_attrs(geom, attrs)
        self.add_onetime(geom)
        return geom

    def get_array(self):
        self.window.flip()
        image_data = pyglet.image.get_buffer_manager().get_color_buffer().get_image_data()
        self.window.flip()
        arr = np.fromstring(image_data.data, dtype=np.uint8, sep='')
        arr = arr.reshape(self.height, self.width, 4)
        return arr[::-1,:,0:3]

def _add_attrs(geom, attrs):
    if "color" in attrs:
        geom.set_color(attrs["color"])
    if "linewidth" in attrs:
        geom.set_linewidth(attrs["linewidth"])

class Geom(object):
    def __init__(self):
        self._color=Color((0, 0, 0, 1.0))
        self.attrs = [self._color]
    def render(self):
        for attr in reversed(self.attrs):
            attr.enable()
        self.render1()
        for attr in self.attrs:
            attr.disable()
    def render1(self):
        raise NotImplementedError
    def add_attr(self, attr):
        self.attrs.append(attr)
    def set_color(self, r, g, b):
        self._color.vec4 = (r, g, b, 1)

class Attr(object):
    def enable(self):
        raise NotImplementedError
    def disable(self):
        pass

class Transform(Attr):
    def __init__(self, translation=(0.0, 0.0), rotation=0.0, scale=(1,1)):
        self.set_translation(*translation)
        self.set_rotation(rotation)
        self.set_scale(*scale)
    def enable(self):
        glPushMatrix()
        glTranslatef(self.translation[0], self.translation[1], 0) # translate to GL loc ppint
        glRotatef(RAD2DEG * self.rotation, 0, 0, 1.0)
        glScalef(self.scale[0], self.scale[1], 1)
    def disable(self):
        glPopMatrix()
    def set_translation(self, newx, newy):
        self.translation = (float(newx), float(newy))
    def set_rotation(self, new):
        self.rotation = float(new)
    def set_scale(self, newx, newy):
        self.scale = (float(newx), float(newy))

class Color(Attr):
    def __init__(self, vec4):
        self.vec4 = vec4
    def enable(self):
        glColor4f(*self.vec4)

class LineStyle(Attr):
    def __init__(self, style):
        self.style = style
    def enable(self):
        glEnable(GL_LINE_STIPPLE)
        glLineStipple(1, self.style)
    def disable(self):
        glDisable(GL_LINE_STIPPLE)

class LineWidth(Attr):
    def __init__(self, stroke):
        self.stroke = stroke
    def enable(self):
        glLineWidth(self.stroke)

class Point(Geom):
    def __init__(self):
        Geom.__init__(self)
    def render1(self):
        glBegin(GL_POINTS) # draw point
        glVertex3f(0.0, 0.0, 0.0)
        glEnd()

class FilledPolygon(Geom):
    def __init__(self, v):
        Geom.__init__(self)
        self.v = v
    def render1(self):
        if   len(self.v) == 4 : glBegin(GL_QUADS)
        elif len(self.v)  > 4 : glBegin(GL_POLYGON)
        else: glBegin(GL_TRIANGLES)
        for p in self.v:
            glVertex3f(p[0], p[1],0)  # draw each vertex
        glEnd()

def make_circle(radius=10, res=30, filled=True):
    points = []
    for i in xrange(res):
        ang = 2*math.pi*i / res
        points.append((math.cos(ang)*radius, math.sin(ang)*radius))
    if filled:
        return FilledPolygon(points)
    else:
        return PolyLine(points, True)

def make_polygon(v, filled=True):
    if filled: return FilledPolygon(v)
    else: return PolyLine(v, True)

def make_polyline(v):
    return PolyLine(v, False)

def make_capsule(length, width):
    l, r, t, b = 0, length, width/2, -width/2
    box = make_polygon([(l,b), (l,t), (r,t), (r,b)])
    circ0 = make_circle(width/2)
    circ1 = make_circle(width/2)
    circ1.add_attr(Transform(translation=(length, 0)))
    geom = Compound([box, circ0, circ1])
    return geom

class Compound(Geom):
    def __init__(self, gs):
        Geom.__init__(self)
        self.gs = gs
        for g in self.gs:
            g.attrs = [a for a in g.attrs if not isinstance(a, Color)]
    def render1(self):
        for g in self.gs:
            g.render()

class PolyLine(Geom):
    def __init__(self, v, close):
        Geom.__init__(self)
        self.v = v
        self.close = close
        self.linewidth = LineWidth(1)
        self.add_attr(self.linewidth)
    def render1(self):
        glBegin(GL_LINE_LOOP if self.close else GL_LINE_STRIP)
        for p in self.v:
            glVertex3f(p[0], p[1],0)  # draw each vertex
        glEnd()
    def set_linewidth(self, x):
        self.linewidth.stroke = x

class Line(Geom):
    def __init__(self, start=(0.0, 0.0), end=(0.0, 0.0)):
        Geom.__init__(self)
        self.start = start
        self.end = end
        self.linewidth = LineWidth(1)
        self.add_attr(self.linewidth)

    def render1(self):
        glBegin(GL_LINES)
        glVertex2f(*self.start)
        glVertex2f(*self.end)
        glEnd()

class Image(Geom):
    def __init__(self, fname, width, height):
        Geom.__init__(self)
        self.width = width
        self.height = height
        img = pyglet.image.load(fname)
        self.img = img
        self.flip = False
    def render1(self):
        self.img.blit(-self.width/2, -self.height/2, width=self.width, height=self.height)

# ================================================================

class SimpleImageViewer(object):
    def __init__(self):
        self.window = None
        self.isopen = False
    def imshow(self, arr):
        if self.window is None:
            height, width, channels = arr.shape
            self.window = pyglet.window.Window(width=width, height=height)
            self.width = width
            self.height = height
            self.isopen = True
        assert arr.shape == (self.height, self.width, 3), "You passed in an image with the wrong number shape"
        image = pyglet.image.ImageData(self.width, self.height, 'RGB', arr.tobytes(), pitch=self.width * -3)
        self.window.clear()
        self.window.switch_to()
        self.window.dispatch_events()
        image.blit(0,0)
        self.window.flip()
    def close(self):
        if self.isopen:
            self.window.close()
            self.isopen = False
    def __del__(self):
        self.close()
<|endoftext|>


<commit_before>#coding:utf-8
# google hosts
# 从定期更新的博客上抓取google host
# 目前支持windows 持续支持mac

import urllib
import re

google_hosts_url = "http://www.360kb.com/kb/2_150.html"

windows_hosts_path = "C:\\Windows\\System32\\drivers\\etc\\hosts"

def get_html(url):
    page = urllib.urlopen(url)
    return page.read()

if __name__ == "__main__":

    data = get_html(google_hosts_url)
    end_data = data[data.find("#google hosts"):data.rfind("#google hosts")].replace(" ", " ").replace("
","")
    f = open(windows_hosts_path,"w")
    f.write(end_data)
    f.close()

    print "ok"

<commit_msg>google_hosts 加上系统判定<commit_after>#coding:utf-8
# google hosts
# 从定期更新的博客上抓取google host
# 目前支持windows 持续支持mac

import urllib
import platform

google_hosts_url = "http://www.360kb.com/kb/2_150.html"

windows_hosts_path = "C:\\Windows\\System32\\drivers\\etc\\hosts"
linux_hosts_path = "/etc/hosts"

def get_html(url):
    page = urllib.urlopen(url)
    return page.read()

if __name__ == "__main__":
    os_name = platform.system()
    data = get_html(google_hosts_url)
    end_data = data[data.find("#google hosts"):data.rfind("#google hosts")].replace(" ", " ").replace("
","")
    path = ""
    if os_name == "Windows":
        path = windows_hosts_path
    elif os_name == "Linux":
        path = linux_hosts_path

    f = open(path,"w")
    f.write(end_data)
    f.close()

    print "ok"

<|endoftext|>


<commit_before>#!/usr/bin/python
import sys

if sys.version_info[0] >= 3:
    def bytes_to_str(b):
        return b.decode()
else:
    def bytes_to_str(b):
        return b

def linux_memory_stats():
    import os
    os.execlp('free', 'free', '-m')

def osx_memory_stats():
    import subprocess, re, os, resource

    def popen_read(command):
        return bytes_to_str(subprocess.Popen(command, stdout = subprocess.PIPE).communicate()[0])

    # Get process info
    ps = popen_read(['ps', '-caxm', '-orss,comm'])
    vm = popen_read(['vm_stat'])

    # Iterate processes
    process_lines = ps.split('\n')
    sep = re.compile('[\s]+')
    rss_total = 0 # kB
    for row in range(1, len(process_lines)):
        row_text = process_lines[row].strip()
        row_elements = sep.split(row_text)
        try:
            rss = float(row_elements[0]) * 1024
        except:
            rss = 0 # ignore...
        rss_total += rss

    # Process vm_stat
    vm_lines = vm.split('\n')
    sep = re.compile(':[\s]+')
    vm_stats = {}
    for row in range(1, len(vm_lines) - 2):
        row_text = vm_lines[row].strip()
        row_elements = sep.split(row_text)
        vm_stats[(row_elements[0])] = int(row_elements[1].strip('\.')) * resource.getpagesize()

    print('---- Summary ----')
    print('Wired Memory:\t\t%.1f MB' % (vm_stats["Pages wired down"] / 1024 / 1024))
    print('Active Memory:\t\t%.1f MB' % (vm_stats["Pages active"] / 1024 / 1024))
    print('Inactive Memory:\t%.1f MB' % (vm_stats["Pages inactive"] / 1024 / 1024))
    print('Free Memory:\t\t%.1f MB' % (vm_stats["Pages free"] / 1024 / 1024))
    print('Real Mem Total (ps):\t%.1f MB' % (rss_total / 1024 / 1024))
    print('')
    print('---- vm_stat ----')
    print(vm.strip())

if sys.platform.find('linux') > -1:
    linux_memory_stats()
elif sys.platform.find('darwin') > -1:
    osx_memory_stats()
<commit_msg>Change shebang line for system-memory-stats.py so that Python doesn't have to be in /usr/bin<commit_after>#!/usr/bin/env python
import sys

if sys.version_info[0] >= 3:
    def bytes_to_str(b):
        return b.decode()
else:
    def bytes_to_str(b):
        return b

def linux_memory_stats():
    import os
    os.execlp('free', 'free', '-m')

def osx_memory_stats():
    import subprocess, re, os, resource

    def popen_read(command):
        return bytes_to_str(subprocess.Popen(command, stdout = subprocess.PIPE).communicate()[0])

    # Get process info
    ps = popen_read(['ps', '-caxm', '-orss,comm'])
    vm = popen_read(['vm_stat'])

    # Iterate processes
    process_lines = ps.split('\n')
    sep = re.compile('[\s]+')
    rss_total = 0 # kB
    for row in range(1, len(process_lines)):
        row_text = process_lines[row].strip()
        row_elements = sep.split(row_text)
        try:
            rss = float(row_elements[0]) * 1024
        except:
            rss = 0 # ignore...
        rss_total += rss

    # Process vm_stat
    vm_lines = vm.split('\n')
    sep = re.compile(':[\s]+')
    vm_stats = {}
    for row in range(1, len(vm_lines) - 2):
        row_text = vm_lines[row].strip()
        row_elements = sep.split(row_text)
        vm_stats[(row_elements[0])] = int(row_elements[1].strip('\.')) * resource.getpagesize()

    print('---- Summary ----')
    print('Wired Memory:\t\t%.1f MB' % (vm_stats["Pages wired down"] / 1024 / 1024))
    print('Active Memory:\t\t%.1f MB' % (vm_stats["Pages active"] / 1024 / 1024))
    print('Inactive Memory:\t%.1f MB' % (vm_stats["Pages inactive"] / 1024 / 1024))
    print('Free Memory:\t\t%.1f MB' % (vm_stats["Pages free"] / 1024 / 1024))
    print('Real Mem Total (ps):\t%.1f MB' % (rss_total / 1024 / 1024))
    print('')
    print('---- vm_stat ----')
    print(vm.strip())

if sys.platform.find('linux') > -1:
    linux_memory_stats()
elif sys.platform.find('darwin') > -1:
    osx_memory_stats()
<|endoftext|>


<commit_before>import requests
import json
from lxml import html
from lxml.cssselect import CSSSelector
import requests
import string
from email_me import email_me

class freeRotation:

	def __init__(self):
		self.freeRotation = None;

	def scrapeRotation(self):
		try:
			forum_page = requests.get('http://us.battle.net/heroes/en/forum/topic/17936383460')
		except request.exceptions.ConnectionError:
			data = json.loads(urllib.urlopen("http://ip.jsontest.com/").read())
			email = email_me()
			message = 'Heroes Bot Didnt Update\n' + 'The Bot didn\'t update because it couldn\'t get ROTATION from Blizzard Forums. Check it out.\nThe IP is ' + str(data["ip"])
			email.populate_email('Heroes Bot Fail', message)
			email.send_email()
			sys.exit("Couldn't get ROTATION from Blizzard Forums")

		tree = html.fromstring(forum_page.text)

		results = CSSSelector('div.TopicPost-bodyContent li')
		preParsedFreeRotation = results(tree)

		heroList = [Hero.text for Hero in preParsedFreeRotation]

		for i in range(10):
			heroList[i] = heroList[i].split("(", 1)[0].replace(' ','').replace('.','')

		return heroList

	def buildRotation(self):
		heroList = self.scrapeRotation()

		self.freeRotation = []
		twoHeros = {
			'Samuro':'',
			'Ragnaros':'',
			'Varian':'',
		}
		subs = {
                        'liming': 'li-ming',
			'ltmorales': 'lt-morales',
                        'sgthammer': 'sgt-hammer',
			'thebutcher': 'the-butcher',
                        'thelostvikings': 'the-lost-vikings',
		}

		self.freeRotation.append("")

		count = 0
		self.freeRotation.append("[](#spacer)")

		for i in range(14):
			if i in range(0, 3) or i in range (4, 8) or i in range(9, 12):
				forURL = heroList[count].lower()

				if forURL in subs:
					forURL = subs[forURL]

				isTwoHero = ''
				if forURL in twoHeros:
					isTwoHero = '2'

				heroName = heroList[count]
				self.freeRotation.append("[](http://us.battle.net/heroes/en/heroes/{0}/#free_rotation{2}#{1})".format(forURL, heroName, isTwoHero))
				count = count + 1

			if i == 3 or i == 8 or i == 12:
				self.freeRotation.append("[](#spacer)")

			if i == 13:
				self.freeRotation.append("[](#bottom)")

		string = ''
		for item in self.freeRotation:
			string += item + '\n'
		return string

# if __name__ == '__main__':
# 	rotation = freeRotation()
# 	print rotation.buildRotation()
<commit_msg>Strip special characters.<commit_after>import requests
import json
from lxml import html
from lxml.cssselect import CSSSelector
import requests
import string
from email_me import email_me

class freeRotation:

	def __init__(self):
		self.freeRotation = None;

	def scrapeRotation(self):
		try:
			forum_page = requests.get('http://us.battle.net/heroes/en/forum/topic/17936383460')
		except request.exceptions.ConnectionError:
			data = json.loads(urllib.urlopen("http://ip.jsontest.com/").read())
			email = email_me()
			message = 'Heroes Bot Didnt Update\n' + 'The Bot didn\'t update because it couldn\'t get ROTATION from Blizzard Forums. Check it out.\nThe IP is ' + str(data["ip"])
			email.populate_email('Heroes Bot Fail', message)
			email.send_email()
			sys.exit("Couldn't get ROTATION from Blizzard Forums")

		tree = html.fromstring(forum_page.text)

		results = CSSSelector('div.TopicPost-bodyContent li')
		preParsedFreeRotation = results(tree)

		heroList = [Hero.text for Hero in preParsedFreeRotation]

		for i in range(10):
			heroList[i] = heroList[i].split("(", 1)[0].replace(' ','').replace('.','').replace("'",'')

		return heroList

	def buildRotation(self):
		heroList = self.scrapeRotation()

		self.freeRotation = []
		twoHeros = {
			'Samuro':'',
			'Ragnaros':'',
			'Varian':'',
		}
		subs = {
                        'liming': 'li-ming',
			'ltmorales': 'lt-morales',
                        'sgthammer': 'sgt-hammer',
			'thebutcher': 'the-butcher',
                        'thelostvikings': 'the-lost-vikings',
		}

		self.freeRotation.append("")

		count = 0
		self.freeRotation.append("[](#spacer)")

		for i in range(14):
			if i in range(0, 3) or i in range (4, 8) or i in range(9, 12):
				forURL = heroList[count].lower()

				if forURL in subs:
					forURL = subs[forURL]

				isTwoHero = ''
				if forURL in twoHeros:
					isTwoHero = '2'

				heroName = heroList[count]
				self.freeRotation.append("[](http://us.battle.net/heroes/en/heroes/{0}/#free_rotation{2}#{1})".format(forURL, heroName, isTwoHero))
				count = count + 1

			if i == 3 or i == 8 or i == 12:
				self.freeRotation.append("[](#spacer)")

			if i == 13:
				self.freeRotation.append("[](#bottom)")

		string = ''
		for item in self.freeRotation:
			string += item + '\n'
		return string

# if __name__ == '__main__':
# 	rotation = freeRotation()
# 	print rotation.buildRotation()
<|endoftext|>


<commit_before>import multiprocessing
import os
import random
import sys

import inspectShell

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
import subprocesses as sps

def memoize(f, cache={}):
    '''Function decorator that caches function results.'''
    # From http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/#c9
    def g(*args, **kwargs):
        key = ( f, tuple(args), frozenset(kwargs.items()) )
        if key not in cache:
            cache[key] = f(*args, **kwargs)
        return cache[key]
    return g

@memoize
def shellSupportsFlag(shellPath, flag):
    return inspectShell.shellSupports(shellPath, [flag, '-e', '42'])

def chance(p):
    return random.random() < p

def randomFlagSet(shellPath):
    '''
    Returns a random list of command-line flags appropriate for the given shell.
    Only works for spidermonkey js shell. Does not work for xpcshell.
    '''

    args = []

    ion = shellSupportsFlag(shellPath, "--ion") and chance(.8)

    if shellSupportsFlag(shellPath, '--fuzzing-safe'):
        args.append("--fuzzing-safe")  # --fuzzing-safe landed in bug 885361

    if shellSupportsFlag(shellPath, '--no-cgc') and chance(.1):
        args.append("--no-cgc")  # --no-cgc landed in bug 1126769

    if shellSupportsFlag(shellPath, '--no-ggc') and chance(.1):
        args.append("--no-ggc")  # --no-ggc landed in bug 706885

    if shellSupportsFlag(shellPath, '--no-incremental-gc') and chance(.1):
        args.append("--no-incremental-gc")  # --no-incremental-gc landed in bug 958492

    if shellSupportsFlag(shellPath, '--unboxed-objects') and chance(.2):
        args.append("--unboxed-objects")

    #if shellSupportsFlag(shellPath, '--ion-sink=on') and chance(.2):
    #    args.append("--ion-sink=on")  # --ion-sink=on landed in bug 1093674

    if shellSupportsFlag(shellPath, '--gc-zeal=0') and chance(.9):
        gczealValue = 14 if chance(0.5) else random.randint(0, 14)  # Focus test compacting GC (14)
        args.append("--gc-zeal=" + str(gczealValue))  # --gc-zeal= landed in bug 1101602

    if shellSupportsFlag(shellPath, '--enable-small-chunk-size') and chance(.1):
        args.append("--enable-small-chunk-size")  # --enable-small-chunk-size landed in bug 941804

    if shellSupportsFlag(shellPath, '--ion-loop-unrolling=on') and chance(.2):
        args.append("--ion-loop-unrolling=on")  # --ion-loop-unrolling=on landed in bug 1039458

    if shellSupportsFlag(shellPath, '--no-threads') and chance(.5):
        args.append("--no-threads")  # --no-threads landed in bug 1031529

    if shellSupportsFlag(shellPath, '--disable-ion') and chance(.05):
        args.append("--disable-ion")  # --disable-ion landed in bug 789319

    # See bug 1026919 comment 60:
    if sps.isARMv7l and \
            shellSupportsFlag(shellPath, '--arm-asm-nop-fill=0') and chance(0.3):
        # It was suggested to focus more on the range between 0 and 1.
        # Reduced the upper limit to 8, see bug 1053996 comment 8.
        asmNopFill = random.randint(1, 8) if chance(0.3) else random.randint(0, 1)
        args.append("--arm-asm-nop-fill=" + str(asmNopFill))  # Landed in bug 1020834

    # See bug 1026919 comment 60:
    if sps.isARMv7l and \
            shellSupportsFlag(shellPath, '--asm-pool-max-offset=1024') and chance(0.3):
        asmPoolMaxOffset = random.randint(5, 1024)
        args.append("--asm-pool-max-offset=" + str(asmPoolMaxOffset))  # Landed in bug 1026919

    if shellSupportsFlag(shellPath, '--no-native-regexp') and chance(.1):
        args.append("--no-native-regexp")  # See bug 976446

    if inspectShell.queryBuildConfiguration(shellPath, 'arm-simulator') and chance(.4):
        args.append('--arm-sim-icache-checks')

    if (shellSupportsFlag(shellPath, '--no-sse3') and \
            shellSupportsFlag(shellPath, '--no-sse4')) and chance(.2):
        # --no-sse3 and --no-sse4 landed in m-c rev 526ba3ace37a.
        if chance(.5):
            args.append("--no-sse3")
        else:
            args.append("--no-sse4")

    if shellSupportsFlag(shellPath, '--no-fpu') and chance(.2):
        args.append("--no-fpu")  # --no-fpu landed in bug 858022

    if shellSupportsFlag(shellPath, '--no-asmjs') and chance(.5):
        args.append("--no-asmjs")

    # --baseline-eager landed after --no-baseline on the IonMonkey branch prior to landing on m-c.
    if shellSupportsFlag(shellPath, '--baseline-eager'):
        if chance(.3):
            args.append('--no-baseline')
        # elif is important, as we want to call --baseline-eager only if --no-baseline is not set.
        elif chance(.6):
            args.append("--baseline-eager")

    if shellSupportsFlag(shellPath, '--ion-offthread-compile=off'):
        if chance(.7):
            # Focus on the reproducible cases
            args.append("--ion-offthread-compile=off")
        elif chance(.5) and multiprocessing.cpu_count() > 1 and \
                shellSupportsFlag(shellPath, '--thread-count=1'):
            # Adjusts default number of threads for parallel compilation (turned on by default)
            totalThreads = random.randint(2, (multiprocessing.cpu_count() * 2))
            args.append('--thread-count=' + str(totalThreads))
        # else:
        #   Default is to have --ion-offthread-compile=on and --thread-count=<some default value>
    elif shellSupportsFlag(shellPath, '--ion-parallel-compile=off'):
        # --ion-parallel-compile=off has gone away as of m-c rev 9ab3b097f304 and f0d67b1ccff9.
        if chance(.7):
            # Focus on the reproducible cases
            args.append("--ion-parallel-compile=off")
        elif chance(.5) and multiprocessing.cpu_count() > 1 and \
                shellSupportsFlag(shellPath, '--thread-count=1'):
            # Adjusts default number of threads for parallel compilation (turned on by default)
            totalThreads = random.randint(2, (multiprocessing.cpu_count() * 2))
            args.append('--thread-count=' + str(totalThreads))
        # else:
        #   The default is to have --ion-parallel-compile=on and --thread-count=<some default value>

    if ion:
        if chance(.6):
            args.append("--ion-eager")
        if chance(.2):
            args.append("--ion-gvn=off")
        if chance(.2):
            args.append("--ion-licm=off")
        if shellSupportsFlag(shellPath, '--ion-edgecase-analysis=off') and chance(.2):
            args.append("--ion-edgecase-analysis=off")
        if chance(.2):
            args.append("--ion-range-analysis=off")
        if chance(.2):
            args.append("--ion-inlining=off")
        if chance(.2):
            args.append("--ion-osr=off")
        if chance(.2):
            args.append("--ion-limit-script-size=off")
        # Backtracking (on by default as of 2015-04-15) and stupid landed in m-c changeset dc4887f61d2e
        # Disabled until bug 871848 is fixed.
        #if shellSupportsFlag(shellPath, '--ion-regalloc=stupid') and chance(.2):
            #args.append('--ion-regalloc=stupid')
        if shellSupportsFlag(shellPath, '--ion-check-range-analysis'):
            if chance(.3):
                args.append('--ion-check-range-analysis')
        if shellSupportsFlag(shellPath, '--ion-extra-checks'):
            if chance(.3):
                args.append('--ion-extra-checks')
    else:
        args.append("--no-ion")

    #if chance(.05):
    #    args.append("--execute=verifyprebarriers()")
    #if chance(.05):
    #    args.append("--execute=verifypostbarriers()")

    if chance(.05):
        args.append("-D") # aka --dump-bytecode

    return args

def basicFlagSets(shellPath):
    '''
    compareJIT uses these combinations of flags (as well as the original set of flags) when run
    through Lithium and autoBisect.
    '''
    if shellSupportsFlag(shellPath, "--no-threads"):
        basicFlagList = [
            # Parts of this flag permutation come from:
            # https://hg.mozilla.org/mozilla-central/file/e3bf27190360/js/src/tests/lib/tests.py#l12
            ['--fuzzing-safe', '--no-threads', '--ion-eager'],  # compareJIT uses this first flag set as the sole baseline when fuzzing
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager'],
            ['--fuzzing-safe', '--no-threads', '--baseline-eager'],
            ['--fuzzing-safe', '--no-threads', '--baseline-eager', '--no-fpu'],
            ['--fuzzing-safe', '--no-threads', '--no-baseline', '--no-ion'],
        ]
        if shellSupportsFlag(shellPath, "--ion-extra-checks"):
            basicFlagList.append(['--fuzzing-safe', '--no-threads', '--ion-eager',
                                  '--ion-check-range-analysis', '--ion-extra-checks', '--no-sse3'])
        return basicFlagList
    elif shellSupportsFlag(shellPath, "--ion-offthread-compile=off"):
        basicFlagList = [
            # Parts of this flag permutation come from:
            # https://hg.mozilla.org/mozilla-central/file/84bd8d9f4256/js/src/tests/lib/tests.py#l12
            # as well as other interesting flag combinations that have found / may find new bugs.
            ['--fuzzing-safe', '--ion-offthread-compile=off'],  # compareJIT uses this first flag set as the sole baseline when fuzzing
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline', '--no-ion'],
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-ion'], # Not in jit_test.py though, see bug 848906 comment 1
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-fpu'],
        ]
        if shellSupportsFlag(shellPath, "--thread-count=1"):
            basicFlagList.append(['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager'])
            # Range analysis had only started to stabilize around the time when --no-sse3 landed.
            if shellSupportsFlag(shellPath, '--no-sse3'):
                basicFlagList.append(['--fuzzing-safe', '--ion-offthread-compile=off',
                                      '--ion-eager', '--ion-check-range-analysis', '--no-sse3'])
        return basicFlagList
    else:
        basicFlagList = [
            # Parts of this flag permutation come from:
            # https://hg.mozilla.org/mozilla-central/file/10932f3a0ba0/js/src/tests/lib/tests.py#l12
            # as well as other interesting flag combinations that have found / may find new bugs.
            ['--fuzzing-safe', '--ion-parallel-compile=off'],  # compareJIT uses this first flag set as the sole baseline when fuzzing
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline', '--no-ion'],
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--baseline-eager'],
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--baseline-eager', '--no-ion'], # See bug 848906 comment 1
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--baseline-eager', '--no-fpu'],
        ]
        if shellSupportsFlag(shellPath, "--thread-count=1"):
            basicFlagList.append(['--fuzzing-safe', '--ion-eager', '--ion-parallel-compile=off'])
            # Range analysis had only started to stabilize around the time when --no-sse3 landed.
            if shellSupportsFlag(shellPath, '--no-sse3'):
                basicFlagList.append(['--fuzzing-safe', '--ion-parallel-compile=off',
                                      '--ion-eager', '--ion-check-range-analysis', '--no-sse3'])
        return basicFlagList

# Consider adding a function (for compareJIT reduction) that takes a flag set
# and returns all its (meaningful) subsets.

def testRandomFlags():
    import sys
    for i in range(100):
        print ' '.join(randomFlagSet(sys.argv[1]))

if __name__ == "__main__":
    testRandomFlags()
<commit_msg>Add support for --unboxed-arrays.<commit_after>import multiprocessing
import os
import random
import sys

import inspectShell

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
import subprocesses as sps

def memoize(f, cache={}):
    '''Function decorator that caches function results.'''
    # From http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/#c9
    def g(*args, **kwargs):
        key = ( f, tuple(args), frozenset(kwargs.items()) )
        if key not in cache:
            cache[key] = f(*args, **kwargs)
        return cache[key]
    return g

@memoize
def shellSupportsFlag(shellPath, flag):
    return inspectShell.shellSupports(shellPath, [flag, '-e', '42'])

def chance(p):
    return random.random() < p

def randomFlagSet(shellPath):
    '''
    Returns a random list of command-line flags appropriate for the given shell.
    Only works for spidermonkey js shell. Does not work for xpcshell.
    '''

    args = []

    ion = shellSupportsFlag(shellPath, "--ion") and chance(.8)

    if shellSupportsFlag(shellPath, '--fuzzing-safe'):
        args.append("--fuzzing-safe")  # --fuzzing-safe landed in bug 885361

    if shellSupportsFlag(shellPath, '--no-cgc') and chance(.1):
        args.append("--no-cgc")  # --no-cgc landed in bug 1126769

    if shellSupportsFlag(shellPath, '--no-ggc') and chance(.1):
        args.append("--no-ggc")  # --no-ggc landed in bug 706885

    if shellSupportsFlag(shellPath, '--no-incremental-gc') and chance(.1):
        args.append("--no-incremental-gc")  # --no-incremental-gc landed in bug 958492

    if shellSupportsFlag(shellPath, '--unboxed-arrays') and chance(.2):
        args.append("--unboxed-arrays")  # --unboxed-arrays landed in bug 1146597

    if shellSupportsFlag(shellPath, '--unboxed-objects') and chance(.2):
        args.append("--unboxed-objects")

    #if shellSupportsFlag(shellPath, '--ion-sink=on') and chance(.2):
    #    args.append("--ion-sink=on")  # --ion-sink=on landed in bug 1093674

    if shellSupportsFlag(shellPath, '--gc-zeal=0') and chance(.9):
        gczealValue = 14 if chance(0.5) else random.randint(0, 14)  # Focus test compacting GC (14)
        args.append("--gc-zeal=" + str(gczealValue))  # --gc-zeal= landed in bug 1101602

    if shellSupportsFlag(shellPath, '--enable-small-chunk-size') and chance(.1):
        args.append("--enable-small-chunk-size")  # --enable-small-chunk-size landed in bug 941804

    if shellSupportsFlag(shellPath, '--ion-loop-unrolling=on') and chance(.2):
        args.append("--ion-loop-unrolling=on")  # --ion-loop-unrolling=on landed in bug 1039458

    if shellSupportsFlag(shellPath, '--no-threads') and chance(.5):
        args.append("--no-threads")  # --no-threads landed in bug 1031529

    if shellSupportsFlag(shellPath, '--disable-ion') and chance(.05):
        args.append("--disable-ion")  # --disable-ion landed in bug 789319

    # See bug 1026919 comment 60:
    if sps.isARMv7l and \
            shellSupportsFlag(shellPath, '--arm-asm-nop-fill=0') and chance(0.3):
        # It was suggested to focus more on the range between 0 and 1.
        # Reduced the upper limit to 8, see bug 1053996 comment 8.
        asmNopFill = random.randint(1, 8) if chance(0.3) else random.randint(0, 1)
        args.append("--arm-asm-nop-fill=" + str(asmNopFill))  # Landed in bug 1020834

    # See bug 1026919 comment 60:
    if sps.isARMv7l and \
            shellSupportsFlag(shellPath, '--asm-pool-max-offset=1024') and chance(0.3):
        asmPoolMaxOffset = random.randint(5, 1024)
        args.append("--asm-pool-max-offset=" + str(asmPoolMaxOffset))  # Landed in bug 1026919

    if shellSupportsFlag(shellPath, '--no-native-regexp') and chance(.1):
        args.append("--no-native-regexp")  # See bug 976446

    if inspectShell.queryBuildConfiguration(shellPath, 'arm-simulator') and chance(.4):
        args.append('--arm-sim-icache-checks')

    if (shellSupportsFlag(shellPath, '--no-sse3') and \
            shellSupportsFlag(shellPath, '--no-sse4')) and chance(.2):
        # --no-sse3 and --no-sse4 landed in m-c rev 526ba3ace37a.
        if chance(.5):
            args.append("--no-sse3")
        else:
            args.append("--no-sse4")

    if shellSupportsFlag(shellPath, '--no-fpu') and chance(.2):
        args.append("--no-fpu")  # --no-fpu landed in bug 858022

    if shellSupportsFlag(shellPath, '--no-asmjs') and chance(.5):
        args.append("--no-asmjs")

    # --baseline-eager landed after --no-baseline on the IonMonkey branch prior to landing on m-c.
    if shellSupportsFlag(shellPath, '--baseline-eager'):
        if chance(.3):
            args.append('--no-baseline')
        # elif is important, as we want to call --baseline-eager only if --no-baseline is not set.
        elif chance(.6):
            args.append("--baseline-eager")

    if shellSupportsFlag(shellPath, '--ion-offthread-compile=off'):
        if chance(.7):
            # Focus on the reproducible cases
            args.append("--ion-offthread-compile=off")
        elif chance(.5) and multiprocessing.cpu_count() > 1 and \
                shellSupportsFlag(shellPath, '--thread-count=1'):
            # Adjusts default number of threads for parallel compilation (turned on by default)
            totalThreads = random.randint(2, (multiprocessing.cpu_count() * 2))
            args.append('--thread-count=' + str(totalThreads))
        # else:
        #   Default is to have --ion-offthread-compile=on and --thread-count=<some default value>
    elif shellSupportsFlag(shellPath, '--ion-parallel-compile=off'):
        # --ion-parallel-compile=off has gone away as of m-c rev 9ab3b097f304 and f0d67b1ccff9.
        if chance(.7):
            # Focus on the reproducible cases
            args.append("--ion-parallel-compile=off")
        elif chance(.5) and multiprocessing.cpu_count() > 1 and \
                shellSupportsFlag(shellPath, '--thread-count=1'):
            # Adjusts default number of threads for parallel compilation (turned on by default)
            totalThreads = random.randint(2, (multiprocessing.cpu_count() * 2))
            args.append('--thread-count=' + str(totalThreads))
        # else:
        #   The default is to have --ion-parallel-compile=on and --thread-count=<some default value>

    if ion:
        if chance(.6):
            args.append("--ion-eager")
        if chance(.2):
            args.append("--ion-gvn=off")
        if chance(.2):
            args.append("--ion-licm=off")
        if shellSupportsFlag(shellPath, '--ion-edgecase-analysis=off') and chance(.2):
            args.append("--ion-edgecase-analysis=off")
        if chance(.2):
            args.append("--ion-range-analysis=off")
        if chance(.2):
            args.append("--ion-inlining=off")
        if chance(.2):
            args.append("--ion-osr=off")
        if chance(.2):
            args.append("--ion-limit-script-size=off")
        # Backtracking (on by default as of 2015-04-15) and stupid landed in m-c changeset dc4887f61d2e
        # Disabled until bug 871848 is fixed.
        #if shellSupportsFlag(shellPath, '--ion-regalloc=stupid') and chance(.2):
            #args.append('--ion-regalloc=stupid')
        if shellSupportsFlag(shellPath, '--ion-check-range-analysis'):
            if chance(.3):
                args.append('--ion-check-range-analysis')
        if shellSupportsFlag(shellPath, '--ion-extra-checks'):
            if chance(.3):
                args.append('--ion-extra-checks')
    else:
        args.append("--no-ion")

    #if chance(.05):
    #    args.append("--execute=verifyprebarriers()")
    #if chance(.05):
    #    args.append("--execute=verifypostbarriers()")

    if chance(.05):
        args.append("-D") # aka --dump-bytecode

    return args

def basicFlagSets(shellPath):
    '''
    compareJIT uses these combinations of flags (as well as the original set of flags) when run
    through Lithium and autoBisect.
    '''
    if shellSupportsFlag(shellPath, "--no-threads"):
        basicFlagList = [
            # Parts of this flag permutation come from:
            # https://hg.mozilla.org/mozilla-central/file/e3bf27190360/js/src/tests/lib/tests.py#l12
            ['--fuzzing-safe', '--no-threads', '--ion-eager'],  # compareJIT uses this first flag set as the sole baseline when fuzzing
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager'],
            ['--fuzzing-safe', '--no-threads', '--baseline-eager'],
            ['--fuzzing-safe', '--no-threads', '--baseline-eager', '--no-fpu'],
            ['--fuzzing-safe', '--no-threads', '--no-baseline', '--no-ion'],
        ]
        if shellSupportsFlag(shellPath, "--ion-extra-checks"):
            basicFlagList.append(['--fuzzing-safe', '--no-threads', '--ion-eager',
                                  '--ion-check-range-analysis', '--ion-extra-checks', '--no-sse3'])
        return basicFlagList
    elif shellSupportsFlag(shellPath, "--ion-offthread-compile=off"):
        basicFlagList = [
            # Parts of this flag permutation come from:
            # https://hg.mozilla.org/mozilla-central/file/84bd8d9f4256/js/src/tests/lib/tests.py#l12
            # as well as other interesting flag combinations that have found / may find new bugs.
            ['--fuzzing-safe', '--ion-offthread-compile=off'],  # compareJIT uses this first flag set as the sole baseline when fuzzing
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline', '--no-ion'],
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-ion'], # Not in jit_test.py though, see bug 848906 comment 1
            ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-fpu'],
        ]
        if shellSupportsFlag(shellPath, "--thread-count=1"):
            basicFlagList.append(['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager'])
            # Range analysis had only started to stabilize around the time when --no-sse3 landed.
            if shellSupportsFlag(shellPath, '--no-sse3'):
                basicFlagList.append(['--fuzzing-safe', '--ion-offthread-compile=off',
                                      '--ion-eager', '--ion-check-range-analysis', '--no-sse3'])
        return basicFlagList
    else:
        basicFlagList = [
            # Parts of this flag permutation come from:
            # https://hg.mozilla.org/mozilla-central/file/10932f3a0ba0/js/src/tests/lib/tests.py#l12
            # as well as other interesting flag combinations that have found / may find new bugs.
            ['--fuzzing-safe', '--ion-parallel-compile=off'],  # compareJIT uses this first flag set as the sole baseline when fuzzing
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline', '--no-ion'],
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--ion-eager'],  # Not in jit_test.py though...
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--baseline-eager'],
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--baseline-eager', '--no-ion'], # See bug 848906 comment 1
            ['--fuzzing-safe', '--ion-parallel-compile=off', '--baseline-eager', '--no-fpu'],
        ]
        if shellSupportsFlag(shellPath, "--thread-count=1"):
            basicFlagList.append(['--fuzzing-safe', '--ion-eager', '--ion-parallel-compile=off'])
            # Range analysis had only started to stabilize around the time when --no-sse3 landed.
            if shellSupportsFlag(shellPath, '--no-sse3'):
                basicFlagList.append(['--fuzzing-safe', '--ion-parallel-compile=off',
                                      '--ion-eager', '--ion-check-range-analysis', '--no-sse3'])
        return basicFlagList

# Consider adding a function (for compareJIT reduction) that takes a flag set
# and returns all its (meaningful) subsets.

def testRandomFlags():
    import sys
    for i in range(100):
        print ' '.join(randomFlagSet(sys.argv[1]))

if __name__ == "__main__":
    testRandomFlags()
<|endoftext|>


<commit_before>"""
Read/Write AMQP frames over network transports.

2009-01-14 Barry Pederson <bp@barryp.org>

"""
# Copyright (C) 2009 Barry Pederson <bp@barryp.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301

import re
import socket

#
# See if Python 2.6+ SSL support is available
#
try:
    import ssl
    HAVE_PY26_SSL = True
except:
    HAVE_PY26_SSL = False

try:
    bytes
except:
    # Python 2.5 and lower
    bytes = str

from struct import pack, unpack

AMQP_PORT = 5672

# Yes, Advanced Message Queuing Protocol Protocol is redundant
AMQP_PROTOCOL_HEADER = 'AMQP\x01\x01\x09\x01'.encode('latin_1')

# Match things like: [fe80::1]:5432, from RFC 2732
IPV6_LITERAL = re.compile(r'\[([\.0-9a-f:]+)\](?::(\d+))?')

class _AbstractTransport(object):
    """
    Common superclass for TCP and SSL transports

    """
    def __init__(self, host, connect_timeout):
        msg = 'socket.getaddrinfo() for %s returned an empty list' % host
        port = AMQP_PORT

        m = IPV6_LITERAL.match(host)
        if m:
            host = m.group(1)
            if m.group(2):
                port = int(m.group(2))
        else:
            if ':' in host:
                host, port = host.rsplit(':', 1)
                port = int(port)

        self.sock = None
        for res in socket.getaddrinfo(host, port, 0, 0, socket.SOL_TCP):
            af, socktype, proto, canonname, sa = res
            try:
                self.sock = socket.socket(af, socktype, proto)
                self.sock.settimeout(connect_timeout)
                self.sock.connect(sa)
            except socket.error, msg:
                self.sock.close()
                self.sock = None
                continue
            break

        if not self.sock:
            # Didn't connect, return the most recent error message
            raise socket.error, msg

        self.sock.settimeout(None)

        self._setup_transport()

        self._write(AMQP_PROTOCOL_HEADER)

    def __del__(self):
        self.close()

    def _read(self, n):
        """
        Read exactly n bytes from the peer

        """
        raise NotImplementedError('Must be overriden in subclass')

    def _setup_transport(self):
        """
        Do any additional initialization of the class (used
        by the subclasses).

        """
        pass

    def _write(self, s):
        """
        Completely write a string to the peer.

        """
        raise NotImplementedError('Must be overriden in subclass')

    def close(self):
        if self.sock is not None:
            self.sock.close()
            self.sock = None

    def read_frame(self):
        """
        Read an AMQP frame.

        """
        frame_type, channel, size = unpack('>BHI', self._read(7))
        payload = self._read(size)
        ch = ord(self._read(1))
        if ch == 206: # '\xce'
            return frame_type, channel, payload
        else:
            raise Exception('Framing Error, received 0x%02x while expecting 0xce' % ch)

    def write_frame(self, frame_type, channel, payload):
        """
        Write out an AMQP frame.

        """
        size = len(payload)
        self._write(pack('>BHI%dsB' % size,
            frame_type, channel, size, payload, 0xce))

class SSLTransport(_AbstractTransport):
    """
    Transport that works over SSL

    """
    def __init__(self, host, connect_timeout, ssl):
        if isinstance(ssl, dict):
            self.sslopts = ssl

        super(SSLTransport, self).__init__(host, connect_timeout)

    def _setup_transport(self):
        """
        Wrap the socket in an SSL object, either the
        new Python 2.6 version, or the older Python 2.5 and
        lower version.

        """
        if HAVE_PY26_SSL:
            if hasattr(self, 'sslopts'):
                self.sslobj = ssl.wrap_socket(self.sock, **self.sslopts)
            else:
                self.sslobj = ssl.wrap_socket(self.sock)
            self.sslobj.do_handshake()
        else:
            self.sslobj = socket.ssl(self.sock)

    def _read(self, n):
        """
        It seems that SSL Objects read() method may not supply as much
        as you're asking for, at least with extremely large messages.
        somewhere > 16K - found this in the test_channel.py test_large
        unittest.

        """
        result = self.sslobj.read(n)

        while len(result) < n:
            s = self.sslobj.read(n - len(result))
            if not s:
                raise IOError('Socket closed')
            result += s

        return result

    def _write(self, s):
        """
        Write a string out to the SSL socket fully.

        """
        while s:
            n = self.sslobj.write(s)
            if not n:
                raise IOError('Socket closed')
            s = s[n:]

class TCPTransport(_AbstractTransport):
    """
    Transport that deals directly with TCP socket.

    """
    def _setup_transport(self):
        """
        Setup to _write() directly to the socket, and
        do our own buffered reads.

        """
        self._write = self.sock.sendall
        self._read_buffer = bytes()

    def _read(self, n):
        """
        Read exactly n bytes from the socket

        """
        while len(self._read_buffer) < n:
            s = self.sock.recv(65536)
            if not s:
                raise IOError('Socket closed')
            self._read_buffer += s

        result = self._read_buffer[:n]
        self._read_buffer = self._read_buffer[n:]

        return result

def create_transport(host, connect_timeout, ssl=False):
    """
    Given a few parameters from the Connection constructor,
    select and create a subclass of _AbstractTransport.

    """
    if ssl:
        return SSLTransport(host, connect_timeout, ssl)
    else:
        return TCPTransport(host, connect_timeout)
<commit_msg>Shutdown a socket before closing it, to ensure pending transmissions aren't discarded. (Issue 30, thanks to chr...@pianocktail.org for pointing this out).<commit_after>"""
Read/Write AMQP frames over network transports.

2009-01-14 Barry Pederson <bp@barryp.org>

"""
# Copyright (C) 2009 Barry Pederson <bp@barryp.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301

import re
import socket

#
# See if Python 2.6+ SSL support is available
#
try:
    import ssl
    HAVE_PY26_SSL = True
except:
    HAVE_PY26_SSL = False

try:
    bytes
except:
    # Python 2.5 and lower
    bytes = str

from struct import pack, unpack

AMQP_PORT = 5672

# Yes, Advanced Message Queuing Protocol Protocol is redundant
AMQP_PROTOCOL_HEADER = 'AMQP\x01\x01\x09\x01'.encode('latin_1')

# Match things like: [fe80::1]:5432, from RFC 2732
IPV6_LITERAL = re.compile(r'\[([\.0-9a-f:]+)\](?::(\d+))?')

class _AbstractTransport(object):
    """
    Common superclass for TCP and SSL transports

    """
    def __init__(self, host, connect_timeout):
        msg = 'socket.getaddrinfo() for %s returned an empty list' % host
        port = AMQP_PORT

        m = IPV6_LITERAL.match(host)
        if m:
            host = m.group(1)
            if m.group(2):
                port = int(m.group(2))
        else:
            if ':' in host:
                host, port = host.rsplit(':', 1)
                port = int(port)

        self.sock = None
        for res in socket.getaddrinfo(host, port, 0, 0, socket.SOL_TCP):
            af, socktype, proto, canonname, sa = res
            try:
                self.sock = socket.socket(af, socktype, proto)
                self.sock.settimeout(connect_timeout)
                self.sock.connect(sa)
            except socket.error, msg:
                self.sock.close()
                self.sock = None
                continue
            break

        if not self.sock:
            # Didn't connect, return the most recent error message
            raise socket.error, msg

        self.sock.settimeout(None)

        self._setup_transport()

        self._write(AMQP_PROTOCOL_HEADER)

    def __del__(self):
        self.close()

    def _read(self, n):
        """
        Read exactly n bytes from the peer

        """
        raise NotImplementedError('Must be overriden in subclass')

    def _setup_transport(self):
        """
        Do any additional initialization of the class (used
        by the subclasses).

        """
        pass

    def _shutdown_transport(self):
        """
        Do any preliminary work in shutting down the connection.

        """
        pass

    def _write(self, s):
        """
        Completely write a string to the peer.

        """
        raise NotImplementedError('Must be overriden in subclass')

    def close(self):
        if self.sock is not None:
            self._shutdown_transport()
            # Call shutdown first to make sure that pending messages
            # reach the AMQP broker if the program exits after
            # calling this method.
            self.sock.shutdown(socket.SHUT_RDWR)
            self.sock.close()
            self.sock = None

    def read_frame(self):
        """
        Read an AMQP frame.

        """
        frame_type, channel, size = unpack('>BHI', self._read(7))
        payload = self._read(size)
        ch = ord(self._read(1))
        if ch == 206: # '\xce'
            return frame_type, channel, payload
        else:
            raise Exception('Framing Error, received 0x%02x while expecting 0xce' % ch)

    def write_frame(self, frame_type, channel, payload):
        """
        Write out an AMQP frame.

        """
        size = len(payload)
        self._write(pack('>BHI%dsB' % size,
            frame_type, channel, size, payload, 0xce))

class SSLTransport(_AbstractTransport):
    """
    Transport that works over SSL

    """
    def __init__(self, host, connect_timeout, ssl):
        if isinstance(ssl, dict):
            self.sslopts = ssl

        self.sslobj = None

        super(SSLTransport, self).__init__(host, connect_timeout)

    def _setup_transport(self):
        """
        Wrap the socket in an SSL object, either the
        new Python 2.6 version, or the older Python 2.5 and
        lower version.

        """
        if HAVE_PY26_SSL:
            if hasattr(self, 'sslopts'):
                self.sslobj = ssl.wrap_socket(self.sock, **self.sslopts)
            else:
                self.sslobj = ssl.wrap_socket(self.sock)
            self.sslobj.do_handshake()
        else:
            self.sslobj = socket.ssl(self.sock)

    def _shutdown_transport(self):
        """
        Unwrap a Python 2.6 SSL socket, so we can call shutdown()

        """
        if HAVE_PY26_SSL and (self.sslobj is not None):
            self.sock = self.sslobj.unwrap()
            self.sslobj = None

    def _read(self, n):
        """
        It seems that SSL Objects read() method may not supply as much
        as you're asking for, at least with extremely large messages.
        somewhere > 16K - found this in the test_channel.py test_large
        unittest.

        """
        result = self.sslobj.read(n)

        while len(result) < n:
            s = self.sslobj.read(n - len(result))
            if not s:
                raise IOError('Socket closed')
            result += s

        return result

    def _write(self, s):
        """
        Write a string out to the SSL socket fully.

        """
        while s:
            n = self.sslobj.write(s)
            if not n:
                raise IOError('Socket closed')
            s = s[n:]

class TCPTransport(_AbstractTransport):
    """
    Transport that deals directly with TCP socket.

    """
    def _setup_transport(self):
        """
        Setup to _write() directly to the socket, and
        do our own buffered reads.

        """
        self._write = self.sock.sendall
        self._read_buffer = bytes()

    def _read(self, n):
        """
        Read exactly n bytes from the socket

        """
        while len(self._read_buffer) < n:
            s = self.sock.recv(65536)
            if not s:
                raise IOError('Socket closed')
            self._read_buffer += s

        result = self._read_buffer[:n]
        self._read_buffer = self._read_buffer[n:]

        return result

def create_transport(host, connect_timeout, ssl=False):
    """
    Given a few parameters from the Connection constructor,
    select and create a subclass of _AbstractTransport.

    """
    if ssl:
        return SSLTransport(host, connect_timeout, ssl)
    else:
        return TCPTransport(host, connect_timeout)
<|endoftext|>


<commit_before>from annotations import stims
from annotations.annotators import Annotator
import numpy as np
from scipy import fft
import matplotlib.pyplot as plt
from annotations.core import Note, Event

class AudioAnnotator(Annotator):

    target = stims.AudioStim

class STFTAnnotator(AudioAnnotator):
    ''' Short-time Fourier Transform annotator. '''
    def __init__(self, frame_size=0.5, hop_size=0.1, bins=5,
                 spectrogram=False):
        self.frame_size = frame_size
        self.hop_size = hop_size
        self.spectrogram = spectrogram
        self.freq_bins = bins

    def stft(self, stim):
        ''' code adapted from http://stackoverflow.com/questions/2459295/invertible-stft-and-istft-in-python '''
        x = stim.data
        framesamp = int(self.frame_size*stim.sampling_rate)
        hopsamp = int(self.hop_size*stim.sampling_rate)
        w = np.hanning(framesamp)
        X = np.array([fft(w*x[i:(i+framesamp)]) for i in range(0, len(x)-framesamp, hopsamp)])
        nyquist_lim = X.shape[1]/2
        X = np.log(X[:, :nyquist_lim])
        X = np.absolute(X)
        if self.spectrogram:
            bins = np.fft.fftfreq(framesamp, d=1./stim.sampling_rate)
            bins = bins[:nyquist_lim]
            plt.imshow(X.T, origin='lower', aspect='auto',
                       interpolation='nearest', cmap='RdYlBu_r',
                       extent=[0, stim.duration, bins.min(), bins.max()])
            plt.xlabel('Time')
            plt.ylabel('Frequency')
            plt.colorbar()
            plt.show()
        return X

    def apply(self, stim):
        data = self.stft(stim)
        events = []
        time_bins = np.arange(0., stim.duration-self.frame_size, self.hop_size)

        if isinstance(self.freq_bins, int):
            bins = []
            bin_size = data.shape[1] / self.freq_bins
            for i in range(self.freq_bins):
                bins.append((i*bin_size, (i+1)*bin_size))
            self.freq_bins = bins

        for i, tb in enumerate(time_bins):
            ev = Event(onset=tb, duration=self.frame_size)
            note_data = {}
            for fb in self.freq_bins:
                label = '%d_%d' % fb
                start, stop = fb
                val = data[i, start:stop].mean()
                if np.isinf(val):
                    val = 0.
                note_data[label] = val
            ev.add_note(Note(stim, self, note_data))
            events.append(ev)
        return events
<commit_msg>mpl should be optional<commit_after>from annotations import stims
from annotations.annotators import Annotator
import numpy as np
from scipy import fft
from annotations.core import Note, Event

class AudioAnnotator(Annotator):

    target = stims.AudioStim

class STFTAnnotator(AudioAnnotator):
    ''' Short-time Fourier Transform annotator. '''
    def __init__(self, frame_size=0.5, hop_size=0.1, bins=5,
                 spectrogram=False):
        self.frame_size = frame_size
        self.hop_size = hop_size
        self.spectrogram = spectrogram
        self.freq_bins = bins

    def stft(self, stim):
        ''' code adapted from http://stackoverflow.com/questions/2459295/invertible-stft-and-istft-in-python '''
        x = stim.data
        framesamp = int(self.frame_size*stim.sampling_rate)
        hopsamp = int(self.hop_size*stim.sampling_rate)
        w = np.hanning(framesamp)
        X = np.array([fft(w*x[i:(i+framesamp)]) for i in range(0, len(x)-framesamp, hopsamp)])
        nyquist_lim = X.shape[1]/2
        X = np.log(X[:, :nyquist_lim])
        X = np.absolute(X)
        if self.spectrogram:
            import matplotlib.pyplot as plt
            bins = np.fft.fftfreq(framesamp, d=1./stim.sampling_rate)
            bins = bins[:nyquist_lim]
            plt.imshow(X.T, origin='lower', aspect='auto',
                       interpolation='nearest', cmap='RdYlBu_r',
                       extent=[0, stim.duration, bins.min(), bins.max()])
            plt.xlabel('Time')
            plt.ylabel('Frequency')
            plt.colorbar()
            plt.show()
        return X

    def apply(self, stim):
        data = self.stft(stim)
        events = []
        time_bins = np.arange(0., stim.duration-self.frame_size, self.hop_size)

        if isinstance(self.freq_bins, int):
            bins = []
            bin_size = data.shape[1] / self.freq_bins
            for i in range(self.freq_bins):
                bins.append((i*bin_size, (i+1)*bin_size))
            self.freq_bins = bins

        for i, tb in enumerate(time_bins):
            ev = Event(onset=tb, duration=self.frame_size)
            note_data = {}
            for fb in self.freq_bins:
                label = '%d_%d' % fb
                start, stop = fb
                val = data[i, start:stop].mean()
                if np.isinf(val):
                    val = 0.
                note_data[label] = val
            ev.add_note(Note(stim, self, note_data))
            events.append(ev)
        return events
<|endoftext|>


<commit_before>from src.utils import customLogger
from src.api.workers import mongoDriver
from dateutil.relativedelta import relativedelta
import datetime
from datetime import datetime
import calendar

workerLogger = customLogger.getWorkerLogger()
def getResponse(hostname = None, scale = None, time_from = None, time_to = None):
  if hostname is None or scale is None or time_from is None or time_to is None:
    workerLogger.error("Invalid input parameters")
    return None, False, "Invalid input parameters"

  if scale == 1:
    modulo = 60
  elif scale == 2:
    modulo = 3600
  elif scale == 3:
    modulo = 86400
  elif scale == 4:
    modulo = 604800
  elif scale == 5:
    modulo = 2592000
  elif scale == 6:
    modulo = 31536000
  else:
    workerLogger.error("Invalid scale")
    return None, False, "Invalid scale"

  if scale != 5:
    dbResDictAll = None
    dbResDictAll, errmsg = mongoDriver.getVisitArrAll(vhost = hostname, modulo = modulo,\
                             startDate = time_from, endDate = time_to)
    if errmsg is None:
      workerLogger.error("Error fetching response from backend")
      return None, False, "Error fetching response from backend"

    dbResDictHtml = None
    dbResDictHtml, errmsg = mongoDriver.getVisitArrHtml(vhost = hostname, modulo = modulo,\
                             startDate = time_from, endDate = time_to)
    if errmsg is None:
      workerLogger.error("Error fetching response from backend")
      return None, False, "Error fetching response from backend"

    visitDict = populateDict(dbResDictAll, dbResDictHtml, time_from, time_to, modulo)

  else:
    i = 0
    dbResDictAll  = []
    dbResDictHtml = []
    time_from_datetime = datetime.utcfromtimestamp(time_from)
    startDate = calendar.timegm((time_from_datetime + relativedelta(day=1)).timetuple()) -1
    endDate = calendar.timegm((time_from_datetime + relativedelta(day=1, months=+1, days=-1)).timetuple())
    modulo = endDate - startDate
    visitDict = []
    while i < 12:
      print "startDate:: ", startDate
      print "endDate:: ", endDate
      tmpResDictAll = None
      modulo = endDate - startDate
      tmpResDictAll, errmsg = mongoDriver.getVisitArrAll(vhost = hostname, modulo = modulo,\
                             startDate = startDate, endDate = endDate)

      dbResDictAll.append({})
      visitDict.append({})
      if len(tmpResDictAll) > 0:
        dbResDictAll[i]['count'] = tmpResDictAll[0]['count']
      else:
        dbResDictAll[i]['count'] = 0

      if errmsg is None:
        workerLogger.error("Error fetching response from backend")
        return None, False, "Error fetching response from backend"

      tmpResDictHtml = None
      tmpResDictHtml, errmsg = mongoDriver.getVisitArrHtml(vhost = hostname, modulo = modulo,\
                             startDate = startDate, endDate = endDate)
      if errmsg is None:
        workerLogger.error("Error fetching response from backend")
        return None, False, "Error fetching response from backend"

      dbResDictHtml.append({})
      if len(tmpResDictHtml) > 0:
        dbResDictHtml[i]['count'] = tmpResDictHtml[0]['count']
      else:
        dbResDictHtml[i]['count'] = 0

      visitDict[i]['visit_time'] = startDate
      visitDict[i]['num_visits_all'] = dbResDictAll[i]['count']
      visitDict[i]['num_visits_html'] = dbResDictHtml[i]['count']

      startDate = calendar.timegm((time_from_datetime + relativedelta(day=1, months=+i)).timetuple()) - 1
      endDate = calendar.timegm((time_from_datetime + relativedelta(day=1, months=+i, days=-1)).timetuple())
      i = i + 1

  #print "visit dict is : ", visitAllDict
  resp_dict = {}
  resp_dict['resp_struct'] = {}
  resp_dict['resp_struct']['visit_struct'] = visitDict
  print "resp dict:: ", resp_dict
  cnt = len(resp_dict["resp_struct"]["visit_struct"])
  print "count is : ", cnt, "***********\n"
  return resp_dict, True, ""

def populateDict(dbResDictAll, dbResDictHtml, time_from, time_to, modulo):
  idx = time_from
  i = 0
  tmp_dict = {}
  tmp_count = len(dbResDictAll)
  while True:
    if tmp_count == 0:
      break
    tmp_dict[str(dbResDictAll[i]['_id'])] = dbResDictAll[i]['count']
    i = i + 1
    if i == tmp_count:
      break
  print "tmp dict:: ", tmp_dict

  i = 0
  tmp_dict2 = {}
  tmp_count2 = len(dbResDictHtml)
  while True:
    if tmp_count2 == 0:
      break
    tmp_dict2[str(dbResDictHtml[i]['_id'])] = dbResDictHtml[i]['count']
    i = i + 1
    if i == tmp_count2:
      break
  print "tmp dict html:: ", tmp_dict2

  i = 0
  visitDict = []
  while True:
    visitDict.append({})
    visitDict[i]['visit_time'] = idx
    tmpDictIdx = str(idx) + ".0"
    if tmpDictIdx in tmp_dict:
      visitDict[i]['num_visits_all'] = tmp_dict[tmpDictIdx]
    else:
      visitDict[i]['num_visits_all'] = 0

    if tmpDictIdx in tmp_dict2:
      visitDict[i]['num_visits_html'] = tmp_dict2[tmpDictIdx]
    else:
      visitDict[i]['num_visits_html'] = 0

    idx = idx + modulo
    i = i + 1
    if idx > time_to:
      break
  return visitDict

<commit_msg>fix year<commit_after>from src.utils import customLogger
from src.api.workers import mongoDriver
from dateutil.relativedelta import relativedelta
import datetime
from datetime import datetime
import calendar

workerLogger = customLogger.getWorkerLogger()
def getResponse(hostname = None, scale = None, time_from = None, time_to = None):
  if hostname is None or scale is None or time_from is None or time_to is None:
    workerLogger.error("Invalid input parameters")
    return None, False, "Invalid input parameters"

  if scale == 1:
    modulo = 60
  elif scale == 2:
    modulo = 3600
  elif scale == 3:
    modulo = 86400
  elif scale == 4:
    modulo = 604800
  elif scale == 5:
    modulo = 2592000
  elif scale == 6:
    modulo = 31536000
  else:
    workerLogger.error("Invalid scale")
    return None, False, "Invalid scale"

  if scale != 5:
    dbResDictAll = None
    dbResDictAll, errmsg = mongoDriver.getVisitArrAll(vhost = hostname, modulo = modulo,\
                             startDate = time_from, endDate = time_to)
    if errmsg is None:
      workerLogger.error("Error fetching response from backend")
      return None, False, "Error fetching response from backend"

    dbResDictHtml = None
    dbResDictHtml, errmsg = mongoDriver.getVisitArrHtml(vhost = hostname, modulo = modulo,\
                             startDate = time_from, endDate = time_to)
    if errmsg is None:
      workerLogger.error("Error fetching response from backend")
      return None, False, "Error fetching response from backend"

    visitDict = populateDict(dbResDictAll, dbResDictHtml, time_from, time_to, modulo)

  else:
    i = 0
    dbResDictAll  = []
    dbResDictHtml = []
    time_from_datetime = datetime.utcfromtimestamp(time_from)
    startDate = calendar.timegm((time_from_datetime + relativedelta(day=1)).timetuple())
    endDate = calendar.timegm((time_from_datetime + relativedelta(day=1, months=+1, days=-1)).timetuple()) -1
    modulo = endDate - startDate
    visitDict = []
    while i < 12:
      print "startDate:: ", startDate
      print "endDate:: ", endDate
      tmpResDictAll = None
      modulo = endDate - startDate
      tmpResDictAll, errmsg = mongoDriver.getVisitArrAll(vhost = hostname, modulo = modulo,\
                             startDate = startDate, endDate = endDate)

      dbResDictAll.append({})
      visitDict.append({})
      if len(tmpResDictAll) > 0:
        dbResDictAll[i]['count'] = tmpResDictAll[0]['count']
      else:
        dbResDictAll[i]['count'] = 0

      if errmsg is None:
        workerLogger.error("Error fetching response from backend")
        return None, False, "Error fetching response from backend"

      tmpResDictHtml = None
      tmpResDictHtml, errmsg = mongoDriver.getVisitArrHtml(vhost = hostname, modulo = modulo,\
                             startDate = startDate, endDate = endDate)
      if errmsg is None:
        workerLogger.error("Error fetching response from backend")
        return None, False, "Error fetching response from backend"

      dbResDictHtml.append({})
      if len(tmpResDictHtml) > 0:
        dbResDictHtml[i]['count'] = tmpResDictHtml[0]['count']
      else:
        dbResDictHtml[i]['count'] = 0

      visitDict[i]['visit_time'] = startDate
      visitDict[i]['num_visits_all'] = dbResDictAll[i]['count']
      visitDict[i]['num_visits_html'] = dbResDictHtml[i]['count']

      i = i + 1
      startDate = calendar.timegm((time_from_datetime + relativedelta(day=1, months=+i)).timetuple())
      endDate = calendar.timegm((time_from_datetime + relativedelta(day=1, months=+i, days=-1)).timetuple()) -1

  #print "visit dict is : ", visitAllDict
  resp_dict = {}
  resp_dict['resp_struct'] = {}
  resp_dict['resp_struct']['visit_struct'] = visitDict
  print "resp dict:: ", resp_dict
  cnt = len(resp_dict["resp_struct"]["visit_struct"])
  print "count is : ", cnt, "***********\n"
  return resp_dict, True, ""

def populateDict(dbResDictAll, dbResDictHtml, time_from, time_to, modulo):
  idx = time_from
  i = 0
  tmp_dict = {}
  tmp_count = len(dbResDictAll)
  while True:
    if tmp_count == 0:
      break
    tmp_dict[str(dbResDictAll[i]['_id'])] = dbResDictAll[i]['count']
    i = i + 1
    if i == tmp_count:
      break
  print "tmp dict:: ", tmp_dict

  i = 0
  tmp_dict2 = {}
  tmp_count2 = len(dbResDictHtml)
  while True:
    if tmp_count2 == 0:
      break
    tmp_dict2[str(dbResDictHtml[i]['_id'])] = dbResDictHtml[i]['count']
    i = i + 1
    if i == tmp_count2:
      break
  print "tmp dict html:: ", tmp_dict2

  i = 0
  visitDict = []
  while True:
    visitDict.append({})
    visitDict[i]['visit_time'] = idx
    tmpDictIdx = str(idx) + ".0"
    if tmpDictIdx in tmp_dict:
      visitDict[i]['num_visits_all'] = tmp_dict[tmpDictIdx]
    else:
      visitDict[i]['num_visits_all'] = 0

    if tmpDictIdx in tmp_dict2:
      visitDict[i]['num_visits_html'] = tmp_dict2[tmpDictIdx]
    else:
      visitDict[i]['num_visits_html'] = 0

    idx = idx + modulo
    i = i + 1
    if idx > time_to:
      break
  return visitDict

<|endoftext|>


<commit_before><commit_msg>Added app property which selects self._app or current_app<commit_after><|endoftext|>


<commit_before>"""Module to handle messages from Home Assistant cloud."""
import asyncio
import logging

from aiohttp import hdrs, client_exceptions, WSMsgType

from homeassistant.const import EVENT_HOMEASSISTANT_STOP
from homeassistant.components.alexa import smart_home as alexa
from homeassistant.components.google_assistant import smart_home as ga
from homeassistant.util.decorator import Registry
from homeassistant.helpers.aiohttp_client import async_get_clientsession
from . import auth_api
from .const import MESSAGE_EXPIRATION

HANDLERS = Registry()
_LOGGER = logging.getLogger(__name__)

STATE_CONNECTING = 'connecting'
STATE_CONNECTED = 'connected'
STATE_DISCONNECTED = 'disconnected'

class UnknownHandler(Exception):
    """Exception raised when trying to handle unknown handler."""

class CloudIoT:
    """Class to manage the IoT connection."""

    def __init__(self, cloud):
        """Initialize the CloudIoT class."""
        self.cloud = cloud
        # The WebSocket client
        self.client = None
        # Scheduled sleep task till next connection retry
        self.retry_task = None
        # Boolean to indicate if we wanted the connection to close
        self.close_requested = False
        # The current number of attempts to connect, impacts wait time
        self.tries = 0
        # Current state of the connection
        self.state = STATE_DISCONNECTED

    @asyncio.coroutine
    def connect(self):
        """Connect to the IoT broker."""
        hass = self.cloud.hass
        if self.cloud.subscription_expired:
            # Try refreshing the token to see if it is still expired.
            yield from hass.async_add_job(auth_api.check_token, self.cloud)

            if self.cloud.subscription_expired:
                hass.components.persistent_notification.async_create(
                    MESSAGE_EXPIRATION, 'Subscription expired',
                    'cloud_subscription_expired')
                self.state = STATE_DISCONNECTED
                return

        if self.state == STATE_CONNECTED:
            raise RuntimeError('Already connected')

        @asyncio.coroutine
        def _handle_hass_stop(event):
            """Handle Home Assistant shutting down."""
            nonlocal remove_hass_stop_listener
            remove_hass_stop_listener = None
            yield from self.disconnect()

        self.state = STATE_CONNECTING
        self.close_requested = False
        remove_hass_stop_listener = hass.bus.async_listen_once(
            EVENT_HOMEASSISTANT_STOP, _handle_hass_stop)
        session = async_get_clientsession(self.cloud.hass)
        client = None
        disconnect_warn = None

        try:
            yield from hass.async_add_job(auth_api.check_token, self.cloud)

            self.client = client = yield from session.ws_connect(
                self.cloud.relayer, heartbeat=55, headers={
                    hdrs.AUTHORIZATION:
                        'Bearer {}'.format(self.cloud.id_token)
                })
            self.tries = 0

            _LOGGER.info('Connected')
            self.state = STATE_CONNECTED

            while not client.closed:
                msg = yield from client.receive()

                if msg.type in (WSMsgType.ERROR, WSMsgType.CLOSED,
                                WSMsgType.CLOSING):
                    disconnect_warn = 'Connection cancelled.'
                    break

                elif msg.type != WSMsgType.TEXT:
                    disconnect_warn = 'Received non-Text message: {}'.format(
                        msg.type)
                    break

                try:
                    msg = msg.json()
                except ValueError:
                    disconnect_warn = 'Received invalid JSON.'
                    break

                _LOGGER.debug('Received message: %s', msg)

                response = {
                    'msgid': msg['msgid'],
                }
                try:
                    result = yield from async_handle_message(
                        hass, self.cloud, msg['handler'], msg['payload'])

                    # No response from handler
                    if result is None:
                        continue

                    response['payload'] = result

                except UnknownHandler:
                    response['error'] = 'unknown-handler'

                except Exception:  # pylint: disable=broad-except
                    _LOGGER.exception('Error handling message')
                    response['error'] = 'exception'

                _LOGGER.debug('Publishing message: %s', response)
                yield from client.send_json(response)

        except auth_api.CloudError:
            _LOGGER.warning('Unable to connect: Unable to refresh token.')

        except client_exceptions.WSServerHandshakeError as err:
            if err.code == 401:
                disconnect_warn = 'Invalid auth.'
                self.close_requested = True
                # Should we notify user?
            else:
                _LOGGER.warning('Unable to connect: %s', err)

        except client_exceptions.ClientError as err:
            _LOGGER.warning('Unable to connect: %s', err)

        except Exception:  # pylint: disable=broad-except
            if not self.close_requested:
                _LOGGER.exception('Unexpected error')

        finally:
            if disconnect_warn is not None:
                _LOGGER.warning('Connection closed: %s', disconnect_warn)

            if remove_hass_stop_listener is not None:
                remove_hass_stop_listener()

            if client is not None:
                self.client = None
                yield from client.close()

            if self.close_requested:
                self.state = STATE_DISCONNECTED

            else:
                self.state = STATE_CONNECTING
                self.tries += 1

                try:
                    # Sleep 0, 5, 10, 15 … up to 30 seconds between retries
                    self.retry_task = hass.async_add_job(asyncio.sleep(
                        min(30, (self.tries - 1) * 5), loop=hass.loop))
                    yield from self.retry_task
                    self.retry_task = None
                    hass.async_add_job(self.connect())
                except asyncio.CancelledError:
                    # Happens if disconnect called
                    pass

    @asyncio.coroutine
    def disconnect(self):
        """Disconnect the client."""
        self.close_requested = True

        if self.client is not None:
            yield from self.client.close()
        elif self.retry_task is not None:
            self.retry_task.cancel()

@asyncio.coroutine
def async_handle_message(hass, cloud, handler_name, payload):
    """Handle incoming IoT message."""
    handler = HANDLERS.get(handler_name)

    if handler is None:
        raise UnknownHandler()

    return (yield from handler(hass, cloud, payload))

@HANDLERS.register('alexa')
@asyncio.coroutine
def async_handle_alexa(hass, cloud, payload):
    """Handle an incoming IoT message for Alexa."""
    result = yield from alexa.async_handle_message(hass, cloud.alexa_config,
                                                   payload)
    return result

@HANDLERS.register('google_assistant')
@asyncio.coroutine
def async_handle_google_assistant(hass, cloud, payload):
    """Handle an incoming IoT message for Google Assistant."""
    result = yield from ga.async_handle_message(hass, cloud.gass_config,
                                                payload)
    return result

@HANDLERS.register('cloud')
@asyncio.coroutine
def async_handle_cloud(hass, cloud, payload):
    """Handle an incoming IoT message for cloud component."""
    action = payload['action']

    if action == 'logout':
        yield from cloud.logout()
        _LOGGER.error('You have been logged out from Home Assistant cloud: %s',
                      payload['reason'])
    else:
        _LOGGER.warning('Received unknown cloud action: %s', action)

    return None
<commit_msg>Google Assistant -> Google Actions (#11442)<commit_after>"""Module to handle messages from Home Assistant cloud."""
import asyncio
import logging

from aiohttp import hdrs, client_exceptions, WSMsgType

from homeassistant.const import EVENT_HOMEASSISTANT_STOP
from homeassistant.components.alexa import smart_home as alexa
from homeassistant.components.google_assistant import smart_home as ga
from homeassistant.util.decorator import Registry
from homeassistant.helpers.aiohttp_client import async_get_clientsession
from . import auth_api
from .const import MESSAGE_EXPIRATION

HANDLERS = Registry()
_LOGGER = logging.getLogger(__name__)

STATE_CONNECTING = 'connecting'
STATE_CONNECTED = 'connected'
STATE_DISCONNECTED = 'disconnected'

class UnknownHandler(Exception):
    """Exception raised when trying to handle unknown handler."""

class CloudIoT:
    """Class to manage the IoT connection."""

    def __init__(self, cloud):
        """Initialize the CloudIoT class."""
        self.cloud = cloud
        # The WebSocket client
        self.client = None
        # Scheduled sleep task till next connection retry
        self.retry_task = None
        # Boolean to indicate if we wanted the connection to close
        self.close_requested = False
        # The current number of attempts to connect, impacts wait time
        self.tries = 0
        # Current state of the connection
        self.state = STATE_DISCONNECTED

    @asyncio.coroutine
    def connect(self):
        """Connect to the IoT broker."""
        hass = self.cloud.hass
        if self.cloud.subscription_expired:
            # Try refreshing the token to see if it is still expired.
            yield from hass.async_add_job(auth_api.check_token, self.cloud)

            if self.cloud.subscription_expired:
                hass.components.persistent_notification.async_create(
                    MESSAGE_EXPIRATION, 'Subscription expired',
                    'cloud_subscription_expired')
                self.state = STATE_DISCONNECTED
                return

        if self.state == STATE_CONNECTED:
            raise RuntimeError('Already connected')

        @asyncio.coroutine
        def _handle_hass_stop(event):
            """Handle Home Assistant shutting down."""
            nonlocal remove_hass_stop_listener
            remove_hass_stop_listener = None
            yield from self.disconnect()

        self.state = STATE_CONNECTING
        self.close_requested = False
        remove_hass_stop_listener = hass.bus.async_listen_once(
            EVENT_HOMEASSISTANT_STOP, _handle_hass_stop)
        session = async_get_clientsession(self.cloud.hass)
        client = None
        disconnect_warn = None

        try:
            yield from hass.async_add_job(auth_api.check_token, self.cloud)

            self.client = client = yield from session.ws_connect(
                self.cloud.relayer, heartbeat=55, headers={
                    hdrs.AUTHORIZATION:
                        'Bearer {}'.format(self.cloud.id_token)
                })
            self.tries = 0

            _LOGGER.info('Connected')
            self.state = STATE_CONNECTED

            while not client.closed:
                msg = yield from client.receive()

                if msg.type in (WSMsgType.ERROR, WSMsgType.CLOSED,
                                WSMsgType.CLOSING):
                    disconnect_warn = 'Connection cancelled.'
                    break

                elif msg.type != WSMsgType.TEXT:
                    disconnect_warn = 'Received non-Text message: {}'.format(
                        msg.type)
                    break

                try:
                    msg = msg.json()
                except ValueError:
                    disconnect_warn = 'Received invalid JSON.'
                    break

                _LOGGER.debug('Received message: %s', msg)

                response = {
                    'msgid': msg['msgid'],
                }
                try:
                    result = yield from async_handle_message(
                        hass, self.cloud, msg['handler'], msg['payload'])

                    # No response from handler
                    if result is None:
                        continue

                    response['payload'] = result

                except UnknownHandler:
                    response['error'] = 'unknown-handler'

                except Exception:  # pylint: disable=broad-except
                    _LOGGER.exception('Error handling message')
                    response['error'] = 'exception'

                _LOGGER.debug('Publishing message: %s', response)
                yield from client.send_json(response)

        except auth_api.CloudError:
            _LOGGER.warning('Unable to connect: Unable to refresh token.')

        except client_exceptions.WSServerHandshakeError as err:
            if err.code == 401:
                disconnect_warn = 'Invalid auth.'
                self.close_requested = True
                # Should we notify user?
            else:
                _LOGGER.warning('Unable to connect: %s', err)

        except client_exceptions.ClientError as err:
            _LOGGER.warning('Unable to connect: %s', err)

        except Exception:  # pylint: disable=broad-except
            if not self.close_requested:
                _LOGGER.exception('Unexpected error')

        finally:
            if disconnect_warn is not None:
                _LOGGER.warning('Connection closed: %s', disconnect_warn)

            if remove_hass_stop_listener is not None:
                remove_hass_stop_listener()

            if client is not None:
                self.client = None
                yield from client.close()

            if self.close_requested:
                self.state = STATE_DISCONNECTED

            else:
                self.state = STATE_CONNECTING
                self.tries += 1

                try:
                    # Sleep 0, 5, 10, 15 … up to 30 seconds between retries
                    self.retry_task = hass.async_add_job(asyncio.sleep(
                        min(30, (self.tries - 1) * 5), loop=hass.loop))
                    yield from self.retry_task
                    self.retry_task = None
                    hass.async_add_job(self.connect())
                except asyncio.CancelledError:
                    # Happens if disconnect called
                    pass

    @asyncio.coroutine
    def disconnect(self):
        """Disconnect the client."""
        self.close_requested = True

        if self.client is not None:
            yield from self.client.close()
        elif self.retry_task is not None:
            self.retry_task.cancel()

@asyncio.coroutine
def async_handle_message(hass, cloud, handler_name, payload):
    """Handle incoming IoT message."""
    handler = HANDLERS.get(handler_name)

    if handler is None:
        raise UnknownHandler()

    return (yield from handler(hass, cloud, payload))

@HANDLERS.register('alexa')
@asyncio.coroutine
def async_handle_alexa(hass, cloud, payload):
    """Handle an incoming IoT message for Alexa."""
    result = yield from alexa.async_handle_message(hass, cloud.alexa_config,
                                                   payload)
    return result

@HANDLERS.register('google_actions')
@asyncio.coroutine
def async_handle_google_assistant(hass, cloud, payload):
    """Handle an incoming IoT message for Google Assistant."""
    result = yield from ga.async_handle_message(hass, cloud.gass_config,
                                                payload)
    return result

@HANDLERS.register('cloud')
@asyncio.coroutine
def async_handle_cloud(hass, cloud, payload):
    """Handle an incoming IoT message for cloud component."""
    action = payload['action']

    if action == 'logout':
        yield from cloud.logout()
        _LOGGER.error('You have been logged out from Home Assistant cloud: %s',
                      payload['reason'])
    else:
        _LOGGER.warning('Received unknown cloud action: %s', action)

    return None
<|endoftext|>