<commit_before>#!/usr/bin/env python

def submit_experiment(kernel, Dw, nwf, ts, nx, ny, nz, nt, is_dp, outfile, target_dir, tgs, cs, exe_cmd):
  import os
  import subprocess
  from string import Template
  from utils import ensure_dir

  job_template=Template(
"""$exec_path --n-tests 2 --disable-source-point --npx 1 --npy 1 --npz 1 --nx $nx --ny $ny --nz $nz  --verbose 1 --target-ts $ts --nt $nt --target-kernel $kernel --cache-size $cs --thread-group-size $tgs --t-dim $Dw --num-wavefronts $nwf --mwd-type 2 | tee $outfile""")

  target_dir = os.path.join(os.path.abspath("."),target_dir)
  ensure_dir(target_dir)
  outpath = os.path.join(target_dir,outfile)

  if(is_dp==1):
    exec_path = os.path.join(os.path.abspath("."),"build_dp/mwd_kernel")
  else:
    exec_path = os.path.join(os.path.abspath("."),"build/mwd_kernel")

  job_cmd = job_template.substitute(Dw=Dw, nwf=nwf, cs=cs, tgs=tgs, nx=nx, ny=ny, nz=nz, nt=nt, kernel=kernel, ts=ts, outfile=outpath, exec_path=exec_path, target_dir=target_dir)

  job_cmd = exe_cmd + job_cmd

  print job_cmd
  sts = subprocess.call(job_cmd, shell=True)

def ts_test(target_dir, exp_name, cs, ts, kernel, tgs, N, group, dw_l):
  is_dp = 1

  if kernel==5: nwf = 20
  if kernel==1: nwf = 35
  for Dw in dw_l:
    exe_cmd = "export OMP_NUM_THREADS=10; likwid-perfctr -m -C S0:0-9 -s 0x03 -g %s " % (group)
    outfile=('ts%d_kernel%d_tgs%d_N%d_%s_TB%d.txt' % (ts, kernel, tgs,  N, group,Dw))
    nx = N
    ny = N
    nz = N
    nt =40 # int(max(10 * 4e9/(nx*ny*nz*3), 50))
    submit_experiment(kernel, Dw=Dw, nwf=nwf, ts=ts, nx=nx, ny=ny, nz=nz, nt=nt, is_dp=is_dp,
         outfile=outfile, target_dir=target_dir, tgs=tgs, cs=cs, exe_cmd=exe_cmd)

def main():

  from utils import create_project_tarball

  import socket
  #hostname = socket.gethostname()
  exp_name = "emmy_"
  exp_name = exp_name + "energy_vs_code_balance"#_at_%s" % (hostname)

  tarball_dir='results/'+exp_name
  create_project_tarball(tarball_dir, "project_"+exp_name)

  target_dir='results/' + exp_name

  cs = 4096
  dw_l = [1,3,5,7]
  ts_test(target_dir, exp_name, cs, ts=2, kernel=1, tgs=5, N=960, group='ENERGY', dw_l=dw_l)
  ts_test(target_dir, exp_name, cs, ts=2, kernel=1, tgs=5, N=960, group='MEM', dw_l=dw_l)

  ts_test(target_dir, exp_name, cs, ts=2, kernel=5, tgs=5, N=480, group='ENERGY', dw_l=dw_l)
  ts_test(target_dir, exp_name, cs, ts=2, kernel=5, tgs=5, N=480, group='MEM', dw_l=dw_l)

if __name__ == "__main__":
  main()
<commit_msg>autotuning frontlines at the energy vs code balance experiments<commit_after>#!/usr/bin/env python

def submit_experiment(kernel, Dw, nwf, ts, nx, ny, nz, nt, is_dp, outfile, target_dir, tgs, cs, exe_cmd):
  import os
  import subprocess
  from string import Template
  from utils import ensure_dir

  job_template=Template(
"""$exec_path --n-tests 2 --disable-source-point --npx 1 --npy 1 --npz 1 --nx $nx --ny $ny --nz $nz  --verbose 1 --target-ts $ts --nt $nt --target-kernel $kernel --cache-size $cs --thread-group-size $tgs --t-dim $Dw --num-wavefronts $nwf --mwd-type 2 | tee $outfile""")

  target_dir = os.path.join(os.path.abspath("."),target_dir)
  ensure_dir(target_dir)
  outpath = os.path.join(target_dir,outfile)

  if(is_dp==1):
    exec_path = os.path.join(os.path.abspath("."),"build_dp/mwd_kernel")
  else:
    exec_path = os.path.join(os.path.abspath("."),"build/mwd_kernel")

  job_cmd = job_template.substitute(Dw=Dw, nwf=nwf, cs=cs, tgs=tgs, nx=nx, ny=ny, nz=nz, nt=nt, kernel=kernel, ts=ts, outfile=outpath, exec_path=exec_path, target_dir=target_dir)

  job_cmd = exe_cmd + job_cmd

  print job_cmd
  sts = subprocess.call(job_cmd, shell=True)

def ts_test(target_dir, exp_name, cs, ts, kernel, tgs, N, group, dw_l):
  is_dp = 1

  nwf=-1
  for Dw in dw_l:
    exe_cmd = "export OMP_NUM_THREADS=10; likwid-perfctr -m -C S0:0-9 -s 0x03 -g %s " % (group)
    outfile=('ts%d_kernel%d_tgs%d_N%d_%s_TB%d.txt' % (ts, kernel, tgs,  N, group,Dw))
    nx = N
    ny = N
    nz = N
    nt =40 # int(max(10 * 4e9/(nx*ny*nz*3), 50))
    submit_experiment(kernel, Dw=Dw, nwf=nwf, ts=ts, nx=nx, ny=ny, nz=nz, nt=nt, is_dp=is_dp,
         outfile=outfile, target_dir=target_dir, tgs=tgs, cs=cs, exe_cmd=exe_cmd)

def main():

  from utils import create_project_tarball

  import socket
  #hostname = socket.gethostname()
  exp_name = "emmy_"
  exp_name = exp_name + "energy_vs_code_balance"#_at_%s" % (hostname)

  tarball_dir='results/'+exp_name
  create_project_tarball(tarball_dir, "project_"+exp_name)

  target_dir='results/' + exp_name

  cs = 4096
  dw_l = [1,3,5,7]
  ts_test(target_dir, exp_name, cs, ts=2, kernel=1, tgs=5, N=960, group='ENERGY', dw_l=dw_l)
  ts_test(target_dir, exp_name, cs, ts=2, kernel=1, tgs=5, N=960, group='MEM', dw_l=dw_l)

  ts_test(target_dir, exp_name, cs, ts=2, kernel=5, tgs=5, N=480, group='ENERGY', dw_l=dw_l)
  ts_test(target_dir, exp_name, cs, ts=2, kernel=5, tgs=5, N=480, group='MEM', dw_l=dw_l)

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>
import re
import sys

from .ansi import AnsiFore, AnsiBack, AnsiStyle, Style
from .winterm import winterm, WinColor, WinStyle

class AnsiToWin32(object):

    ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])')

    def __init__(self, wrapped, autoreset=False):
        self.wrapped = wrapped
        self.autoreset = autoreset
        self.strip = sys.platform.startswith('win')
        self.convert = (
            self.strip and
            hasattr(self.wrapped, 'isatty') and
            self.wrapped.isatty())
        self.win32_calls = self.get_win32_calls()
        self.on_stderr = self.wrapped is sys.stderr

    def get_win32_calls(self):
        if self.convert and winterm:
            return {
                AnsiStyle.RESET_ALL: (winterm.reset_all, ),
                AnsiStyle.BRIGHT: (winterm.style, WinStyle.BRIGHT),
                AnsiStyle.DIM: (winterm.style, WinStyle.DIM),
                AnsiStyle.NORMAL: (winterm.style, WinStyle.NORMAL),
                AnsiFore.BLACK: (winterm.fore, WinColor.BLACK),
                AnsiFore.RED: (winterm.fore, WinColor.RED),
                AnsiFore.GREEN: (winterm.fore, WinColor.GREEN),
                AnsiFore.YELLOW: (winterm.fore, WinColor.YELLOW),
                AnsiFore.BLUE: (winterm.fore, WinColor.BLUE),
                AnsiFore.MAGENTA: (winterm.fore, WinColor.MAGENTA),
                AnsiFore.CYAN: (winterm.fore, WinColor.CYAN),
                AnsiFore.WHITE: (winterm.fore, WinColor.GREY),
                AnsiFore.RESET: (winterm.fore, ),
                AnsiBack.BLACK: (winterm.back, WinColor.BLACK),
                AnsiBack.RED: (winterm.back, WinColor.RED),
                AnsiBack.GREEN: (winterm.back, WinColor.GREEN),
                AnsiBack.YELLOW: (winterm.back, WinColor.YELLOW),
                AnsiBack.BLUE: (winterm.back, WinColor.BLUE),
                AnsiBack.MAGENTA: (winterm.back, WinColor.MAGENTA),
                AnsiBack.CYAN: (winterm.back, WinColor.CYAN),
                AnsiBack.WHITE: (winterm.back, WinColor.GREY),
                AnsiBack.RESET: (winterm.back, ),
            }

    def __getattr__(self, name):
        return getattr(self.wrapped, name)

    def reset_all(self):
        if self.convert:
            self.call_win32('m', [0])
        else:
            self.wrapped.write(Style.RESET_ALL)

    def write(self, text):
        if self.strip:
            self.write_and_convert(text)
        else:
            self.wrapped.write(text)
        if self.autoreset:
            self.reset_all()

    def write_and_convert(self, text):
        cursor = 0
        for match in self.ANSI_RE.finditer(text):
            start, end = match.span()
            self.write_snippet(text, cursor, start)

            if self.convert:
                paramstring, command = match.groups()
                params = self.extract_params(paramstring)
                self.call_win32(command, params)

            cursor = end

        self.write_snippet(text, cursor, len(text))

    def write_snippet(self, text, start, end):
        if start < end:
            self.wrapped.write(text[start:end])

    def extract_params(self, paramstring):
        def split(paramstring):
            for p in paramstring.split(';'):
                if p != '':
                    yield int(p)
        return tuple(split(paramstring))

    def call_win32(self, command, params):
        if params == []:
            params = [0]
        if command == 'm':
            for param in params:
                if param in self.win32_calls:
                    func_args = self.win32_calls[param]
                    func = func_args[0]
                    args = func_args[1:]
                    func(*args, on_stderr=self.on_stderr)

<commit_msg>ansitowin32.py: add docstrings and extract convert_ansi method<commit_after>
import re
import sys

from .ansi import AnsiFore, AnsiBack, AnsiStyle, Style
from .winterm import winterm, WinColor, WinStyle

class AnsiToWin32(object):

    ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])')

    def __init__(self, wrapped, autoreset=False):
        # The stream we are wrapping (normally sys.stdout or sys.stderr)
        self.wrapped = wrapped

        # True if we reset colors to defaults after every .write()
        self.autoreset = autoreset

        # True if we strip ANSI sequences from our output
        self.strip = sys.platform.startswith('win')

        # True if we convert stripped ANSI into win32 calls
        self.convert = (
            self.strip and
            hasattr(self.wrapped, 'isatty') and
            self.wrapped.isatty())

        # map ansi codes to win32 functions and parameters
        self.win32_calls = self.get_win32_calls()

        # true if we are wrapping stderr
        self.on_stderr = self.wrapped is sys.stderr

    def get_win32_calls(self):
        if self.convert and winterm:
            return {
                AnsiStyle.RESET_ALL: (winterm.reset_all, ),
                AnsiStyle.BRIGHT: (winterm.style, WinStyle.BRIGHT),
                AnsiStyle.DIM: (winterm.style, WinStyle.DIM),
                AnsiStyle.NORMAL: (winterm.style, WinStyle.NORMAL),
                AnsiFore.BLACK: (winterm.fore, WinColor.BLACK),
                AnsiFore.RED: (winterm.fore, WinColor.RED),
                AnsiFore.GREEN: (winterm.fore, WinColor.GREEN),
                AnsiFore.YELLOW: (winterm.fore, WinColor.YELLOW),
                AnsiFore.BLUE: (winterm.fore, WinColor.BLUE),
                AnsiFore.MAGENTA: (winterm.fore, WinColor.MAGENTA),
                AnsiFore.CYAN: (winterm.fore, WinColor.CYAN),
                AnsiFore.WHITE: (winterm.fore, WinColor.GREY),
                AnsiFore.RESET: (winterm.fore, ),
                AnsiBack.BLACK: (winterm.back, WinColor.BLACK),
                AnsiBack.RED: (winterm.back, WinColor.RED),
                AnsiBack.GREEN: (winterm.back, WinColor.GREEN),
                AnsiBack.YELLOW: (winterm.back, WinColor.YELLOW),
                AnsiBack.BLUE: (winterm.back, WinColor.BLUE),
                AnsiBack.MAGENTA: (winterm.back, WinColor.MAGENTA),
                AnsiBack.CYAN: (winterm.back, WinColor.CYAN),
                AnsiBack.WHITE: (winterm.back, WinColor.GREY),
                AnsiBack.RESET: (winterm.back, ),
            }

    def __getattr__(self, name):
        return getattr(self.wrapped, name)

    def reset_all(self):
        if self.convert:
            self.call_win32('m', [0])
        else:
            self.wrapped.write(Style.RESET_ALL)

    def write(self, text):
        if self.strip:
            self.write_and_convert(text)
        else:
            self.wrapped.write(text)
        if self.autoreset:
            self.reset_all()

    def write_and_convert(self, text):
        '''
        Write the given text to our wrapped stream, stripping any ANSI
        sequences from the text, and optionally converting them into win32
        calls.
        '''
        cursor = 0
        for match in self.ANSI_RE.finditer(text):
            start, end = match.span()
            self.write_plain_text(text, cursor, start)
            self.convert_ansi(*match.groups())
            cursor = end
        self.write_plain_text(text, cursor, len(text))

    def write_plain_text(self, text, start, end):
        if start < end:
            self.wrapped.write(text[start:end])

    def convert_ansi(self, paramstring, command):
        if self.convert:
            params = self.extract_params(paramstring)
            self.call_win32(command, params)

    def extract_params(self, paramstring):
        def split(paramstring):
            for p in paramstring.split(';'):
                if p != '':
                    yield int(p)
        return tuple(split(paramstring))

    def call_win32(self, command, params):
        if params == []:
            params = [0]
        if command == 'm':
            for param in params:
                if param in self.win32_calls:
                    func_args = self.win32_calls[param]
                    func = func_args[0]
                    args = func_args[1:]
                    func(*args, on_stderr=self.on_stderr)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import collections
from selenium.webdriver import Ie, Opera, Chrome, Firefox
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support.ui import WebDriverWait

def create(drivername):
    if not isinstance(drivername, str):
        msg = "drivername should be an instance of string. given {0}".format(type(drivername))
        raise TypeError(msg)
    drivers = {'ie': Ie,
               'opera': Opera,
               'chrome': Chrome,
               'firefox': Firefox}
    dname = drivername.lower()
    if dname in drivers:
        try:
            return SeleniumWrapper(drivers[dname]())
        except Exception, e:
            raise e
    else:
        msg = "drivername should be one of [IE, Opera, Chrome, Firefox](case-insentive). given {0}".format(drivername)
        raise ValueError(msg)

def _is_wrappable(obj):
    if isinstance(obj, WebDriver) or isinstance(obj, WebElement):
        return True
    else:
        return False

def _chainreact(__getattr__):
    def containment(*methodname):
        self, methodobj = __getattr__(*methodname)
        def reaction(*realargs):
            result = methodobj(*realargs)
            result = result if result else self
            if _is_wrappable(result):
                return SeleniumWrapper(result)
            else:
                return result
        return reaction
    return containment

class SeleniumWrapper(object):

    def __init__(self, driver):
        if _is_wrappable(driver):
            self._driver = driver
        else:
            msg = "2nd argument should be an instance of WebDriver or WebElement. given {0}.".format(type(driver))
            raise TypeError(msg)

    @property
    def unwrap(self):
        return self._driver

    def __getattribute__(self, name):
        return object.__getattribute__(self, name)

    @_chainreact
    def __getattr__(self, name):
        return self._driver, getattr(self._driver, name)

    def waitfor(self, type, target, eager=False, timeout=10):
        if eager:
            types = {"id":lambda d: d.find_elements_by_id(target),
                     "name":lambda d: d.find_elements_by_name(target),
                     "xpath":lambda d: d.find_elements_by_xpath(target),
                     "link_text":lambda d: d.find_elements_by_link_text(target),
                     "partial_link_text":lambda d: d.find_elements_by_partial_link_text(target),
                     "tag":lambda d: d.find_elements_by_tag_name(target),
                     "class":lambda d: d.find_elements_by_class_name(target),
                     "css":lambda d: d.find_elements_by_css_selector(target), }
        else:
            types = {"id":lambda d: d.find_element_by_id(target),
                     "name":lambda d: d.find_element_by_name(target),
                     "xpath":lambda d: d.find_element_by_xpath(target),
                     "link_text":lambda d: d.find_element_by_link_text(target),
                     "partial_link_text":lambda d: d.find_element_by_partial_link_text(target),
                     "tag":lambda d: d.find_element_by_tag_name(target),
                     "class":lambda d: d.find_element_by_class_name(target),
                     "css":lambda d: d.find_element_by_css_selector(target), }
        finder = types[type]
        result = WebDriverWait(self._driver, timeout).until(finder)
        if eager and len(result):
            return SeleniumContainerWrapper(result)
        elif _is_wrappable(result):
            return SeleniumWrapper(result)
        else:
            return result

    def xpath(self, target, eager=False, timeout=10):
        return self.waitfor("xpath", target, eager, timeout)

    def css(self, target, eager=False, timeout=10):
        return self.waitfor("css", target, eager, timeout)

    def tag(self, target, eager=False, timeout=10):
        return self.waitfor("tag", target, eager, timeout)

    def by_class(self, target, eager=False, timeout=10):
        return self.waitfor("class", target, eager, timeout)

    def by_id(self, target, eager=False, timeout=10):
        return self.waitfor("id", target, eager, timeout)

    def by_name(self, target, eager=False, timeout=10):
        return self.waitfor("name", target, eager, timeout)

    def by_linktxt(self, target, eager=False, timeout=10, partial=False):
        if partial:
            return self.waitfor("partial_link_text", target, eager, timeout=10)
        else:
            return self.waitfor("link_text", target, eager, timeout)

    def href(self, partialurl=None, eager=False, timeout=10):
        if partialurl:
            return self.xpath("//a[contains(@href, '{0}')]".format(partialurl), eager, timeout)
        return self.xpath("//a", eager, timeout)

    def img(self, eager=True, ext=None, timeout=10):
        if ext:
            return self.xpath("//img[contains(@src, '{0})']".format(ext), eager, timeout)
        return self.xpath("//img", eager, timeout)

class SeleniumContainerWrapper(object):

    def __init__(self, iterable):
        if not isinstance(iterable, collections.Sequence):
            msg = "2nd argument should be an instance of collections.Sequence. given {0}".format(type(iterable))
            raise TypeError(msg)
        self._iterable = iterable

    @_chainreact
    def __getattr__(self, name):
        """Wrap return value using '_chanreact'."""
        return self._iterable, getattr(self._iterable, name)

    def __getitem__(self, key):
        obj = self._iterable[key]
        if _is_wrappable(obj):
            return SeleniumWrapper(obj)
        return obj

    def __len__(self):
        return len(self._iterable)

    def __contains__(self, key):
        return key in self._iterable
<commit_msg>Fix bug<commit_after># -*- coding: utf-8 -*-

import collections
from selenium.webdriver import Ie, Opera, Chrome, Firefox
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support.ui import WebDriverWait

def create(drivername):
    if not isinstance(drivername, str):
        msg = "drivername should be an instance of string. given {0}".format(type(drivername))
        raise TypeError(msg)
    drivers = {'ie': Ie,
               'opera': Opera,
               'chrome': Chrome,
               'firefox': Firefox}
    dname = drivername.lower()
    if dname in drivers:
        try:
            return SeleniumWrapper(drivers[dname]())
        except Exception, e:
            raise e
    else:
        msg = "drivername should be one of [IE, Opera, Chrome, Firefox](case-insentive). given {0}".format(drivername)
        raise ValueError(msg)

def _is_wrappable(obj):
    if isinstance(obj, WebDriver) or isinstance(obj, WebElement):
        return True
    else:
        return False

def _chainreact(__getattr__):
    def containment(*methodname):
        self, methodobj = __getattr__(*methodname)
        def reaction(*realargs):
            result = methodobj(*realargs)
            result = result if result else self
            if _is_wrappable(result):
                return SeleniumWrapper(result)
            else:
                return result
        return reaction
    return containment

class SeleniumWrapper(object):

    def __init__(self, driver):
        if _is_wrappable(driver):
            self._driver = driver
        else:
            msg = "2nd argument should be an instance of WebDriver or WebElement. given {0}.".format(type(driver))
            raise TypeError(msg)

    @property
    def unwrap(self):
        return self._driver

    def __getattribute__(self, name):
        return object.__getattribute__(self, name)

    @_chainreact
    def __getattr__(self, name):
        return self._driver, getattr(self._driver, name)

    def waitfor(self, type, target, eager=False, timeout=10):
        if eager:
            types = {"id":lambda d: d.find_elements_by_id(target),
                     "name":lambda d: d.find_elements_by_name(target),
                     "xpath":lambda d: d.find_elements_by_xpath(target),
                     "link_text":lambda d: d.find_elements_by_link_text(target),
                     "partial_link_text":lambda d: d.find_elements_by_partial_link_text(target),
                     "tag":lambda d: d.find_elements_by_tag_name(target),
                     "class":lambda d: d.find_elements_by_class_name(target),
                     "css":lambda d: d.find_elements_by_css_selector(target), }
        else:
            types = {"id":lambda d: d.find_element_by_id(target),
                     "name":lambda d: d.find_element_by_name(target),
                     "xpath":lambda d: d.find_element_by_xpath(target),
                     "link_text":lambda d: d.find_element_by_link_text(target),
                     "partial_link_text":lambda d: d.find_element_by_partial_link_text(target),
                     "tag":lambda d: d.find_element_by_tag_name(target),
                     "class":lambda d: d.find_element_by_class_name(target),
                     "css":lambda d: d.find_element_by_css_selector(target), }
        finder = types[type]
        result = WebDriverWait(self._driver, timeout).until(finder)
        if eager and len(result):
            return SeleniumContainerWrapper(result)
        elif _is_wrappable(result):
            return SeleniumWrapper(result)
        else:
            return result

    def xpath(self, target, eager=False, timeout=10):
        return self.waitfor("xpath", target, eager, timeout)

    def css(self, target, eager=False, timeout=10):
        return self.waitfor("css", target, eager, timeout)

    def tag(self, target, eager=False, timeout=10):
        return self.waitfor("tag", target, eager, timeout)

    def by_class(self, target, eager=False, timeout=10):
        return self.waitfor("class", target, eager, timeout)

    def by_id(self, target, eager=False, timeout=10):
        return self.waitfor("id", target, eager, timeout)

    def by_name(self, target, eager=False, timeout=10):
        return self.waitfor("name", target, eager, timeout)

    def by_linktxt(self, target, eager=False, timeout=10, partial=False):
        if partial:
            return self.waitfor("partial_link_text", target, eager, timeout=10)
        else:
            return self.waitfor("link_text", target, eager, timeout)

    def href(self, partialurl=None, eager=False, timeout=10):
        if partialurl:
            return self.xpath("//a[contains(@href, '{0}')]".format(partialurl), eager, timeout)
        return self.xpath("//a", eager, timeout)

    def img(self, eager=True, ext=None, timeout=10):
        if ext:
            return self.xpath("//img[contains(@src, '{0}')]".format(ext), eager, timeout)
        return self.xpath("//img", eager, timeout)

class SeleniumContainerWrapper(object):

    def __init__(self, iterable):
        if not isinstance(iterable, collections.Sequence):
            msg = "2nd argument should be an instance of collections.Sequence. given {0}".format(type(iterable))
            raise TypeError(msg)
        self._iterable = iterable

    @_chainreact
    def __getattr__(self, name):
        """Wrap return value using '_chanreact'."""
        return self._iterable, getattr(self._iterable, name)

    def __getitem__(self, key):
        obj = self._iterable[key]
        if _is_wrappable(obj):
            return SeleniumWrapper(obj)
        return obj

    def __len__(self):
        return len(self._iterable)

    def __contains__(self, key):
        return key in self._iterable
<|endoftext|>


<commit_before>#!python

# -*- coding: utf-8 -*- {{{
# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et:
#
# Copyright (c) 2015, Battelle Memorial Institute
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are those
# of the authors and should not be interpreted as representing official policies,
# either expressed or implied, of the FreeBSD Project.
#

# This material was prepared as an account of work sponsored by an
# agency of the United States Government.  Neither the United States
# Government nor the United States Department of Energy, nor Battelle,
# nor any of their employees, nor any jurisdiction or organization
# that has cooperated in the development of these materials, makes
# any warranty, express or implied, or assumes any legal liability
# or responsibility for the accuracy, completeness, or usefulness or
# any information, apparatus, product, software, or process disclosed,
# or represents that its use would not infringe privately owned rights.
#
# Reference herein to any specific commercial product, process, or
# service by trade name, trademark, manufacturer, or otherwise does
# not necessarily constitute or imply its endorsement, recommendation,
# r favoring by the United States Government or any agency thereof,
# or Battelle Memorial Institute. The views and opinions of authors
# expressed herein do not necessarily state or reflect those of the
# United States Government or any agency thereof.
#
# PACIFIC NORTHWEST NATIONAL LABORATORY
# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY
# under Contract DE-AC05-76RL01830

#}}}

#This is based on example code from the pymodbus source.

#---------------------------------------------------------------------------#
# import the various server implementations
#---------------------------------------------------------------------------#
from pymodbus.server.sync import StartTcpServer

from pymodbus.device import ModbusDeviceIdentification
from pymodbus.datastore import ModbusSequentialDataBlock
from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext
from pymodbus.pdu import ModbusPDU

from csv import DictReader

#---------------------------------------------------------------------------#
# configure the service logging
#---------------------------------------------------------------------------#

import argparse
import struct
import logging

from utils import createDaemon

parser = argparse.ArgumentParser(description='Run a test pymodbus driver')
parser.add_argument('config', help='device registry configuration')
parser.add_argument('interface', help='interface address')
parser.add_argument('--port', default=5020, type=int, help='port for device to listen on')
args = parser.parse_args()

logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)

MODBUS_REGISTER_SIZE = 2

class Register(object):
    def __init__(self, address, register_type, read_only, register_struct=''):
        self.read_only = read_only
        self.register_type = register_type
        self.address = address

        if register_type == "byte":
            self.parse_struct = struct.Struct(register_struct)
            self.register_count = self.parse_struct.size // MODBUS_REGISTER_SIZE
        else:
            self.register_count = 1

    def get_register_type(self):
        '''Get (type, read_only) tuple'''
        return self.register_type, self.read_only

class DeviceAbstraction(object):
    def __init__(self, config_file):
        self.build_register_map()
        self.build_ranges_map()
        self.parse_config(config_file)

    def build_register_map(self):
        self.registers = {('byte',True):[],
                          ('byte',False):[],
                          ('bit',True):[],
                          ('bit',False):[]}

    def build_ranges_map(self):
        self.register_ranges = {('byte',True):[None,None],
                                ('byte',False):[None,None],
                                ('bit',True):[None,None],
                                ('bit',False):[None,None]}

    def insert_register(self, register):
        register_type = register.get_register_type()
        self.registers[register_type].append(register)

        register_type = register.get_register_type()

        register_range = self.register_ranges[register_type]
        register_count = register.register_count

        start, end = register.address, register.address + register_count - 1

        if register_range[0] is None:
            register_range[:] = start, end
        else:
            if register_range[0] > start:
                register_range[0] = start
            if register_range[1] < end:
                register_range[1] = end

    def parse_config(self, config_file):
        with open(config_file, 'rb') as f:
            configDict = DictReader(f)

            for regDef in configDict:
                io_type = regDef['Modbus Register']
                bit_register = io_type.lower() == 'bool'
                read_only = regDef['Writable'].lower() != 'true'
                address = int(regDef['Point Address'])

                register_type = 'bit' if bit_register else 'byte'
                register = Register(address, register_type, read_only, io_type)

                self.insert_register(register)

    def get_server_context(self):
        start, end = self.register_ranges[('bit',True)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "bit", True, start, count
        di = ModbusSequentialDataBlock(start, [0]*count)

        start, end = self.register_ranges[('bit',False)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "bit", False, start, count
        co = ModbusSequentialDataBlock(start, [0]*count)

        start, end = self.register_ranges[('byte',True)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "byte", True, start, count
        ir = ModbusSequentialDataBlock(start, [0]*count)

        start, end = self.register_ranges[('byte',False)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "byte", False, start, count
        hr = ModbusSequentialDataBlock(start, [0]*count)

        store = ModbusSlaveContext(
            di = di,
            co = co,
            hr = hr,
            ir = ir)
        context = ModbusServerContext(slaves=store, single=True)

        return context

#---------------------------------------------------------------------------#
# initialize the server information
#---------------------------------------------------------------------------#
# If you don't set this or any fields, they are defaulted to empty strings.
#---------------------------------------------------------------------------#
identity = ModbusDeviceIdentification()
identity.VendorName  = 'VOLTTRON'
identity.ProductCode = 'VT'
identity.VendorUrl   = 'http://github.com/VOLTTRON/volttron'
identity.ProductName = 'VOLTTRON Modbus Test Device'
identity.ModelName   = 'VOLTTRON Modbus Test Device'
identity.MajorMinorRevision = '1.0'

abstraction = DeviceAbstraction(args.config)

#Create the deamon as soon as we've loaded the device configuration.
createDaemon()

context = abstraction.get_server_context()

#---------------------------------------------------------------------------#
# run the server you want
#---------------------------------------------------------------------------#
StartTcpServer(context, identity=identity, address=(args.interface, args.port))
<commit_msg>Specify weather or not to daemonize modbus virtual device.<commit_after>#!python

# -*- coding: utf-8 -*- {{{
# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et:
#
# Copyright (c) 2015, Battelle Memorial Institute
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation are those
# of the authors and should not be interpreted as representing official policies,
# either expressed or implied, of the FreeBSD Project.
#

# This material was prepared as an account of work sponsored by an
# agency of the United States Government.  Neither the United States
# Government nor the United States Department of Energy, nor Battelle,
# nor any of their employees, nor any jurisdiction or organization
# that has cooperated in the development of these materials, makes
# any warranty, express or implied, or assumes any legal liability
# or responsibility for the accuracy, completeness, or usefulness or
# any information, apparatus, product, software, or process disclosed,
# or represents that its use would not infringe privately owned rights.
#
# Reference herein to any specific commercial product, process, or
# service by trade name, trademark, manufacturer, or otherwise does
# not necessarily constitute or imply its endorsement, recommendation,
# r favoring by the United States Government or any agency thereof,
# or Battelle Memorial Institute. The views and opinions of authors
# expressed herein do not necessarily state or reflect those of the
# United States Government or any agency thereof.
#
# PACIFIC NORTHWEST NATIONAL LABORATORY
# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY
# under Contract DE-AC05-76RL01830

#}}}

#This is based on example code from the pymodbus source.

#---------------------------------------------------------------------------#
# import the various server implementations
#---------------------------------------------------------------------------#
from pymodbus.server.sync import StartTcpServer

from pymodbus.device import ModbusDeviceIdentification
from pymodbus.datastore import ModbusSequentialDataBlock
from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext
from pymodbus.pdu import ModbusPDU

from csv import DictReader

#---------------------------------------------------------------------------#
# configure the service logging
#---------------------------------------------------------------------------#

import argparse
import struct
import logging

from utils import createDaemon

parser = argparse.ArgumentParser(description='Run a test pymodbus driver')
parser.add_argument('config', help='device registry configuration')
parser.add_argument('interface', help='interface address')
parser.add_argument('--port', default=5020, type=int, help='port for device to listen on')
parser.add_argument('--no-daemon', help='do not create a daemon process', action='store_true')
args = parser.parse_args()

logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)

MODBUS_REGISTER_SIZE = 2

class Register(object):
    def __init__(self, address, register_type, read_only, register_struct=''):
        self.read_only = read_only
        self.register_type = register_type
        self.address = address

        if register_type == "byte":
            self.parse_struct = struct.Struct(register_struct)
            self.register_count = self.parse_struct.size // MODBUS_REGISTER_SIZE
        else:
            self.register_count = 1

    def get_register_type(self):
        '''Get (type, read_only) tuple'''
        return self.register_type, self.read_only

class DeviceAbstraction(object):
    def __init__(self, config_file):
        self.build_register_map()
        self.build_ranges_map()
        self.parse_config(config_file)

    def build_register_map(self):
        self.registers = {('byte',True):[],
                          ('byte',False):[],
                          ('bit',True):[],
                          ('bit',False):[]}

    def build_ranges_map(self):
        self.register_ranges = {('byte',True):[None,None],
                                ('byte',False):[None,None],
                                ('bit',True):[None,None],
                                ('bit',False):[None,None]}

    def insert_register(self, register):
        register_type = register.get_register_type()
        self.registers[register_type].append(register)

        register_type = register.get_register_type()

        register_range = self.register_ranges[register_type]
        register_count = register.register_count

        start, end = register.address, register.address + register_count - 1

        if register_range[0] is None:
            register_range[:] = start, end
        else:
            if register_range[0] > start:
                register_range[0] = start
            if register_range[1] < end:
                register_range[1] = end

    def parse_config(self, config_file):
        with open(config_file, 'rb') as f:
            configDict = DictReader(f)

            for regDef in configDict:
                io_type = regDef['Modbus Register']
                bit_register = io_type.lower() == 'bool'
                read_only = regDef['Writable'].lower() != 'true'
                address = int(regDef['Point Address'])

                register_type = 'bit' if bit_register else 'byte'
                register = Register(address, register_type, read_only, io_type)

                self.insert_register(register)

    def get_server_context(self):
        start, end = self.register_ranges[('bit',True)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "bit", True, start, count
        di = ModbusSequentialDataBlock(start, [0]*count)

        start, end = self.register_ranges[('bit',False)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "bit", False, start, count
        co = ModbusSequentialDataBlock(start, [0]*count)

        start, end = self.register_ranges[('byte',True)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "byte", True, start, count
        ir = ModbusSequentialDataBlock(start, [0]*count)

        start, end = self.register_ranges[('byte',False)]
        count = end - start + 1

        #See http://www.modbus.org/docs/Modbus_Application_Protocol_V1_1b3.pdf
        # section 4.4 about this nonsense.
        start += 1
        print "byte", False, start, count
        hr = ModbusSequentialDataBlock(start, [0]*count)

        store = ModbusSlaveContext(
            di = di,
            co = co,
            hr = hr,
            ir = ir)
        context = ModbusServerContext(slaves=store, single=True)

        return context

#---------------------------------------------------------------------------#
# initialize the server information
#---------------------------------------------------------------------------#
# If you don't set this or any fields, they are defaulted to empty strings.
#---------------------------------------------------------------------------#
identity = ModbusDeviceIdentification()
identity.VendorName  = 'VOLTTRON'
identity.ProductCode = 'VT'
identity.VendorUrl   = 'http://github.com/VOLTTRON/volttron'
identity.ProductName = 'VOLTTRON Modbus Test Device'
identity.ModelName   = 'VOLTTRON Modbus Test Device'
identity.MajorMinorRevision = '1.0'

abstraction = DeviceAbstraction(args.config)

#Create the deamon as soon as we've loaded the device configuration.
if not args.no_daemon:
    createDaemon()

context = abstraction.get_server_context()

#---------------------------------------------------------------------------#
# run the server you want
#---------------------------------------------------------------------------#
StartTcpServer(context, identity=identity, address=(args.interface, args.port))
<|endoftext|>


<commit_before>"""
Wifipiano 2

This file has been written taking by reference code from
osu-performance (https://github.com/ppy/osu-performance)
by Tom94, licensed under the GNU AGPL 3 License.
"""
from common.constants import mods
from common.log import logUtils as log
from constants import exceptions

class piano:
	def __init__(self, __beatmap, __score):
		self.beatmap = __beatmap
		self.score = __score
		self.pp = 0
		self.getPP()

	def getPP(self):
		try:
			stars = self.beatmap.starsMania
			if stars == 0:
				# This beatmap can't be converted to mania
				raise exceptions.invalidBeatmapException()
			od = self.beatmap.OD
			objects = self.score.c50+self.score.c100+self.score.c300+self.score.cKatu+self.score.cGeki+self.score.cMiss

			score = self.score.score
			accuracy = self.score.accuracy
			scoreMods = self.score.mods

			log.debug("[WIFIPIANO2] SCORE DATA: Stars: {stars}, OD: {od}, obj: {objects}, score: {score}, acc: {acc}, mods: {mods}".format(stars=stars, od=od, objects=objects, score=score, acc=accuracy, mods=scoreMods))

			# ---------- STRAIN PP
			# Scale score to mods multiplier
			scoreMultiplier = 1.0
			if scoreMods & mods.EASY > 0: #Doubles score if EZ
				scoreMultiplier *= 0.50
			if scoreMods & mods.HALFTIME > 0:  #Doubles score if HT
				scoreMultiplier *= 0.50
			if scoreMultiplier <= 0:
				strainPP = 0
			else:
				score *= int(1.0/scoreMultiplier)
				strainPP = pow(5.0 * max(1.0, stars / 0.0825) - 4.0, 3.0) / 110000.0
				strainPP *= 1 + 0.1 * min(1.0, float(objects) / 1500.0)
				if score <= 500000:
					strainPP *= (float(score) / 500000.0) * 0.1
				elif score <= 600000:
					strainPP *= 0.1 + float(score - 500000) / 100000.0 * 0.2
				elif score <= 700000:
					strainPP *= 0.3 + float(score - 600000) / 100000.0 * 0.35
				elif score <= 800000:
					strainPP *= 0.65 + float(score - 700000) / 100000.0 * 0.20
				elif score <= 900000:
					strainPP *= 0.85 + float(score - 800000) / 100000.0 * 0.1
				else:
					strainPP *= 0.95 + float(score - 900000) / 100000.0 * 0.05

			# ---------- ACC PP
            if scoreMods & mods.EASY > 0: #Assuming Halved OD by EZ submitted by osu. Can't test this on my end.
				od *= 2;
			scrubbedOD = min(10.0, max(0, 10.0 - od)); #makes sure OD is in range 0-10. If this is done elsewhere, remove this.
			hitWindow300 = (34 + 3 * scrubbedOD);      #old formula but done backwards.
			if (scoreMods & mods.EASY): #increases hitWindow if EZ is on
            	hitWindow300 *= 1.4;
			hitWindow300 = int(hitWindow300) + 0.5; #makes hitwindow match what it is ingame.
			accPP = pow((150.0 / hitWindow300) * pow(accuracy, 16), 1.8) * 2.5
			accPP *= min(1.15, pow(float(objects) / 1500.0, 0.3))

			# ---------- TOTAL PP
			multiplier = 1.1
			if scoreMods & mods.NOFAIL > 0:
				multiplier *= 0.90
			#if scoreMods & mods.SPUNOUT > 0: #Why was this here. Mania doesn't have spunout lol.
			#	multiplier *= 0.95
			if scoreMods & mods.EASY > 0:
				multiplier *= 0.50
			pp = pow(pow(strainPP, 1.1) + pow(accPP, 1.1), 1.0 / 1.1) * multiplier
			log.debug("[WIFIPIANO2] Calculated PP: {}".format(pp))
			self.pp = pp
		except exceptions.invalidBeatmapException:
			log.warning("Invalid beatmap {}".format(self.beatmap.beatmapID))
			self.pp = 0
		finally:
			return self.pp
<commit_msg>Update 'pp/wifipiano2.py'<commit_after>"""
Wifipiano 2

This file has been written taking by reference code from
osu-performance (https://github.com/ppy/osu-performance)
by Tom94, licensed under the GNU AGPL 3 License.
"""
from common.constants import mods
from common.log import logUtils as log
from constants import exceptions

class piano:
	def __init__(self, __beatmap, __score):
		self.beatmap = __beatmap
		self.score = __score
		self.pp = 0
		self.getPP()

	def getPP(self):
		try:
			stars = self.beatmap.starsMania
			if stars == 0:
				# This beatmap can't be converted to mania
				raise exceptions.invalidBeatmapException()
			od = self.beatmap.OD
			objects = self.score.c50+self.score.c100+self.score.c300+self.score.cKatu+self.score.cGeki+self.score.cMiss

			score = self.score.score
			accuracy = self.score.accuracy
			scoreMods = self.score.mods

			log.debug("[WIFIPIANO2] SCORE DATA: Stars: {stars}, OD: {od}, obj: {objects}, score: {score}, acc: {acc}, mods: {mods}".format(stars=stars, od=od, objects=objects, score=score, acc=accuracy, mods=scoreMods))

			# ---------- STRAIN PP
			# Scale score to mods multiplier
			scoreMultiplier = 1.0
			if scoreMods & mods.EASY > 0: #Doubles score if EZ
				scoreMultiplier *= 0.50
			if scoreMods & mods.HALFTIME > 0:  #Doubles score if HT
				scoreMultiplier *= 0.50
			if scoreMultiplier <= 0:
				strainPP = 0
			else:
				score *= int(1.0/scoreMultiplier)
				strainPP = pow(5.0 * max(1.0, stars / 0.0825) - 4.0, 3.0) / 110000.0
				strainPP *= 1 + 0.1 * min(1.0, float(objects) / 1500.0)
				if score <= 500000:
					strainPP *= (float(score) / 500000.0) * 0.1
				elif score <= 600000:
					strainPP *= 0.1 + float(score - 500000) / 100000.0 * 0.2
				elif score <= 700000:
					strainPP *= 0.3 + float(score - 600000) / 100000.0 * 0.35
				elif score <= 800000:
					strainPP *= 0.65 + float(score - 700000) / 100000.0 * 0.20
				elif score <= 900000:
					strainPP *= 0.85 + float(score - 800000) / 100000.0 * 0.1
				else:
					strainPP *= 0.95 + float(score - 900000) / 100000.0 * 0.05

			# ---------- ACC PP
			if scoreMods & mods.EASY > 0: #Assuming Halved OD by EZ submitted by osu. Can't test this on my end.
				od *= 2;
			scrubbedOD = min(10.0, max(0, 10.0 - od)); #makes sure OD is in range 0-10. If this is done elsewhere, remove this.
			hitWindow300 = (34 + 3 * scrubbedOD);      #old formula but done backwards.
			if (scoreMods & mods.EASY): #increases hitWindow if EZ is on
				hitWindow300 *= 1.4;
			hitWindow300 = int(hitWindow300) + 0.5; #makes hitwindow match what it is ingame.
			accPP = pow((150.0 / hitWindow300) * pow(accuracy, 16), 1.8) * 2.5
			accPP *= min(1.15, pow(float(objects) / 1500.0, 0.3))

			# ---------- TOTAL PP
			multiplier = 1.1
			if scoreMods & mods.NOFAIL > 0:
				multiplier *= 0.90
			#if scoreMods & mods.SPUNOUT > 0: #Why was this here. Mania doesn't have spunout lol.
			#	multiplier *= 0.95
			if scoreMods & mods.EASY > 0:
				multiplier *= 0.50
			pp = pow(pow(strainPP, 1.1) + pow(accPP, 1.1), 1.0 / 1.1) * multiplier
			log.debug("[WIFIPIANO2] Calculated PP: {}".format(pp))
			self.pp = pp
		except exceptions.invalidBeatmapException:
			log.warning("Invalid beatmap {}".format(self.beatmap.beatmapID))
			self.pp = 0
		finally:
			return self.pp
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

from plumbum import FG, local
from plumbum.commands import ProcessExecutionError
from plumbum.cmd import find, echo, rm, mkdir, rmdir, cp, ln, cat, make, chmod

from os import path, listdir
from glob import glob
from functools import wraps
from settings import config

import sys
import logging

# Configure the log
formatter = logging.Formatter('%(asctime)s - %(levelname)s :: %(message)s')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)

log = logging.getLogger(__name__)
log.addHandler(handler)

PROJECT_LIKWID_F_EXT = ".txt"
PROJECT_BIN_F_EXT = ".bin"
PROJECT_BLOB_F_EXT = ".postproc"
PROJECT_TIME_F_EXT = ".time"
PROJECT_CALLS_F_EXT = ".calls"
PROJECT_RESULT_F_EXT = ".result"
PROJECT_CALIB_CALLS_F_EXT = ".calibrate.calls"
PROJECT_CALIB_PROFILE_F_EXT = ".calibrate.profile.out"

class ProjectFactory:
    factories = {}

    def addFactory(id, projectFactory):
        ProjectFactory.factories[id] = projectFactory
    addFactory = staticmethod(addFactory)

    def createProject(id, exp):
        if not ProjectFactory.factories.has_key(id):
            ProjectFactory.factories[id] = \
                eval(id + '.Factory()')
            return ProjectFactory.factories[id].create(self, exp)
    createProject = staticmethod(createProject)

class Project(object):

    """ A project defines how run-time testing and cleaning is done for this
        IR project
    """

    def __init__(self, exp, name, domain, group=None):
        self.experiment = exp
        self.name = name
        self.domain = domain
        self.group_name = group

        self.sourcedir = path.join(config["sourcedir"], self.name)
        self.builddir = path.join(config["builddir"], exp.name, self.name)
        if group:
            self.testdir = path.join(config["testdir"], domain, group, name)
        else:
            self.testdir = path.join(config["testdir"], domain, name)

        self.inputs = set()
        self.outputs = set()

        self.products = set()
        self.cflags = []
        self.ldflags = []

        self.setup_derived_filenames()

    def setup_derived_filenames(self):
        self.run_f = path.join(self.builddir, self.name)
        self.result_f = self.run_f + PROJECT_RESULT_F_EXT
        self.bin_f = self.run_f + PROJECT_BIN_F_EXT
        self.time_f = self.run_f + PROJECT_TIME_F_EXT
        self.calibrate_calls_f = self.run_f + PROJECT_CALIB_CALLS_F_EXT
        self.calls_f = path.join(self.builddir, "papi.calls.out")
        self.likwid_f = self.run_f + PROJECT_LIKWID_F_EXT

        self.products.clear()
        self.products.add(self.run_f)
        self.products.add(self.bin_f)
        self.products.add(self.time_f)
        self.products.add(self.likwid_f)
        self.products.add(self.calls_f)
        self.products.add(self.result_f)

    def run_tests(self, experiment):
        exp = wrap(self.run_f, experiment)
        exp()

    run_uuid = None

    def run(self, experiment):
        from uuid import uuid4
        with local.cwd(self.builddir):
            if self.run_uuid is None:
                self.run_uuid = uuid4()
            with local.env( PPROF_USE_DATABASE=1,
                           PPROF_DB_RUN_GROUP=self.run_uuid,
                           PPROF_DOMAIN=self.domain,
                           PPROF_GROUP=self.group_name,
                           PPROF_SRC_URI=self.src_uri):
                self.run_tests(experiment)

    def clean(self):
        dirs_to_remove = set([])

        for product in self.products:
            if path.exists(product) and not path.isdir(product):
                rm[product] & FG
            elif path.isdir(product):
                dirs_to_remove.add(product)

        for d in dirs_to_remove:
            if listdir(d) == []:
                rmdir[d] & FG

        if path.exists(self.builddir) and listdir(self.builddir) == []:
            rmdir[self.builddir] & FG
        elif path.exists(self.builddir) and listdir(self.builddir) != []:
            log.warn(self.name + " project unclean, force removing " +
                     self.builddir)
            rm["-rf", self.builddir] & FG

    def prepare(self):
        if not path.exists(self.builddir):
            mkdir[self.builddir] & FG

    def print_result_header(self):
        (echo["---------------------------------------------------------------"]
            >> self.result_f) & FG
        (echo[">>> ========= " + self.name + " Program"]
            >> self.result_f) & FG
        (echo["---------------------------------------------------------------"]
            >> self.result_f) & FG

        def download(self):
            pass

        def configure(self):
            pass

        def build(self):
            pass

def wrap(name, runner):
    """Wrap the binary :name: with the function :runner:.

    This module generates a python tool that replaces :name:
    The function in runner only accepts the replaced binaries
    name as argument. We use PiCloud's cloudpickle library to
    perform the serialization, make sure :runner: can be serialized
    with it and you're fine.

    :name: Binary we want to wrap
    :runner: Function that should run instead of :name:
    :returns: plumbum command, readty to launch.

    """
    from plumbum import local
    from plumbum.cmd import mv, chmod
    from cloud.serialization import cloudpickle as cp
    from os import path

    name_absolute = path.abspath(name)
    real_f = name_absolute + PROJECT_BIN_F_EXT
    mv(name_absolute, real_f)

    blob_f = name_absolute + PROJECT_BLOB_F_EXT
    with open(blob_f, 'wb') as b:
        b.write(cp.dumps(runner))

    with open(name_absolute, 'w') as w:
        lines = '''#!/usr/bin/env python
# encoding: utf-8

from plumbum import cli, local
from os import path
import sys
import pickle

run_f = "{runf}"
args = sys.argv[1:]
f = None
if path.exists("{blobf}"):
    with open("{blobf}", "rb") as p:
        f = pickle.load(p)
    with local.env(PPROF_DB_HOST="{db_host}",
               PPROF_DB_PORT="{db_port}",
               PPROF_DB_NAME="{db_name}",
               PPROF_DB_USER="{db_user}",
               PPROF_DB_PASS="{db_pass}",
               PPROF_CMD=run_f + " ".join(args)):
        if f is not None:
            f(run_f, *args)
        else:
            raise'''.format(db_host=config["db_host"],
                            db_port=config["db_port"],
                            db_name=config["db_name"],
                            db_user=config["db_user"],
                            db_pass=config["db_pass"],
                            blobf=blob_f,
                            runf=real_f)
        w.write(lines)
    chmod("+x", name_absolute)
    return local[name_absolute]

def wrap_dynamic(name, runner):
    """Wrap the binary :name: with the function :runner:.

    This module generates a python tool :name: that can replace
    a yet unspecified binary.
    It behaves similar to the :wrap: function. However, the first
    argument is the actual binary name.

    :name: name of the python module
    :runner: Function that should run the real binary
    :returns: plumbum command, readty to launch.

    """
    from plumbum import local
    from plumbum.cmd import mv, chmod
    from cloud.serialization import cloudpickle as cp
    from os import path

    name_absolute = path.abspath(name)
    blob_f = name_absolute + PROJECT_BLOB_F_EXT
    with open(blob_f, 'wb') as b:
        b.write(cp.dumps(runner))

    with open(name_absolute, 'w') as w:
        lines = '''#!/usr/bin/env python
# encoding: utf-8

from plumbum import cli, local
from os import path
import sys
import pickle

f = None
args = sys.argv[1:]
run_f = sys.argv[0]
if path.exists("{blobf}"):
    with open("{blobf}", "rb") as p:
        f = pickle.load(p)
    with local.env(PPROF_DB_HOST="{db_host}",
               PPROF_DB_PORT="{db_port}",
               PPROF_DB_NAME="{db_name}",
               PPROF_DB_USER="{db_user}",
               PPROF_DB_PASS="{db_pass}",
               PPROF_CMD=run_f):
        if f is not None:
            f(run_f, *args)'''.format(db_host=config["db_host"],
                                              db_port=config["db_port"],
                                              db_name=config["db_name"],
                                              db_user=config["db_user"],
                                              db_pass=config["db_pass"],
                                              blobf=blob_f)
        w.write(lines)
    chmod("+x", name_absolute)
    return local[name_absolute]
<commit_msg>project: wrap/wrap_dynamic can handle stdin arguments too now<commit_after>#!/usr/bin/env python
# encoding: utf-8

from plumbum import FG, local
from plumbum.commands import ProcessExecutionError
from plumbum.cmd import find, echo, rm, mkdir, rmdir, cp, ln, cat, make, chmod

from os import path, listdir
from glob import glob
from functools import wraps
from settings import config

import sys
import logging

# Configure the log
formatter = logging.Formatter('%(asctime)s - %(levelname)s :: %(message)s')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)

log = logging.getLogger(__name__)
log.addHandler(handler)

PROJECT_LIKWID_F_EXT = ".txt"
PROJECT_BIN_F_EXT = ".bin"
PROJECT_BLOB_F_EXT = ".postproc"
PROJECT_TIME_F_EXT = ".time"
PROJECT_CALLS_F_EXT = ".calls"
PROJECT_RESULT_F_EXT = ".result"
PROJECT_CALIB_CALLS_F_EXT = ".calibrate.calls"
PROJECT_CALIB_PROFILE_F_EXT = ".calibrate.profile.out"

class ProjectFactory:
    factories = {}

    def addFactory(id, projectFactory):
        ProjectFactory.factories[id] = projectFactory
    addFactory = staticmethod(addFactory)

    def createProject(id, exp):
        if not ProjectFactory.factories.has_key(id):
            ProjectFactory.factories[id] = \
                eval(id + '.Factory()')
            return ProjectFactory.factories[id].create(self, exp)
    createProject = staticmethod(createProject)

class Project(object):

    """ A project defines how run-time testing and cleaning is done for this
        IR project
    """

    def __init__(self, exp, name, domain, group=None):
        self.experiment = exp
        self.name = name
        self.domain = domain
        self.group_name = group

        self.sourcedir = path.join(config["sourcedir"], self.name)
        self.builddir = path.join(config["builddir"], exp.name, self.name)
        if group:
            self.testdir = path.join(config["testdir"], domain, group, name)
        else:
            self.testdir = path.join(config["testdir"], domain, name)

        self.inputs = set()
        self.outputs = set()

        self.products = set()
        self.cflags = []
        self.ldflags = []

        self.setup_derived_filenames()

    def setup_derived_filenames(self):
        self.run_f = path.join(self.builddir, self.name)
        self.result_f = self.run_f + PROJECT_RESULT_F_EXT
        self.bin_f = self.run_f + PROJECT_BIN_F_EXT
        self.time_f = self.run_f + PROJECT_TIME_F_EXT
        self.calibrate_calls_f = self.run_f + PROJECT_CALIB_CALLS_F_EXT
        self.calls_f = path.join(self.builddir, "papi.calls.out")
        self.likwid_f = self.run_f + PROJECT_LIKWID_F_EXT

        self.products.clear()
        self.products.add(self.run_f)
        self.products.add(self.bin_f)
        self.products.add(self.time_f)
        self.products.add(self.likwid_f)
        self.products.add(self.calls_f)
        self.products.add(self.result_f)

    def run_tests(self, experiment):
        exp = wrap(self.run_f, experiment)
        exp()

    run_uuid = None

    def run(self, experiment):
        from uuid import uuid4
        with local.cwd(self.builddir):
            if self.run_uuid is None:
                self.run_uuid = uuid4()
            with local.env( PPROF_USE_DATABASE=1,
                           PPROF_DB_RUN_GROUP=self.run_uuid,
                           PPROF_DOMAIN=self.domain,
                           PPROF_GROUP=self.group_name,
                           PPROF_SRC_URI=self.src_uri):
                self.run_tests(experiment)

    def clean(self):
        dirs_to_remove = set([])

        for product in self.products:
            if path.exists(product) and not path.isdir(product):
                rm[product] & FG
            elif path.isdir(product):
                dirs_to_remove.add(product)

        for d in dirs_to_remove:
            if listdir(d) == []:
                rmdir[d] & FG

        if path.exists(self.builddir) and listdir(self.builddir) == []:
            rmdir[self.builddir] & FG
        elif path.exists(self.builddir) and listdir(self.builddir) != []:
            log.warn(self.name + " project unclean, force removing " +
                     self.builddir)
            rm["-rf", self.builddir] & FG

    def prepare(self):
        if not path.exists(self.builddir):
            mkdir[self.builddir] & FG

    def print_result_header(self):
        (echo["---------------------------------------------------------------"]
            >> self.result_f) & FG
        (echo[">>> ========= " + self.name + " Program"]
            >> self.result_f) & FG
        (echo["---------------------------------------------------------------"]
            >> self.result_f) & FG

        def download(self):
            pass

        def configure(self):
            pass

        def build(self):
            pass

def wrap(name, runner):
    """Wrap the binary :name: with the function :runner:.

    This module generates a python tool that replaces :name:
    The function in runner only accepts the replaced binaries
    name as argument. We use PiCloud's cloudpickle library to
    perform the serialization, make sure :runner: can be serialized
    with it and you're fine.

    :name: Binary we want to wrap
    :runner: Function that should run instead of :name:
    :returns: plumbum command, readty to launch.

    """
    from plumbum import local
    from plumbum.cmd import mv, chmod
    from cloud.serialization import cloudpickle as cp
    from os import path

    name_absolute = path.abspath(name)
    real_f = name_absolute + PROJECT_BIN_F_EXT
    mv(name_absolute, real_f)

    blob_f = name_absolute + PROJECT_BLOB_F_EXT
    with open(blob_f, 'wb') as b:
        b.write(cp.dumps(runner))

    with open(name_absolute, 'w') as w:
        lines = '''#!/usr/bin/env python
# encoding: utf-8

from plumbum import cli, local
from os import path
import sys
import pickle

run_f = "{runf}"
args = sys.argv[1:]
f = None
if path.exists("{blobf}"):
    with open("{blobf}", "rb") as p:
        f = pickle.load(p)
    with local.env(PPROF_DB_HOST="{db_host}",
               PPROF_DB_PORT="{db_port}",
               PPROF_DB_NAME="{db_name}",
               PPROF_DB_USER="{db_user}",
               PPROF_DB_PASS="{db_pass}",
               PPROF_CMD=run_f + " ".join(args)):
        if not sys.stdin.isatty():
            f(run_f, args, has_stdin = True)
        if f is not None:
            f(run_f, args)
        else:
            raise'''.format(db_host=config["db_host"],
                            db_port=config["db_port"],
                            db_name=config["db_name"],
                            db_user=config["db_user"],
                            db_pass=config["db_pass"],
                            blobf=blob_f,
                            runf=real_f)
        w.write(lines)
    chmod("+x", name_absolute)
    return local[name_absolute]

def wrap_dynamic(name, runner):
    """Wrap the binary :name: with the function :runner:.

    This module generates a python tool :name: that can replace
    a yet unspecified binary.
    It behaves similar to the :wrap: function. However, the first
    argument is the actual binary name.

    :name: name of the python module
    :runner: Function that should run the real binary
    :returns: plumbum command, readty to launch.

    """
    from plumbum import local
    from plumbum.cmd import mv, chmod
    from cloud.serialization import cloudpickle as cp
    from os import path

    name_absolute = path.abspath(name)
    blob_f = name_absolute + PROJECT_BLOB_F_EXT
    with open(blob_f, 'wb') as b:
        b.write(cp.dumps(runner))

    with open(name_absolute, 'w') as w:
        lines = '''#!/usr/bin/env python
# encoding: utf-8

from plumbum import cli, local
from os import path
import sys
import pickle

f = None
args = sys.argv[1:]
run_f = sys.argv[0]
if path.exists("{blobf}"):
    with open("{blobf}", "rb") as p:
        f = pickle.load(p)
    with local.env(PPROF_DB_HOST="{db_host}",
               PPROF_DB_PORT="{db_port}",
               PPROF_DB_NAME="{db_name}",
               PPROF_DB_USER="{db_user}",
               PPROF_DB_PASS="{db_pass}",
               PPROF_CMD=run_f):
        if not sys.stdin.isatty():
            f(run_f, args, has_stdin = True)
        if f is not None:
            f(run_f, args)'''.format(db_host=config["db_host"],
                                              db_port=config["db_port"],
                                              db_name=config["db_name"],
                                              db_user=config["db_user"],
                                              db_pass=config["db_pass"],
                                              blobf=blob_f)
        w.write(lines)
    chmod("+x", name_absolute)
    return local[name_absolute]
<|endoftext|>


<commit_before>import json
import optparse
import subprocess

class Job:

    def __init__(self,jtype):
        self.jtype = jtype
        self.totals = []

    def add_job_count(self, njobs):
        if njobs:
            self.totals.append(int(njobs))

    def get_average_jobs(self):
        sum = 0.0
        for eff in self.totals:
            sum = sum+eff
        return int(sum/len(self.totals))

class CurrentJobs:
    def __init__(self,config,template):
        self.access_fifemon = True
        self.jobs = {}

        for state in ('current', 'idle', 'held'):
            self.add_job(state, json.loads(self.query_graphite(config['jobs']['{0:s}_jobs_curl'.format(state)]))[0])

        self.limit = int(config['jobs']['slot_quota'])
        self.held_limit = int(config['jobs']['held_limit'])

        self.template = template
        self.should_check_held = False
        self.should_check_running = False

    def add_job(self,jtype,item):
        job = Job(jtype)
        for count in item["datapoints"]:
            job.add_job_count(count[0])
        if len(job.totals):
            self.jobs[jtype] = job

    def query_graphite(self,url):
        cmd = "curl -k \'%s\'" % (url)
        proc = subprocess.Popen(cmd,shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        # Reads from pipes, avoides blocking
        result, error = proc.communicate()
        return_code = proc.wait()
        if return_code != 0:
            self.access_fifemon = False
            return None
        return result

    def update_template(self):
        color_red = ""
        color_black = ""
        end_color = ""
        table_content = ""
        self.template = self.template.replace("$CURRENT_JOBS_FIFEMON_ACCESS", "" if self.access_fifemon else
                                              "FIFEMON_ACCESS: %sDOWN%s" % (color_red, end_color) )
        self.template = self.template.replace("$CURRENT_JOBS_LEVEL_1_STARTS", "" if self.access_fifemon else "")
        return self.template

def parse_opts():
        """Parses command line options"""

        usage = "Usage: %prog [options]"
        parser = optparse.OptionParser(usage)
        parser.add_option("-c", "--config", dest="config", type="string",
                  help="report configuration file (required)")
        opts, args = parser.parse_args()
        return opts, args

if __name__ == '__main__':
    opts, args = parse_opts()
    config = opts.confi
    template = "".join(open(config['common']['template']).readlines())
    cjobs = CurrentJobs(config, template)
    report = open("minerva_report.html", 'w')
    report.write(cjobs.update_template())
    report.close()
<commit_msg>Fixed bug where if there were no {current|idle|held} jobs, report would fail with KeyError<commit_after>import json
import optparse
import subprocess

class Job:

    def __init__(self,jtype):
        self.jtype = jtype
        self.totals = []

    def add_job_count(self, njobs):
        if njobs:
            self.totals.append(int(njobs))

    def get_average_jobs(self):
        sum = 0.0
        for eff in self.totals:
            sum = sum+eff
        return int(sum/len(self.totals))

class CurrentJobs:
    def __init__(self,config,template):
        self.access_fifemon = True
        self.jobs = {}

        for state in ('current', 'idle', 'held'):
            self.add_job(state, json.loads(self.query_graphite(config['jobs']['{0:s}_jobs_curl'.format(state)]))[0])

        self.limit = int(config['jobs']['slot_quota'])
        self.held_limit = int(config['jobs']['held_limit'])

        self.template = template
        self.should_check_held = False
        self.should_check_running = False

    def add_job(self,jtype,item):
        job = Job(jtype)
        for count in item["datapoints"]:
            job.add_job_count(count[0])
        if len(job.totals):
            self.jobs[jtype] = job

    def query_graphite(self,url):
        cmd = "curl -k \'%s\'" % (url)
        proc = subprocess.Popen(cmd,shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        # Reads from pipes, avoides blocking
        result, error = proc.communicate()
        return_code = proc.wait()
        if return_code != 0:
            self.access_fifemon = False
            return None
        return result

    def update_template(self):
        color_red = ""
        color_black = ""
        end_color = ""
        table_content = ""
        self.template = self.template.replace("$CURRENT_JOBS_FIFEMON_ACCESS", "" if self.access_fifemon else
                                              "FIFEMON_ACCESS: %sDOWN%s" % (color_red, end_color) )
        self.template = self.template.replace("$CURRENT_JOBS_LEVEL_1_STARTS", "" if self.access_fifemon else "")
        return self.template

def parse_opts():
        """Parses command line options"""

        usage = "Usage: %prog [options]"
        parser = optparse.OptionParser(usage)
        parser.add_option("-c", "--config", dest="config", type="string",
                  help="report configuration file (required)")
        opts, args = parser.parse_args()
        return opts, args

if __name__ == '__main__':
    opts, args = parse_opts()
    config = opts.confi
    template = "".join(open(config['common']['template']).readlines())
    cjobs = CurrentJobs(config, template)
    report = open("minerva_report.html", 'w')
    report.write(cjobs.update_template())
    report.close()
<|endoftext|>


<commit_before>"""
Compile our environment from directories and files.
"""
import os

from heapq import heappush

import yaml

from withenv.flatten import flatten

def find_yml_in_dir(dirname):
    def is_yaml(fn):
        return fn.endswith(('yml', 'yaml'))

    fnames = []  # a heap

    for dirpath, dirnames, filenames in os.walk(dirname):
        for fn in filter(is_yaml, filenames):
            heappush(fnames, os.path.join(dirpath, fn))

    return fnames

def update_env_from_dir(dirname, env):
    for fname in find_yml_in_dir(dirname):
        update_env_from_file(fname, env)

def update_env_from_file(fname, env):
    new_env = yaml.safe_load(open(fname))
    flat_env = {}

    # Order isn't important
    if isinstance(new_env, dict):
        new_env = [new_env]

    for item in new_env:
        for k, v in flatten(item):
            env[k] = v

def update_env_from_alias(fname, env):
    action_list = yaml.safe_load(open(fname))
    actions = [
        (k, v) for action in action_list
        for k, v in action.items()
    ]
    print(actions)
    return compile(actions, env)

def find_action(name):
    actions = {
        'file': update_env_from_file,
        'directory': update_env_from_dir,
        'alias': update_env_from_alias,
    }
    return actions[name]

def compile(actions=None, env=None):
    actions = actions or []
    env = env if env is not None else os.environ

    for action, arg in actions:
        find_action(action)(arg, env)
    return env
<commit_msg>Remove a print<commit_after>"""
Compile our environment from directories and files.
"""
import os

from heapq import heappush

import yaml

from withenv.flatten import flatten

def find_yml_in_dir(dirname):
    def is_yaml(fn):
        return fn.endswith(('yml', 'yaml'))

    fnames = []  # a heap

    for dirpath, dirnames, filenames in os.walk(dirname):
        for fn in filter(is_yaml, filenames):
            heappush(fnames, os.path.join(dirpath, fn))

    return fnames

def update_env_from_dir(dirname, env):
    for fname in find_yml_in_dir(dirname):
        update_env_from_file(fname, env)

def update_env_from_file(fname, env):
    new_env = yaml.safe_load(open(fname))
    flat_env = {}

    # Order isn't important
    if isinstance(new_env, dict):
        new_env = [new_env]

    for item in new_env:
        for k, v in flatten(item):
            env[k] = v

def update_env_from_alias(fname, env):
    action_list = yaml.safe_load(open(fname))
    actions = [
        (k, v) for action in action_list
        for k, v in action.items()
    ]
    return compile(actions, env)

def find_action(name):
    actions = {
        'file': update_env_from_file,
        'directory': update_env_from_dir,
        'alias': update_env_from_alias,
    }
    return actions[name]

def compile(actions=None, env=None):
    actions = actions or []
    env = env if env is not None else os.environ

    for action, arg in actions:
        find_action(action)(arg, env)
    return env
<|endoftext|>


<commit_before>import re

import arrow
import requests
from dateutil import tz

from provider import get_logger, Provider, Status, ProviderException

logger = get_logger('yvbeach')

class YVBeach(Provider):
    provider_code = 'yvbeach'
    provider_name = 'yvbeach.com'
    provider_url = 'http://www.yvbeach.com/yvmeteo.htm'

    def process_data(self):
        station_id = 'yvbeach'
        try:
            logger.info("Processing yvbeach data...")

            date_pattern = re.compile(r'Relevés du
(?P<date>.*?) à (?P<time>.*?)
')
            wind_pattern = re.compile(r'VENT
'
                                      r'Moy10min (?P<wind_avg>[0-9]{1,3}\.[0-9]) km/h
'
                                      r'Max/1h (?P<wind_max>[0-9]{1,3}\.[0-9]) km/h
'
                                      r'[A-Z]{1,3} - (?P<wind_dir>[0-9]{2})°')

            yvbeach_tz = tz.gettz('Europe/Zurich')

            session = requests.Session()
            session.headers.update({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 '
                                                  '(KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'})
            content = session.get('http://www.yvbeach.com/yvmeteo.wml',
                                  timeout=(self.connect_timeout, self.read_timeout)).text.replace('\r\n', '')

            station = self.save_station(
                'yvbeach',
                'yvbeach',
                'Yvonand plage',
                float(46.806048),
                float(6.738763),
                Status.GREEN,
                url=self.provider_url
            )
            station_id = station['_id']

            date = date_pattern.search(content).groupdict()
            key = arrow.get('{0} {1}'.format(date['date'], date['time']), 'DD.MM.YYYY HH[h]mm').replace(
                tzinfo=yvbeach_tz).timestamp

            measures_collection = self.measures_collection(station_id)
            new_measures = []

            if not self.has_measure(measures_collection, key):
                wind = wind_pattern.search(content).groupdict()

                temp_pattern = re.compile(r'TEMPERATURES
Air (?P<temp>[-+]?[0-9]*\.?[0-9]+)°C')
                temp = temp_pattern.search(content).groupdict()

                measure = self.create_measure(
                    key,
                    wind['wind_dir'],
                    wind['wind_avg'],
                    wind['wind_max'],
                    temperature=temp['temp'],
                )
                new_measures.append(measure)

            self.insert_new_measures(measures_collection, station, new_measures, logger)

        except ProviderException as e:
            logger.warn("Error while processing station '{0}': {1}".format(station_id, e))
        except Exception as e:
            logger.exception("Error while processing station '{0}': {1}".format(station_id, e))
            self.raven_client.captureException()

        logger.info("...Done!")

YVBeach().process_data()
<commit_msg>Fixed pattern<commit_after>import re

import arrow
import requests
from dateutil import tz

from provider import get_logger, Provider, Status, ProviderException

logger = get_logger('yvbeach')

class YVBeach(Provider):
    provider_code = 'yvbeach'
    provider_name = 'yvbeach.com'
    provider_url = 'http://www.yvbeach.com/yvmeteo.htm'

    def process_data(self):
        station_id = 'yvbeach'
        try:
            logger.info("Processing yvbeach data...")

            date_pattern = re.compile(r'Relevés du
(?P<date>.*?) à (?P<time>.*?)
')
            wind_pattern = re.compile(r'VENT
'
                                      r'Moy10min (?P<wind_avg>[0-9]{1,3}\.[0-9]) km/h
'
                                      r'Max/1h (?P<wind_max>[0-9]{1,3}\.[0-9]) km/h
'
                                      r'[A-Z]{1,3} - (?P<wind_dir>[0-9]{1,3})°')
            temp_pattern = re.compile(r'TEMPERATURES
Air (?P<temp>[-+]?[0-9]*\.?[0-9]+)°C')

            yvbeach_tz = tz.gettz('Europe/Zurich')

            session = requests.Session()
            session.headers.update({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 '
                                                  '(KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'})
            content = session.get('http://www.yvbeach.com/yvmeteo.wml',
                                  timeout=(self.connect_timeout, self.read_timeout)).text.replace('\r\n', '')

            station = self.save_station(
                'yvbeach',
                'yvbeach',
                'Yvonand plage',
                float(46.806048),
                float(6.738763),
                Status.GREEN,
                url=self.provider_url
            )
            station_id = station['_id']

            date = date_pattern.search(content).groupdict()
            key = arrow.get('{0} {1}'.format(date['date'], date['time']), 'DD.MM.YYYY HH[h]mm').replace(
                tzinfo=yvbeach_tz).timestamp

            measures_collection = self.measures_collection(station_id)
            new_measures = []

            if not self.has_measure(measures_collection, key):
                wind = wind_pattern.search(content).groupdict()
                temp = temp_pattern.search(content).groupdict()

                measure = self.create_measure(
                    key,
                    wind['wind_dir'],
                    wind['wind_avg'],
                    wind['wind_max'],
                    temperature=temp['temp'],
                )
                new_measures.append(measure)

            self.insert_new_measures(measures_collection, station, new_measures, logger)

        except ProviderException as e:
            logger.warn("Error while processing station '{0}': {1}".format(station_id, e))
        except Exception as e:
            logger.exception("Error while processing station '{0}': {1}".format(station_id, e))
            self.raven_client.captureException()

        logger.info("...Done!")

YVBeach().process_data()
<|endoftext|>