<commit_before>from os import listdir, path
import pandas as pd
from matplotlib import pyplot as plt

def get_files(directory, file_extension):
    """Get a list of filenames with a specific extension."""
    return filter(lambda f: f.endswith(file_extension), listdir(directory))

def concat_data(file_list):
    """Take multiple csv files and merge the tables in one big table."""
    df_list = []
    for file in file_list:
        with open(file, 'r') as f:
            df_list.append(pd.read_csv(f, index_col='func_name'))
    return pd.concat(df_list)

if __name__ == '__main__':
    dirs = {1: '../bench_results/raw/selected_1graph/',
            5: '../bench_results/raw/selected_5graph/'}

    means = {k: None for k in dirs.keys()}

    # Get the means for all runs
    for n_graph, checkup_dir in dirs.items():
        files = get_files(checkup_dir, '.csv')
        csv_dir = path.dirname(checkup_dir)
        abs_files = map(lambda f: path.join(csv_dir, f), files)  # list of csv files with absolute paths

        big_df = concat_data(abs_files)
        mean = big_df.groupby(big_df.index).mean()
        means[n_graph] = mean
        mean.to_csv(path.join(csv_dir, 'mean.csv'))  # outputs mean

        big_df.groupby(big_df.index).std().to_csv(path.join(csv_dir, 'std.csv'))  # outputs standard deviation

    # Build a dataframe per store
    first_mean_df = means.values()[0]
    store_names = set(first_mean_df.columns)  # populate before we do any intersection, otherwise it's always empty
    for dataframe in means.values():
        store_names.intersection_update(dataframe.columns)

    all_store_query = {}
    # Put empty dataframes in dict with store names as keys
    for store_name in store_names:
        all_store_query[store_name] = pd.DataFrame(index=means.keys(),
                                                   columns=first_mean_df.index.values.tolist())
    # Build each dataframe with the real values
    for n_graph, mean_df in means.items():
        for store_name in store_names:
            for func_name in mean_df.index:
                all_store_query[store_name][func_name][n_graph] = means[n_graph][store_name][func_name]

    # Display all the figures
    plt.figure()
    n_subplots = len(all_store_query)
    for ind_subplot, store_name in enumerate(all_store_query):
        ax = plt.subplot(n_subplots, 1, ind_subplot)
        ax.set_xlim(0, 6)
        df = all_store_query[store_name]
        plt.plot(df.index, df, 'o')
        plt.title(store_name)
    plt.show()
<commit_msg>Improve plot of collect csv<commit_after>from os import listdir, path
import pandas as pd
from matplotlib import pyplot as plt

def get_files(directory, file_extension):
    """Get a list of filenames with a specific extension."""
    return filter(lambda f: f.endswith(file_extension), listdir(directory))

def concat_data(file_list):
    """Take multiple csv files and merge the tables in one big table."""
    df_list = []
    for file in file_list:
        with open(file, 'r') as f:
            df_list.append(pd.read_csv(f, index_col='func_name'))
    return pd.concat(df_list)

if __name__ == '__main__':
    dirs = {1: '../bench_results/raw/selected_1graph/',
            5: '../bench_results/raw/selected_5graph/',
            10: '../bench_results/raw/selected_10graph/'}

    means = {k: None for k in dirs.keys()}

    # Get the means for all runs
    for n_graph, checkup_dir in dirs.items():
        files = get_files(checkup_dir, '.csv')
        csv_dir = path.dirname(checkup_dir)
        abs_files = map(lambda f: path.join(csv_dir, f), files)  # list of csv files with absolute paths

        big_df = concat_data(abs_files)
        mean = big_df.groupby(big_df.index).mean()
        means[n_graph] = mean
        mean.to_csv(path.join(csv_dir, 'mean.csv'))  # outputs mean

        big_df.groupby(big_df.index).std().to_csv(path.join(csv_dir, 'std.csv'))  # outputs standard deviation

    # Build a dataframe per store
    first_mean_df = means.values()[0]
    store_names = set(first_mean_df.columns)  # populate before we do any intersection, otherwise it's always empty
    for dataframe in means.values():
        store_names.intersection_update(dataframe.columns)

    all_store_query = {}
    # Put empty dataframes in dict with store names as keys
    for store_name in store_names:
        all_store_query[store_name] = pd.DataFrame(index=means.keys(),
                                                   columns=first_mean_df.index.values.tolist())
    # Build each dataframe with the real values
    for n_graph, mean_df in means.items():
        for store_name in store_names:
            for func_name in mean_df.index:
                all_store_query[store_name][func_name][n_graph] = means[n_graph][store_name][func_name]

    # Display all the figures
    print('displaying figure')
    fig = plt.figure()
    n_subplots = len(all_store_query)
    lines, labels = [], []
    for ind_subplot, store_name in enumerate(all_store_query):
        ax = plt.subplot(1, n_subplots, ind_subplot)
        ax.set_xlim(min(dirs.keys()) - 1, max(dirs.keys()) + 1)
        df = all_store_query[store_name]
        plot_lines = plt.plot(df.index, df, 'o')
        if ind_subplot == 0:
            lines += plot_lines
            labels += df.columns.values.tolist()
        plt.title(store_name)
    fig.legend(lines, labels, loc='upper right')
    plt.show()
<|endoftext|>


<commit_before>import os
import sys
import unittest

from bento.core.node \
    import \
        Node, NodeWithBuild, create_root_with_source_tree

def get_root():
    # FIXME: not exactly...
    if sys.platform == "win32":
        return ""
    else:
        return "/"

class TestNode(unittest.TestCase):
    def setUp(self):
        self.root = Node("", None)

        if sys.version_info[0] < 3:
            self._string_classes = [str, unicode]
        else:
            self._string_classes = [str]

    def test_scratch_creation(self):
        root = Node("", None)
        self.assertEqual(root.abspath(), get_root())

    def test_find_node(self):
        r_n = os.path.abspath(os.getcwd())

        for f in self._string_classes:
            n = self.root.find_node(f(os.getcwd()))
            self.assertTrue(n)
            assert n.abspath() == r_n

    def test_make_node(self):
        r_n = os.path.abspath(os.getcwd())

        for f in self._string_classes:
            n = self.root.make_node(f(os.getcwd()))
            self.assertTrue(n)
            assert n.abspath() == r_n, "%s vs %s" % (n.abspath(), r_n)

class TestNodeWithBuild(unittest.TestCase):
    def setUp(self):
        top = os.getcwd()
        build = os.path.join(os.getcwd(), "_tmp_build")

        self.root = create_root_with_source_tree(top, build)

    def test_root(self):
        self.assertEqual(self.root.abspath(), get_root())

    def test_cwd_node(self):
        cur_node = self.root.make_node(os.getcwd())
        self.assertEqual(os.getcwd(), cur_node.abspath())
<commit_msg>REF: remove useless get_root test function.<commit_after>import os
import sys
import unittest

from bento.core.node \
    import \
        Node, create_root_with_source_tree, find_root

class TestNode(unittest.TestCase):
    def setUp(self):
        self.root = Node("", None)

        if sys.version_info[0] < 3:
            self._string_classes = [str, unicode]
        else:
            self._string_classes = [str]

    def test_scratch_creation(self):
        root = Node("", None)
        self.assertEqual(root, root)

    def test_find_node(self):
        r_n = os.path.abspath(os.getcwd())

        for f in self._string_classes:
            n = self.root.find_node(f(os.getcwd()))
            self.assertTrue(n)
            assert n.abspath() == r_n

    def test_make_node(self):
        r_n = os.path.abspath(os.getcwd())

        for f in self._string_classes:
            n = self.root.make_node(f(os.getcwd()))
            self.assertTrue(n)
            assert n.abspath() == r_n, "%s vs %s" % (n.abspath(), r_n)

class TestNodeWithBuild(unittest.TestCase):
    def setUp(self):
        top = os.getcwd()
        build = os.path.join(os.getcwd(), "_tmp_build")

        self.root = create_root_with_source_tree(top, build)

    def test_root(self):
        self.assertEqual(self.root, find_root(self.root))

    def test_cwd_node(self):
        cur_node = self.root.make_node(os.getcwd())
        self.assertEqual(os.getcwd(), cur_node.abspath())
<|endoftext|>


<commit_before>import os
import csv
from datetime import datetime
from random import choice
from textwrap import dedent
from itertools import groupby
import requests
from slackbot import settings
from slackbot.bot import respond_to
from sqlalchemy import func
from db import Session
from utils import get_user_name
from beproudbot.plugins.redbull_models import RedbullHistory

# ランダムな文字列生成する為、定義
ALLOWED_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'

_cache = {'token': None}

@respond_to('^redbull\s+count$')
def count_redbull_stock(message):
    """現在のRedBullの在庫本数を返すコマンド
    """
    s = Session()
    q = s.query(func.sum(RedbullHistory.delta).label('total_delta'))
    total_delta = q.one().total_delta
    if total_delta is None:
        total_delta = 0
    message.send('レッドブル残り %s 本' % total_delta)

@respond_to('^redbull\s+([-]?[0-9]+)$')
def manage_redbull_stock(message, delta):
    """RedBullの本数の増減を行うコマンド

    :param str delta: POSTされた増減する本数
    """
    delta = -int(delta)
    user_name = get_user_name(message.body['user'])

    s = Session()
    s.add(RedbullHistory(who=user_name, delta=delta))
    s.commit()

    if delta > 0:
        message.send('レッドブルが%sにより%s本投入されました'
                     % (user_name, delta))
    else:
        message.send('レッドブルが%sにより%s本消費されました'
                     % (user_name, -delta))

@respond_to('^redbull\s+history$')
def show_redbull_user_history(message):
    """RedBullのUserごとの消費履歴を返すコマンド
    """
    user_name = get_user_name(message.body['user'])
    s = Session()
    qs = (s.query(RedbullHistory)
          .filter(RedbullHistory.who == user_name,
                  RedbullHistory.delta < 0)
          .order_by(RedbullHistory.id.asc()))
    tmp = []
    for line in qs:
        _created_at = line.created_at.strftime('%Y年%m月%d日 %H:%M:%S')
        tmp.append('[%s]  %s本' % (_created_at, -line.delta))

    ret = '消費履歴はありません'
    if tmp:
        ret = '\n'.join(tmp)

    message.send('%sの消費したレッドブル:\n%s' % (user_name, ret))

@respond_to('^redbull\s+csv$')
def show_redbull_history_csv(message):
    """RedBullの月単位の消費履歴をCSVに出力する
    """
    s = Session()
    consume_hisotry = (s.query(RedbullHistory)
                       .filter(RedbullHistory.delta > 0)
                       .order_by(RedbullHistory.id.desc()))

    # func.month関数を使って月ごとでgroupby count書けるが、
    # SQLiteにはMONTH()関数がないので月集計はPythonで処理する
    def grouper(item):
        return item.created_at.year, item.created_at.month

    ret = []
    for ((year, month), items) in groupby(consume_hisotry, grouper):
        for item in items:
            ret.append([
                '%s/%s' % (year, month),
                item.delta,
            ])

    current_now = datetime.now()
    csv_filepath = os.path.join(settings.TMP_PATH,
                                'redbull_hisotry_%s.csv' % current_now)

    with open(csv_filepath, 'w') as f:
        w = csv.writer(f)
        w.writerows(ret)

    with open(csv_filepath, 'rb') as f:
        param = {
            'token': settings.API_TOKEN,
            'channels': message.body['channel'],
            'title': 'RedBull Consume History'
        }
        requests.post(settings.FILE_UPLOAD_URL,
                      params=param,
                      files={'file': f})

    if os.path.isfile(csv_filepath):
        os.remove(csv_filepath)

@respond_to('^redbull\s+clear$')
@respond_to('^redbull\s+clear\s+([a-zA-Z]+)$')
def clear_redbull_history(message, token=None):
    """RedBullの履歴データを削除するコマンド

    `$redbull clear` のみだと削除するためのトークンを生成して表示する
    `$redbull clear 表示されたトークン` をPOSTすると
        redbull_historyテーブルのレコード全削除を行う

    :param str token: `$redbull clear` の後に入力されたトークン
    """
    if token is None:
        _cache['token'] = ''.join(choice(ALLOWED_CHARS) for i in range(16))
        message.send('履歴をDBからすべてクリアします。'
                     '続けるには\n$redbull clear %s\nと書いてください'
                     % _cache['token'])
        return

    if token == _cache['token']:
        s = Session()
        s.query(RedbullHistory).delete()
        s.commit()
        message.send('履歴をクリアしました')
    else:
        message.send('コマンドが一致しないため'
                     '履歴をクリアできませんでした')

@respond_to('^redbull\s+help$')
def show_help_redbull_commands(message):
    """RedBullコマンドのhelpを表示
    """
    help_text = dedent('''\
    ```
    $redbull count        : RedBullの残り本数を表示する
    $redbull (num)        : numの数だけRedBullの本数を減らす
    $redbull -(num)       : numの数だけRedBullの本数を増やす
    $redbull history      : 自分のRedBullの消費履歴を表示する
    $redbull clear        : RedBullのDBデータを削除する為の認証tokenを表示する
    $redbull clear token  : tokenが合っていればRedBull履歴データを削除する
    $redbull csv          : RedBullの月単位の消費履歴をCSV形式で表示する

    ```''')
    message.send(help_text)
<commit_msg>refs #7 定義されたコマンド実装終了<commit_after>import os
import csv
from datetime import datetime
from random import choice
from textwrap import dedent
from itertools import groupby
import requests
from slackbot import settings
from slackbot.bot import respond_to
from sqlalchemy import func
from db import Session
from utils import get_user_name
from beproudbot.plugins.redbull_models import RedbullHistory

# ランダムな文字列生成する為、定義
ALLOWED_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'

_cache = {'token': None}

@respond_to('^redbull\s+count$')
def count_redbull_stock(message):
    """現在のRedBullの在庫本数を返すコマンド
    """
    s = Session()
    q = s.query(func.sum(RedbullHistory.delta).label('total_delta'))
    total_delta = q.one().total_delta
    if total_delta is None:
        total_delta = 0
    message.send('レッドブル残り %s 本' % total_delta)

@respond_to('^redbull\s+([-]?[0-9]+)$')
def manage_redbull_stock(message, delta):
    """RedBullの本数の増減を行うコマンド

    :param str delta: POSTされた増減する本数
    """
    delta = -int(delta)
    user_name = get_user_name(message.body['user'])

    s = Session()
    s.add(RedbullHistory(who=user_name, delta=delta))
    s.commit()

    if delta > 0:
        message.send('レッドブルが%sにより%s本投入されました'
                     % (user_name, delta))
    else:
        message.send('レッドブルが%sにより%s本消費されました'
                     % (user_name, -delta))

@respond_to('^redbull\s+history$')
def show_user_redbull_history(message):
    """RedBullのUserごとの消費履歴を返すコマンド
    """
    user_name = get_user_name(message.body['user'])
    s = Session()
    qs = (s.query(RedbullHistory)
          .filter(RedbullHistory.who == user_name,
                  RedbullHistory.delta < 0)
          .order_by(RedbullHistory.id.asc()))
    tmp = []
    for line in qs:
        _created_at = line.created_at.strftime('%Y年%m月%d日 %H:%M:%S')
        tmp.append('[%s]  %s本' % (_created_at, -line.delta))

    ret = '消費履歴はありません'
    if tmp:
        ret = '\n'.join(tmp)

    message.send('%sの消費したレッドブル:\n%s' % (user_name, ret))

@respond_to('^redbull\s+csv$')
def show_redbull_history_csv(message):
    """RedBullの月単位の消費履歴をCSVに出力する
    """
    s = Session()
    consume_hisotry = (s.query(RedbullHistory)
                       .filter(RedbullHistory.delta > 0)
                       .order_by(RedbullHistory.id.desc()))

    # func.month関数を使って月ごとでgroupby count書けるが、
    # SQLiteにはMONTH()関数がないので月集計はPythonで処理する
    def grouper(item):
        return item.created_at.year, item.created_at.month

    ret = []
    for ((year, month), items) in groupby(consume_hisotry, grouper):
        for item in items:
            ret.append([
                '%s/%s' % (year, month),
                item.delta,
            ])

    current_now = datetime.now()
    csv_filepath = os.path.join(settings.TMP_PATH,
                                'redbull_hisotry_%s.csv' % current_now)

    with open(csv_filepath, 'w') as f:
        w = csv.writer(f)
        w.writerows(ret)

    with open(csv_filepath, 'rb') as f:
        param = {
            'token': settings.API_TOKEN,
            'channels': message.body['channel'],
            'title': 'RedBull Consume History'
        }
        requests.post(settings.FILE_UPLOAD_URL,
                      params=param,
                      files={'file': f})

    if os.path.isfile(csv_filepath):
        os.remove(csv_filepath)

@respond_to('^redbull\s+clear$')
@respond_to('^redbull\s+clear\s+([a-zA-Z]+)$')
def clear_redbull_history(message, token=None):
    """RedBullの履歴データを削除するコマンド

    `$redbull clear` のみだと削除するためのトークンを生成して表示する
    `$redbull clear 表示されたトークン` をPOSTすると
        redbull_historyテーブルのレコード全削除を行う

    :param str token: `$redbull clear` の後に入力されたトークン
    """
    if token is None:
        _cache['token'] = ''.join(choice(ALLOWED_CHARS) for i in range(16))
        message.send('履歴をDBからすべてクリアします。'
                     '続けるには\n$redbull clear %s\nと書いてください'
                     % _cache['token'])
        return

    if token == _cache['token']:
        s = Session()
        s.query(RedbullHistory).delete()
        s.commit()
        message.send('履歴をクリアしました')
    else:
        message.send('コマンドが一致しないため'
                     '履歴をクリアできませんでした')

@respond_to('^redbull\s+help$')
def show_help_redbull_commands(message):
    """RedBullコマンドのhelpを表示
    """
    help_text = dedent('''\
    ```
    $redbull count        : RedBullの残り本数を表示する
    $redbull (num)        : numの数だけRedBullの本数を減らす
    $redbull -(num)       : numの数だけRedBullの本数を増やす
    $redbull history      : 自分のRedBullの消費履歴を表示する
    $redbull clear        : RedBullのDBデータを削除する為の認証tokenを表示する
    $redbull clear token  : tokenが合っていればRedBull履歴データを削除する
    $redbull csv          : RedBullの月単位の消費履歴をCSV形式で表示する
    ```''')
    message.send(help_text)
<|endoftext|>


<commit_before>"""Beautiful and helpful exceptions

Just set your `BETTER_EXCEPTIONS` environment variable. It handles the rest.

   Name: better_exceptions
 Author: Josh Junon
  Email: josh@junon.me
    URL: github.com/qix-/better-exceptions
License: Copyright (c) 2017 Josh Junon, licensed under the MIT license
"""

from __future__ import absolute_import
from __future__ import print_function

import logging
import sys

from .formatter import THEME, MAX_LENGTH, PIPE_CHAR, CAP_CHAR, ExceptionFormatter
from .encoding import to_byte
from .context import PY3
from .color import SUPPORTS_COLOR, SHOULD_ENCODE, STREAM
from .log import BetExcLogger, patch as patch_logging
from .repl import interact, get_repl

__version__ = '0.3.2'

THEME = THEME.copy()  # Users customizing the theme should not impact core

def write_stream(data, stream=STREAM):
    if SHOULD_ENCODE:
        data = to_byte(data)

        if PY3:
            stream.buffer.write(data)
        else:
            stream.write(data)
    else:
        stream.write(data)

def format_exception(exc, value, tb):
    # Rebuild each time to take into account any changes made by the user to the global parameters
    formatter = ExceptionFormatter(colored=SUPPORTS_COLOR, theme=THEME, max_length=MAX_LENGTH,
                                   pipe_char=PIPE_CHAR, cap_char=CAP_CHAR)
    return list(formatter.format_exception(exc, value, tb))

def excepthook(exc, value, tb):
    formatted = u''.join(format_exception(exc, value, tb))
    write_stream(formatted, STREAM)

def hook():
    sys.excepthook = excepthook

    logging.setLoggerClass(BetExcLogger)
    patch_logging()

    if hasattr(sys, 'ps1'):
        print('WARNING: better_exceptions will only inspect code from the command line\n'
              '         when using: `python -m better_exceptions\'. Otherwise, only code\n'
              '         loaded from files will be inspected!', file=sys.stderr)
<commit_msg>0.3.3<commit_after>"""Beautiful and helpful exceptions

Just set your `BETTER_EXCEPTIONS` environment variable. It handles the rest.

   Name: better_exceptions
 Author: Josh Junon
  Email: josh@junon.me
    URL: github.com/qix-/better-exceptions
License: Copyright (c) 2017 Josh Junon, licensed under the MIT license
"""

from __future__ import absolute_import
from __future__ import print_function

import logging
import sys

from .formatter import THEME, MAX_LENGTH, PIPE_CHAR, CAP_CHAR, ExceptionFormatter
from .encoding import to_byte
from .context import PY3
from .color import SUPPORTS_COLOR, SHOULD_ENCODE, STREAM
from .log import BetExcLogger, patch as patch_logging
from .repl import interact, get_repl

__version__ = '0.3.3'

THEME = THEME.copy()  # Users customizing the theme should not impact core

def write_stream(data, stream=STREAM):
    if SHOULD_ENCODE:
        data = to_byte(data)

        if PY3:
            stream.buffer.write(data)
        else:
            stream.write(data)
    else:
        stream.write(data)

def format_exception(exc, value, tb):
    # Rebuild each time to take into account any changes made by the user to the global parameters
    formatter = ExceptionFormatter(colored=SUPPORTS_COLOR, theme=THEME, max_length=MAX_LENGTH,
                                   pipe_char=PIPE_CHAR, cap_char=CAP_CHAR)
    return list(formatter.format_exception(exc, value, tb))

def excepthook(exc, value, tb):
    formatted = u''.join(format_exception(exc, value, tb))
    write_stream(formatted, STREAM)

def hook():
    sys.excepthook = excepthook

    logging.setLoggerClass(BetExcLogger)
    patch_logging()

    if hasattr(sys, 'ps1'):
        print('WARNING: better_exceptions will only inspect code from the command line\n'
              '         when using: `python -m better_exceptions\'. Otherwise, only code\n'
              '         loaded from files will be inspected!', file=sys.stderr)
<|endoftext|>


<commit_before>'''This package controls syndication for the :mod:papers, mainly for :class:`~papers.models.Publication` objects.

The available feeds are as follows:

+--------------------+---------------------------+---------------------------------------------+
| Feed               | Feed Location             | Feed Class                                  |
+====================+===========================+=============================================+
| Laboratory papers  | /feeds/lab-papers         | :class:`~papers.feeds.LabPapersFeed         |
+--------------------+---------------------------+---------------------------------------------+
| Interesting papers | /feeds/interesting-papers | :class:`~papers.feeds.InterestingPapersFeed |
+--------------------+---------------------------+---------------------------------------------+

A main page describing all feeds is available at **/feeds**.
'''

from django.contrib.syndication.views import Feed
from django.core.urlresolvers import reverse
from django.conf import settings

from papers.models import Publication

class LabPapersFeed(Feed):
    title = "Papers From the %s" % settings.LAB_NAME
    link = "/feeds/lab-papers"
    description = "This feed contains papers published by members of the %s." % settings.LAB_NAME

    def items(self):
        return Publication.objects.filter(laboratory_paper=True)

    def item_title(self, item):
        return item.title

    def item_description(self, item):
        return item.abstract

class InterestingPapersFeed(Feed):
    title = "Papers of Interest"
    link = "/feeds/interesting-papers"
    description = "This feed contains papers that the %s is particularly interested in." % settings.LAB_NAME

    def items(self):
        return Publication.objects.filter(interesting_paper=True)

    def item_title(self, item):
        return item.title

    def item_description(self, item):
        return item.abstract<commit_msg>Subclassed out PaperFeed to make maintenance easier.  Part of issue #18 and issue #26.<commit_after>'''This package controls syndication for the :mod:papers, mainly for :class:`~papers.models.Publication` objects.

The available feeds are as follows:

+--------------------+---------------------------+---------------------------------------------+
| Feed               | Feed Location             | Feed Class                                  |
+====================+===========================+=============================================+
| Laboratory papers  | /feeds/lab-papers         | :class:`~papers.feeds.LabPapersFeed         |
+--------------------+---------------------------+---------------------------------------------+
| Interesting papers | /feeds/interesting-papers | :class:`~papers.feeds.InterestingPapersFeed |
+--------------------+---------------------------+---------------------------------------------+

A main page describing all feeds is available at **/feeds**.
'''

from django.contrib.syndication.views import Feed
from django.core.urlresolvers import reverse
from django.conf import settings

from papers.models import Publication

class PapersFeed(Feed):
    '''This is the main class for all feeds related to publications.'''

    def item_title(self, item):
        return item.title

    def item_description(self, item):
        return item.abstract

class LabPapersFeed(PapersFeed):
    title = "Papers From the %s" % settings.LAB_NAME
    link = "/feeds/lab-papers"
    description = "This feed contains papers published by members of the %s." % settings.LAB_NAME

    def items(self):
        return Publication.objects.filter(laboratory_paper=True)

class InterestingPapersFeed(PapersFeed):
    title = "Papers of Interest"
    link = "/feeds/interesting-papers"
    description = "This feed contains papers that the %s is particularly interested in." % settings.LAB_NAME

    def items(self):
        return Publication.objects.filter(interesting_paper=True)<|endoftext|>


<commit_before>#! /usr/bin/env python
#coding=utf-8
#
# Author: limodou@gmail.com
# This program is based on pyPEG
#
# license: BSD
#
from pyPEG import *
import re
import types

__author__ = 'limodou'
__author_email__ = 'limodou@gmail.com'
__url__ = 'https://github.com/limodou/par'
__license__ = 'BSD'
__version__ = '0.8'

_ = re.compile

class SimpleVisitor(object):
    def __init__(self, grammar=None):
        self.grammar = grammar

    def visit(self, nodes, root=False):
        buf = []
        if not isinstance(nodes, (list, tuple)):
            nodes = [nodes]
        if root:
            method = getattr(self, '__begin__', None)
            if method:
                buf.append(method())
        for node in nodes:
            if isinstance(node, (str, unicode)):
                buf.append(node)
            else:
                if hasattr(self, 'before_visit'):
                    buf.append(self.before_visit(node))
                method = getattr(self, 'visit_' + node.__name__ + '_begin', None)
                if method:
                    buf.append(method(node))
                method = getattr(self, 'visit_' + node.__name__, None)
                if method:
                    buf.append(method(node))
                else:
                    if isinstance(node.what, (str, unicode)):
                        buf.append(node.what)
                    else:
                        buf.append(self.visit(node.what))
                method = getattr(self, 'visit_' + node.__name__ + '_end', None)
                if method:
                    buf.append(method(node))
                if hasattr(self, 'after_visit'):
                    buf.append(self.after_visit(node))

        if root:
            method = getattr(self, '__end__', None)
            if method:
                buf.append(method())
        return ''.join(buf)
<commit_msg>ready for release 0.9<commit_after>#! /usr/bin/env python
#coding=utf-8
#
# Author: limodou@gmail.com
# This program is based on pyPEG
#
# license: BSD
#
from pyPEG import *
import re
import types

__author__ = 'limodou'
__author_email__ = 'limodou@gmail.com'
__url__ = 'https://github.com/limodou/par'
__license__ = 'BSD'
__version__ = '0.9'

_ = re.compile

class SimpleVisitor(object):
    def __init__(self, grammar=None):
        self.grammar = grammar

    def visit(self, nodes, root=False):
        buf = []
        if not isinstance(nodes, (list, tuple)):
            nodes = [nodes]
        if root:
            method = getattr(self, '__begin__', None)
            if method:
                buf.append(method())
        for node in nodes:
            if isinstance(node, (str, unicode)):
                buf.append(node)
            else:
                if hasattr(self, 'before_visit'):
                    buf.append(self.before_visit(node))
                method = getattr(self, 'visit_' + node.__name__ + '_begin', None)
                if method:
                    buf.append(method(node))
                method = getattr(self, 'visit_' + node.__name__, None)
                if method:
                    buf.append(method(node))
                else:
                    if isinstance(node.what, (str, unicode)):
                        buf.append(node.what)
                    else:
                        buf.append(self.visit(node.what))
                method = getattr(self, 'visit_' + node.__name__ + '_end', None)
                if method:
                    buf.append(method(node))
                if hasattr(self, 'after_visit'):
                    buf.append(self.after_visit(node))

        if root:
            method = getattr(self, '__end__', None)
            if method:
                buf.append(method())
        return ''.join(buf)
<|endoftext|>


<commit_before>#! /usr/bin/env python
#coding=utf-8
#
# Author: limodou@gmail.com
# This program is based on pyPEG
#
# license: BSD
#
from pyPEG import *
import re
import types

__author__ = 'limodou'
__author_email__ = 'limodou@gmail.com'
__url__ = 'https://github.com/limodou/par'
__license__ = 'BSD'
__version__ = '0.7'

_ = re.compile

class SimpleVisitor(object):
    def __init__(self, grammar=None):
        self.grammar = grammar

    def visit(self, nodes, root=False):
        buf = []
        if not isinstance(nodes, (list, tuple)):
            nodes = [nodes]
        if root:
            method = getattr(self, '__begin__', None)
            if method:
                buf.append(method())
        for node in nodes:
            if isinstance(node, (str, unicode)):
                buf.append(node)
            else:
                if hasattr(self, 'before_visit'):
                    buf.append(self.before_visit(node))
                method = getattr(self, 'visit_' + node.__name__ + '_begin', None)
                if method:
                    buf.append(method(node))
                method = getattr(self, 'visit_' + node.__name__, None)
                if method:
                    buf.append(method(node))
                else:
                    if isinstance(node.what, (str, unicode)):
                        buf.append(node.what)
                    else:
                        buf.append(self.visit(node.what))
                method = getattr(self, 'visit_' + node.__name__ + '_end', None)
                if method:
                    buf.append(method(node))
                if hasattr(self, 'after_visit'):
                    buf.append(self.after_visit(node))

        if root:
            method = getattr(self, '__end__', None)
            if method:
                buf.append(method())
        return ''.join(buf)
<commit_msg>change version to 0.8<commit_after>#! /usr/bin/env python
#coding=utf-8
#
# Author: limodou@gmail.com
# This program is based on pyPEG
#
# license: BSD
#
from pyPEG import *
import re
import types

__author__ = 'limodou'
__author_email__ = 'limodou@gmail.com'
__url__ = 'https://github.com/limodou/par'
__license__ = 'BSD'
__version__ = '0.8'

_ = re.compile

class SimpleVisitor(object):
    def __init__(self, grammar=None):
        self.grammar = grammar

    def visit(self, nodes, root=False):
        buf = []
        if not isinstance(nodes, (list, tuple)):
            nodes = [nodes]
        if root:
            method = getattr(self, '__begin__', None)
            if method:
                buf.append(method())
        for node in nodes:
            if isinstance(node, (str, unicode)):
                buf.append(node)
            else:
                if hasattr(self, 'before_visit'):
                    buf.append(self.before_visit(node))
                method = getattr(self, 'visit_' + node.__name__ + '_begin', None)
                if method:
                    buf.append(method(node))
                method = getattr(self, 'visit_' + node.__name__, None)
                if method:
                    buf.append(method(node))
                else:
                    if isinstance(node.what, (str, unicode)):
                        buf.append(node.what)
                    else:
                        buf.append(self.visit(node.what))
                method = getattr(self, 'visit_' + node.__name__ + '_end', None)
                if method:
                    buf.append(method(node))
                if hasattr(self, 'after_visit'):
                    buf.append(self.after_visit(node))

        if root:
            method = getattr(self, '__end__', None)
            if method:
                buf.append(method())
        return ''.join(buf)
<|endoftext|>


<commit_before>from parcels.field import Field, UnitConverter, Geographic, GeographicPolar
from parcels.particleset import ParticleSet
import numpy as np
from py import path
from glob import glob
from collections import defaultdict

__all__ = ['Grid']

def unit_converters(mesh):
    if mesh == 'spherical':
        u_units = GeographicPolar()
        v_units = Geographic()
    elif mesh == 'flat':
        u_units = None
        v_units = None
    else:
        raise ValueError("Unsupported mesh type. Choose either: 'spherical' or 'flat'")
    return u_units, v_units

class Grid(object):
    """Grid class used to generate and read NEMO output files

    :param U: :class:`Field` for zonal velocity component
    :param V: :class:`Field` for meridional velocity component
    :param depth: Depth coordinates of the grid
    :param time: Time coordinates of the grid
    :param fields: Dictionary of additional fields
    """
    def __init__(self, U, V, depth, time, fields={}):
        self.U = U
        self.V = V
        self.depth = depth
        self.time = time

        # Add additional fields as attributes
        for name, field in fields.items():
            setattr(self, name, field)

    @classmethod
    def from_data(cls, data_u, lon_u, lat_u, data_v, lon_v, lat_v,
                  depth=None, time=None, field_data={}, transpose=True,
                  mesh='spherical', **kwargs):
        """Initialise Grid object from raw data

        :param data_u: Zonal velocity data
        :param lon_u: Longitude coordinates of the U components
        :param lat_u: Latitude coordinates of the U components
        :param data_v: Meridional velocity data
        :param lon_v: Longitude coordinates of the V components
        :param lat_v: Latitude coordinates of the V components
        :param depth: Depth coordinates of the grid
        :param time: Time coordinates of the grid
        :param mesh: String indicating the type of mesh coordinates and
                     units used during velocity interpolation:
                       * sperical (default): Lat and lon in degree, with a
                         correction for zonal velocity U near the poles.
                       * flat: No conversion, lat/lon are assumed to be in m.
        """
        depth = np.zeros(1, dtype=np.float32) if depth is None else depth
        time = np.zeros(1, dtype=np.float64) if time is None else time
        u_units, v_units = unit_converters(mesh)
        # Create velocity fields
        ufield = Field('U', data_u, lon_u, lat_u, depth=depth,
                       time=time, transpose=transpose,
                       units=u_units, **kwargs)
        vfield = Field('V', data_v, lon_v, lat_v, depth=depth,
                       time=time, transpose=transpose,
                       units=v_units, **kwargs)
        # Create additional data fields
        fields = {}
        for name, data in field_data.items():
            fields[name] = Field(name, data, lon_v, lat_u, depth=depth,
                                 time=time, transpose=transpose, **kwargs)
        return cls(ufield, vfield, depth, time, fields=fields)

    @classmethod
    def from_netcdf(cls, filenames, variables, dimensions,
                    mesh='spherical', **kwargs):
        """Initialises grid data from files using NEMO conventions.

        :param filenames: Dictionary mapping variables to file(s). The
        filepath may contain wildcards to indicate multiple files.
        :param variabels: Dictionary mapping variables to variable
        names in the netCDF file(s).
        :param dimensions: Dictionary mapping data dimensions (lon,
        lat, depth, time, data) to dimensions in the netCF file(s).
        :param mesh: String indicating the type of mesh coordinates and
                     units used during velocity interpolation:
                       * sperical (default): Lat and lon in degree, with a
                         correction for zonal velocity U near the poles.
                       * flat: No conversion, lat/lon are assumed to be in m.
        """
        # Determine unit converters for all fields
        u_units, v_units = unit_converters(mesh)
        units = defaultdict(UnitConverter)
        units.update({'U': u_units, 'V': v_units})
        fields = {}
        for var, name in variables.items():
            # Resolve all matching paths for the current variable
            basepath = path.local(filenames[var])
            paths = [path.local(fp) for fp in glob(str(basepath))]
            if len(paths) == 0:
                raise IOError("Grid files not found: %s" % str(basepath))
            for fp in paths:
                if not fp.exists():
                    raise IOError("Grid file not found: %s" % str(fp))
            dimensions['data'] = name
            fields[var] = Field.from_netcdf(var, dimensions, paths,
                                            units=units[var], **kwargs)
        u = fields.pop('U')
        v = fields.pop('V')
        return cls(u, v, u.depth, u.time, fields=fields)

    @classmethod
    def from_nemo(cls, basename, uvar='vozocrtx', vvar='vomecrty',
                  extra_vars={}, **kwargs):
        """Initialises grid data from files using NEMO conventions.

        :param basename: Base name of the file(s); may contain
        wildcards to indicate multiple files.
        """
        dimensions = {'lon': 'nav_lon', 'lat': 'nav_lat',
                      'depth': 'depth', 'time': 'time_counter'}
        extra_vars.update({'U': uvar, 'V': vvar})
        filenames = dict([(v, str(path.local("%s%s.nc" % (basename, v))))
                          for v in extra_vars.keys()])
        return cls.from_netcdf(filenames, variables=extra_vars,
                               dimensions=dimensions, **kwargs)

    @property
    def fields(self):
        """List of fields associated with this grid"""
        return [v for v in self.__dict__.values() if isinstance(v, Field)]

    def add_field(self, field):
        setattr(self, field.name, field)

    def ParticleSet(self, *args, **kwargs):
        return ParticleSet(*args, grid=self, **kwargs)

    def eval(self, x, y):
        u = self.U.eval(x, y)
        v = self.V.eval(x, y)
        return u, v

    def write(self, filename):
        """Write flow field to NetCDF file using NEMO convention

        :param filename: Basename of the output fileset"""
        print("Generating NEMO grid output with basename: %s" % filename)

        self.U.write(filename, varname='vozocrtx')
        self.V.write(filename, varname='vomecrty')

        for v in self.fields:
            if (v.name is not 'U') and (v.name is not 'V'):
                v.write(filename)

    def advancetime(self, gridnew):
        """Replace oldest time on grid with newgrid

        :param gridnew: Grid snapshot with which the oldest time has to be replaced"""

        if len(gridnew.time) is not 1:
            raise RuntimeError('New grid needs to have only one snapshot')

        for v in self.fields:
            v.data = np.concatenate((v.data[1:, :, :], getattr(gridnew, v.name).data[:, :, :]), 0)
            v.time = np.concatenate((v.time[1:], getattr(gridnew, v.name).time))

        self.time = np.concatenate((self.time[1:], gridnew.time))
<commit_msg>Allowing grid.fromnetcdf to be called with list of filenames<commit_after>from parcels.field import Field, UnitConverter, Geographic, GeographicPolar
from parcels.particleset import ParticleSet
import numpy as np
from py import path
from glob import glob
from collections import defaultdict

__all__ = ['Grid']

def unit_converters(mesh):
    if mesh == 'spherical':
        u_units = GeographicPolar()
        v_units = Geographic()
    elif mesh == 'flat':
        u_units = None
        v_units = None
    else:
        raise ValueError("Unsupported mesh type. Choose either: 'spherical' or 'flat'")
    return u_units, v_units

class Grid(object):
    """Grid class used to generate and read NEMO output files

    :param U: :class:`Field` for zonal velocity component
    :param V: :class:`Field` for meridional velocity component
    :param depth: Depth coordinates of the grid
    :param time: Time coordinates of the grid
    :param fields: Dictionary of additional fields
    """
    def __init__(self, U, V, depth, time, fields={}):
        self.U = U
        self.V = V
        self.depth = depth
        self.time = time

        # Add additional fields as attributes
        for name, field in fields.items():
            setattr(self, name, field)

    @classmethod
    def from_data(cls, data_u, lon_u, lat_u, data_v, lon_v, lat_v,
                  depth=None, time=None, field_data={}, transpose=True,
                  mesh='spherical', **kwargs):
        """Initialise Grid object from raw data

        :param data_u: Zonal velocity data
        :param lon_u: Longitude coordinates of the U components
        :param lat_u: Latitude coordinates of the U components
        :param data_v: Meridional velocity data
        :param lon_v: Longitude coordinates of the V components
        :param lat_v: Latitude coordinates of the V components
        :param depth: Depth coordinates of the grid
        :param time: Time coordinates of the grid
        :param mesh: String indicating the type of mesh coordinates and
                     units used during velocity interpolation:
                       * sperical (default): Lat and lon in degree, with a
                         correction for zonal velocity U near the poles.
                       * flat: No conversion, lat/lon are assumed to be in m.
        """
        depth = np.zeros(1, dtype=np.float32) if depth is None else depth
        time = np.zeros(1, dtype=np.float64) if time is None else time
        u_units, v_units = unit_converters(mesh)
        # Create velocity fields
        ufield = Field('U', data_u, lon_u, lat_u, depth=depth,
                       time=time, transpose=transpose,
                       units=u_units, **kwargs)
        vfield = Field('V', data_v, lon_v, lat_v, depth=depth,
                       time=time, transpose=transpose,
                       units=v_units, **kwargs)
        # Create additional data fields
        fields = {}
        for name, data in field_data.items():
            fields[name] = Field(name, data, lon_v, lat_u, depth=depth,
                                 time=time, transpose=transpose, **kwargs)
        return cls(ufield, vfield, depth, time, fields=fields)

    @classmethod
    def from_netcdf(cls, filenames, variables, dimensions,
                    mesh='spherical', **kwargs):
        """Initialises grid data from files using NEMO conventions.

        :param filenames: Dictionary mapping variables to file(s). The
        filepath may contain wildcards to indicate multiple files,
        or be a list of file.
        :param variabels: Dictionary mapping variables to variable
        names in the netCDF file(s).
        :param dimensions: Dictionary mapping data dimensions (lon,
        lat, depth, time, data) to dimensions in the netCF file(s).
        :param mesh: String indicating the type of mesh coordinates and
                     units used during velocity interpolation:
                       * sperical (default): Lat and lon in degree, with a
                         correction for zonal velocity U near the poles.
                       * flat: No conversion, lat/lon are assumed to be in m.
        """
        # Determine unit converters for all fields
        u_units, v_units = unit_converters(mesh)
        units = defaultdict(UnitConverter)
        units.update({'U': u_units, 'V': v_units})
        fields = {}
        for var, name in variables.items():
            # Resolve all matching paths for the current variable
            if isinstance(filenames[var], list):
                paths = filenames[var]
            else:
                basepath = path.local(filenames[var])
                paths = [path.local(fp) for fp in glob(str(basepath))]
            if len(paths) == 0:
                raise IOError("Grid files not found: %s" % str(basepath))
            for fp in paths:
                if not fp.exists():
                    raise IOError("Grid file not found: %s" % str(fp))
            dimensions['data'] = name
            fields[var] = Field.from_netcdf(var, dimensions, paths,
                                            units=units[var], **kwargs)
        u = fields.pop('U')
        v = fields.pop('V')
        return cls(u, v, u.depth, u.time, fields=fields)

    @classmethod
    def from_nemo(cls, basename, uvar='vozocrtx', vvar='vomecrty',
                  extra_vars={}, **kwargs):
        """Initialises grid data from files using NEMO conventions.

        :param basename: Base name of the file(s); may contain
        wildcards to indicate multiple files.
        """
        dimensions = {'lon': 'nav_lon', 'lat': 'nav_lat',
                      'depth': 'depth', 'time': 'time_counter'}
        extra_vars.update({'U': uvar, 'V': vvar})
        filenames = dict([(v, str(path.local("%s%s.nc" % (basename, v))))
                          for v in extra_vars.keys()])
        return cls.from_netcdf(filenames, variables=extra_vars,
                               dimensions=dimensions, **kwargs)

    @property
    def fields(self):
        """List of fields associated with this grid"""
        return [v for v in self.__dict__.values() if isinstance(v, Field)]

    def add_field(self, field):
        setattr(self, field.name, field)

    def ParticleSet(self, *args, **kwargs):
        return ParticleSet(*args, grid=self, **kwargs)

    def eval(self, x, y):
        u = self.U.eval(x, y)
        v = self.V.eval(x, y)
        return u, v

    def write(self, filename):
        """Write flow field to NetCDF file using NEMO convention

        :param filename: Basename of the output fileset"""
        print("Generating NEMO grid output with basename: %s" % filename)

        self.U.write(filename, varname='vozocrtx')
        self.V.write(filename, varname='vomecrty')

        for v in self.fields:
            if (v.name is not 'U') and (v.name is not 'V'):
                v.write(filename)

    def advancetime(self, gridnew):
        """Replace oldest time on grid with newgrid

        :param gridnew: Grid snapshot with which the oldest time has to be replaced"""

        if len(gridnew.time) is not 1:
            raise RuntimeError('New grid needs to have only one snapshot')

        for v in self.fields:
            v.data = np.concatenate((v.data[1:, :, :], getattr(gridnew, v.name).data[:, :, :]), 0)
            v.time = np.concatenate((v.time[1:], getattr(gridnew, v.name).time))

        self.time = np.concatenate((self.time[1:], gridnew.time))
<|endoftext|>


<commit_before>from datetime import datetime
from os import getloadavg

from flask import Flask, jsonify, abort, request
import psycopg2
from park_api import scraper, util, env, db
from park_api.forecast import find_forecast
from park_api.crossdomain import crossdomain

app = Flask(__name__)

cache = {}

def user_agent(request):
    ua = request.headers.get("User-Agent")
    return "no user-agent" if ua is None else ua

@app.route("/")
@crossdomain("*")
def get_meta():
    app.logger.info("GET / - " + user_agent(request))

    cities = {}
    for module in env.supported_cities().values():
        city = module.geodata.city
        cities[city.id] = {
                "name": city.name,
                "coords": city.coords,
                "source": city.public_source,
                "url": city.url,
                "active_support": city.active_support,
                "attribution": city.attribution
        }

    return jsonify({
        "cities": cities,
        "api_version": env.API_VERSION,
        "server_version": env.SERVER_VERSION,
        "reference": env.SOURCE_REPOSITORY
    })

@app.route("/status")
@crossdomain("*")
def get_api_status():
    return jsonify({
        "status": "online",
        "server_time": util.utc_now(),
        "load": getloadavg()
    })

@app.route("/<city>")
@crossdomain("*")
def get_lots(city):
    if city == "favicon.ico" or city == "robots.txt":
        abort(404)

    app.logger.info("GET /" + city + " - " + user_agent(request))

    city_module = env.supported_cities().get(city, None)

    if city_module is None:
        app.logger.info("Unsupported city: " + city)
        return ("Error 404: Sorry, '" +
                city +
                "' isn't supported at the current time.", 404)

    if env.LIVE_SCRAPE:
        return jsonify(scraper._live(city_module))

    try:
      with db.cursor() as cursor:
          if city in cache:
              sql = "SELECT timestamp_downloaded FROM parkapi WHERE city=%s ORDER BY timestamp_downloaded DESC LIMIT 1;"
              cursor.execute(sql, (city,))
              ts = cursor.fetchall()[0]["timestamp_downloaded"]
              if cache[city][0] == ts:
                  return cache[city][1]
          sql = "SELECT timestamp_updated, timestamp_downloaded, data" \
                  " FROM parkapi WHERE city=%s ORDER BY timestamp_downloaded DESC LIMIT 1;"
          cursor.execute(sql, (city,))
          raw = cursor.fetchall()[0]
          data = raw["data"]
          cache[city] = (raw["timestamp_downloaded"], jsonify(data))
    except (psycopg2.OperationalError, psycopg2.ProgrammingError) as e:
        app.logger.error("Unable to connect to database: " + str(e))
        abort(500)

    return jsonify(data)

@app.route("/<city>/<lot_id>/timespan")
@crossdomain("*")
def get_longtime_forecast(city, lot_id):
    app.logger.info("GET /%s/%s/timespan %s" %
                    (city, lot_id, user_agent(request)))

    try:
        datetime.strptime(request.args["from"], '%Y-%m-%dT%H:%M:%S')
        datetime.strptime(request.args["to"], '%Y-%m-%dT%H:%M:%S')
    except ValueError:
        return ("Error 400: from and/or to URL params "
                "are not in ISO format, e.g. 2015-06-26T18:00:00", 400)

    data = find_forecast(lot_id, request.args["from"], request.args["to"])
    if data is not None:
        return jsonify(data)
    else:
        abort(404)

@app.route("/coffee")
def make_coffee():
    app.logger.info("GET /coffee - " + user_agent(request))

    return """

I'm a teapot

This server is a teapot, not a coffee machine.

    """, 418
<commit_msg>return correct error if no data is available<commit_after>from datetime import datetime
from os import getloadavg

from flask import Flask, jsonify, abort, request
import psycopg2
from park_api import scraper, util, env, db
from park_api.forecast import find_forecast
from park_api.crossdomain import crossdomain

app = Flask(__name__)

cache = {}

def user_agent(request):
    ua = request.headers.get("User-Agent")
    return "no user-agent" if ua is None else ua

@app.route("/")
@crossdomain("*")
def get_meta():
    app.logger.info("GET / - " + user_agent(request))

    cities = {}
    for module in env.supported_cities().values():
        city = module.geodata.city
        cities[city.id] = {
                "name": city.name,
                "coords": city.coords,
                "source": city.public_source,
                "url": city.url,
                "active_support": city.active_support,
                "attribution": city.attribution
        }

    return jsonify({
        "cities": cities,
        "api_version": env.API_VERSION,
        "server_version": env.SERVER_VERSION,
        "reference": env.SOURCE_REPOSITORY
    })

@app.route("/status")
@crossdomain("*")
def get_api_status():
    return jsonify({
        "status": "online",
        "server_time": util.utc_now(),
        "load": getloadavg()
    })

@app.route("/<city>")
@crossdomain("*")
def get_lots(city):
    if city == "favicon.ico" or city == "robots.txt":
        abort(404)

    app.logger.info("GET /" + city + " - " + user_agent(request))

    city_module = env.supported_cities().get(city, None)

    if city_module is None:
        app.logger.info("Unsupported city: " + city)
        return ("Error 404: Sorry, '" +
                city +
                "' isn't supported at the current time.", 404)

    if env.LIVE_SCRAPE:
        return jsonify(scraper._live(city_module))

    try:
      with db.cursor() as cursor:
          if city in cache:
              sql = "SELECT timestamp_downloaded FROM parkapi WHERE city=%s ORDER BY timestamp_downloaded DESC LIMIT 1;"
              cursor.execute(sql, (city,))
              ts = cursor.fetchall()[0]["timestamp_downloaded"]
              if cache[city][0] == ts:
                  return cache[city][1]
          sql = "SELECT timestamp_updated, timestamp_downloaded, data" \
                  " FROM parkapi WHERE city=%s ORDER BY timestamp_downloaded DESC LIMIT 1;"
          cursor.execute(sql, (city,))
          raw = cursor.fetchall()[0]
          data = raw["data"]
          cache[city] = (raw["timestamp_downloaded"], jsonify(data))
    except IndexError:
        return ("Error 503: There is currently no data available for " + city + ".", 503)
    except (psycopg2.OperationalError, psycopg2.ProgrammingError) as e:
        app.logger.error("Unable to connect to database: " + str(e))
        abort(500)

    return jsonify(data)

@app.route("/<city>/<lot_id>/timespan")
@crossdomain("*")
def get_longtime_forecast(city, lot_id):
    app.logger.info("GET /%s/%s/timespan %s" %
                    (city, lot_id, user_agent(request)))

    try:
        datetime.strptime(request.args["from"], '%Y-%m-%dT%H:%M:%S')
        datetime.strptime(request.args["to"], '%Y-%m-%dT%H:%M:%S')
    except ValueError:
        return ("Error 400: from and/or to URL params "
                "are not in ISO format, e.g. 2015-06-26T18:00:00", 400)

    data = find_forecast(lot_id, request.args["from"], request.args["to"])
    if data is not None:
        return jsonify(data)
    else:
        abort(404)

@app.route("/coffee")
def make_coffee():
    app.logger.info("GET /coffee - " + user_agent(request))

    return """

I'm a teapot

This server is a teapot, not a coffee machine.

    """, 418
<|endoftext|>


<commit_before>from django import forms
from django.forms.models import ModelFormMetaclass, BaseInlineFormSet
from django.utils.translation import get_language
from django.utils import six
from parler.models import TranslationDoesNotExist
from parler.utils import compat

__all__ = (
    'TranslatableModelForm',
    'TranslatedField',
    'TranslatableModelFormMixin',
    #'TranslatableModelFormMetaclass',
)

class TranslatedField(object):
    """
    A wrapper for a translated form field.

    This wrapper can be used to declare translated fields on the form, e.g.

    .. code-block:: python

        class MyForm(TranslatableModelForm):
            title = TranslatedField()
            slug = TranslatedField()

            description = TranslatedField(form_class=forms.CharField, widget=TinyMCE)
    """
    def __init__(self, **kwargs):
        # The metaclass performs the magic replacement with the actual formfield.
        self.kwargs = kwargs

class TranslatableModelFormMixin(object):
    """
    The base methods added to :class:`TranslatableModelForm` to fetch and store translated fields.
    """
    language_code = None    # Set by TranslatableAdmin.get_form() on the constructed subclass.

    def __init__(self, *args, **kwargs):
        current_language = kwargs.pop('_current_language', None)   # Used for TranslatableViewMixin
        super(TranslatableModelFormMixin, self).__init__(*args, **kwargs)

        # Load the initial values for the translated fields
        instance = kwargs.get('instance', None)
        if instance:
            for meta in instance._parler_meta:
                try:
                    # By not auto creating a model, any template code that reads the fields
                    # will continue to see one of the other translations.
                    # This also causes admin inlines to show the fallback title in __unicode__.
                    translation = instance._get_translated_model(meta=meta)
                except TranslationDoesNotExist:
                    pass
                else:
                    for field in meta.get_translated_fields():
                        self.initial.setdefault(field, getattr(translation, field))

        # Typically already set by admin
        if self.language_code is None:
            if instance:
                self.language_code = instance.get_current_language()
                return
            else:
                self.language_code = current_language or get_language()

    def _post_clean(self):
        # Copy the translated fields into the model
        # Make sure the language code is set as early as possible (so it's active during most clean() methods)
        self.instance.set_current_language(self.language_code)
        self.save_translated_fields()

        # Perform the regular clean checks, this also updates self.instance
        super(TranslatableModelFormMixin, self)._post_clean()

    def save_translated_fields(self):
        """
        Save all translated fields.
        """
        # Assign translated fields to the model (using the TranslatedAttribute descriptor)
        for field in self._get_translated_fields():
            try:
                value = self.cleaned_data[field]
            except KeyError:  # Field has a ValidationError
                continue
            setattr(self.instance, field, value)

    def _get_translated_fields(self):
        field_names = self._meta.model._parler_meta.get_all_fields()
        return [f_name for f_name in field_names if f_name in self.fields]

class TranslatableModelFormMetaclass(ModelFormMetaclass):
    """
    Meta class to add translated form fields to the form.
    """
    def __new__(mcs, name, bases, attrs):
        # Before constructing class, fetch attributes from bases list.
        form_meta = _get_mro_attribute(bases, '_meta')
        form_base_fields = _get_mro_attribute(bases, 'base_fields', {})  # set by previous class level.

        if form_meta:
            # Not declaring the base class itself, this is a subclass.

            # Read the model from the 'Meta' attribute. This even works in the admin,
            # as `modelform_factory()` includes a 'Meta' attribute.
            # The other options can be read from the base classes.
            form_new_meta = attrs.get('Meta', form_meta)
            form_model = form_new_meta.model if form_new_meta else form_meta.model

            # Detect all placeholders at this class level.
            translated_fields = [
                f_name for f_name, attr_value in six.iteritems(attrs) if isinstance(attr_value, TranslatedField)
            ]

            # Include the translated fields as attributes, pretend that these exist on the form.
            # This also works when assigning `form = TranslatableModelForm` in the admin,
            # since the admin always uses modelform_factory() on the form class, and therefore triggering this metaclass.
            if form_model:
                for translations_model in form_model._parler_meta.get_all_models():
                    fields = getattr(form_new_meta, 'fields', form_meta.fields)
                    exclude = getattr(form_new_meta, 'exclude', form_meta.exclude) or ()
                    widgets = getattr(form_new_meta, 'widgets', form_meta.widgets) or ()
                    formfield_callback = attrs.get('formfield_callback', None)

                    if fields == '__all__':
                        fields = None

                    for f_name in translations_model.get_translated_fields():
                        # Add translated field if not already added, and respect exclude options.
                        if f_name in translated_fields:
                            # The TranslatedField placeholder can be replaced directly with actual field, so do that.
                            attrs[f_name] = _get_model_form_field(translations_model, f_name, formfield_callback=formfield_callback, **attrs[f_name].kwargs)
                        # The next code holds the same logic as fields_for_model()
                        # The f.editable check happens in _get_model_form_field()
                        elif f_name not in form_base_fields \
                         and (fields is None or f_name in fields) \
                         and f_name not in exclude \
                         and not f_name in attrs:
                            # Get declared widget kwargs
                            if f_name in widgets:
                                # Not combined with declared fields (e.g. the TranslatedField placeholder)
                                kwargs = {'widget': widgets[f_name]}
                            else:
                                kwargs = {}

                            # See if this formfield was previously defined using a TranslatedField placeholder.
                            placeholder = _get_mro_attribute(bases, f_name)
                            if placeholder and isinstance(placeholder, TranslatedField):
                                kwargs.update(placeholder.kwargs)

                            # Add the form field as attribute to the class.
                            formfield = _get_model_form_field(translations_model, f_name, formfield_callback=formfield_callback, **kwargs)
                            if formfield is not None:
                                attrs[f_name] = formfield

        # Call the super class with updated `attrs` dict.
        return super(TranslatableModelFormMetaclass, mcs).__new__(mcs, name, bases, attrs)

def _get_mro_attribute(bases, name, default=None):
    for base in bases:
        try:
            return getattr(base, name)
        except AttributeError:
            continue
    return default

def _get_model_form_field(model, name, formfield_callback=None, **kwargs):
    """
    Utility to create the formfield from a model field.
    When a field is not editable, a ``None`` will be returned.
    """
    field = model._meta.get_field(name)
    if not field.editable:  # see fields_for_model() logic in Django.
        return None

    # Apply admin formfield_overrides
    if formfield_callback is None:
        formfield = field.formfield(**kwargs)
    elif not callable(formfield_callback):
        raise TypeError('formfield_callback must be a function or callable')
    else:
        formfield = formfield_callback(field, **kwargs)

    return formfield

class TranslatableModelForm(compat.with_metaclass(TranslatableModelFormMetaclass, TranslatableModelFormMixin, forms.ModelForm)):
    """
    The model form to use for translated models.
    """
    # six.with_metaclass does not handle more than 2 parent classes for django < 1.6
    # but we need all of them in django 1.7 to pass check admin.E016:
    #       "The value of 'form' must inherit from 'BaseModelForm'"
    # so we use our copied version in parler.utils.compat

class TranslatableBaseInlineFormSet(BaseInlineFormSet):
    """
    The formset base for creating inlines with translatable models.
    """
    language_code = None

    def _construct_form(self, i, **kwargs):
        form = super(TranslatableBaseInlineFormSet, self)._construct_form(i, **kwargs)
        form.language_code = self.language_code   # Pass the language code for new objects!
        return form

    def save_new(self, form, commit=True):
        obj = super(TranslatableBaseInlineFormSet, self).save_new(form, commit)
        return obj
<commit_msg>Fix admin for nullable foreign key in translation object<commit_after>from django import forms
from django.forms.models import ModelFormMetaclass, BaseInlineFormSet
from django.utils.translation import get_language
from django.utils import six
from parler.models import TranslationDoesNotExist
from parler.utils import compat

__all__ = (
    'TranslatableModelForm',
    'TranslatedField',
    'TranslatableModelFormMixin',
    #'TranslatableModelFormMetaclass',
)

class TranslatedField(object):
    """
    A wrapper for a translated form field.

    This wrapper can be used to declare translated fields on the form, e.g.

    .. code-block:: python

        class MyForm(TranslatableModelForm):
            title = TranslatedField()
            slug = TranslatedField()

            description = TranslatedField(form_class=forms.CharField, widget=TinyMCE)
    """
    def __init__(self, **kwargs):
        # The metaclass performs the magic replacement with the actual formfield.
        self.kwargs = kwargs

class TranslatableModelFormMixin(object):
    """
    The base methods added to :class:`TranslatableModelForm` to fetch and store translated fields.
    """
    language_code = None    # Set by TranslatableAdmin.get_form() on the constructed subclass.

    def __init__(self, *args, **kwargs):
        current_language = kwargs.pop('_current_language', None)   # Used for TranslatableViewMixin
        super(TranslatableModelFormMixin, self).__init__(*args, **kwargs)

        # Load the initial values for the translated fields
        instance = kwargs.get('instance', None)
        if instance:
            for meta in instance._parler_meta:
                try:
                    # By not auto creating a model, any template code that reads the fields
                    # will continue to see one of the other translations.
                    # This also causes admin inlines to show the fallback title in __unicode__.
                    translation = instance._get_translated_model(meta=meta)
                except TranslationDoesNotExist:
                    pass
                else:
                    for field in meta.get_translated_fields():
                        self.initial.setdefault(field, getattr(translation, field, None))

        # Typically already set by admin
        if self.language_code is None:
            if instance:
                self.language_code = instance.get_current_language()
                return
            else:
                self.language_code = current_language or get_language()

    def _post_clean(self):
        # Copy the translated fields into the model
        # Make sure the language code is set as early as possible (so it's active during most clean() methods)
        self.instance.set_current_language(self.language_code)
        self.save_translated_fields()

        # Perform the regular clean checks, this also updates self.instance
        super(TranslatableModelFormMixin, self)._post_clean()

    def save_translated_fields(self):
        """
        Save all translated fields.
        """
        # Assign translated fields to the model (using the TranslatedAttribute descriptor)
        for field in self._get_translated_fields():
            try:
                value = self.cleaned_data[field]
            except KeyError:  # Field has a ValidationError
                continue
            setattr(self.instance, field, value)

    def _get_translated_fields(self):
        field_names = self._meta.model._parler_meta.get_all_fields()
        return [f_name for f_name in field_names if f_name in self.fields]

class TranslatableModelFormMetaclass(ModelFormMetaclass):
    """
    Meta class to add translated form fields to the form.
    """
    def __new__(mcs, name, bases, attrs):
        # Before constructing class, fetch attributes from bases list.
        form_meta = _get_mro_attribute(bases, '_meta')
        form_base_fields = _get_mro_attribute(bases, 'base_fields', {})  # set by previous class level.

        if form_meta:
            # Not declaring the base class itself, this is a subclass.

            # Read the model from the 'Meta' attribute. This even works in the admin,
            # as `modelform_factory()` includes a 'Meta' attribute.
            # The other options can be read from the base classes.
            form_new_meta = attrs.get('Meta', form_meta)
            form_model = form_new_meta.model if form_new_meta else form_meta.model

            # Detect all placeholders at this class level.
            translated_fields = [
                f_name for f_name, attr_value in six.iteritems(attrs) if isinstance(attr_value, TranslatedField)
            ]

            # Include the translated fields as attributes, pretend that these exist on the form.
            # This also works when assigning `form = TranslatableModelForm` in the admin,
            # since the admin always uses modelform_factory() on the form class, and therefore triggering this metaclass.
            if form_model:
                for translations_model in form_model._parler_meta.get_all_models():
                    fields = getattr(form_new_meta, 'fields', form_meta.fields)
                    exclude = getattr(form_new_meta, 'exclude', form_meta.exclude) or ()
                    widgets = getattr(form_new_meta, 'widgets', form_meta.widgets) or ()
                    formfield_callback = attrs.get('formfield_callback', None)

                    if fields == '__all__':
                        fields = None

                    for f_name in translations_model.get_translated_fields():
                        # Add translated field if not already added, and respect exclude options.
                        if f_name in translated_fields:
                            # The TranslatedField placeholder can be replaced directly with actual field, so do that.
                            attrs[f_name] = _get_model_form_field(translations_model, f_name, formfield_callback=formfield_callback, **attrs[f_name].kwargs)
                        # The next code holds the same logic as fields_for_model()
                        # The f.editable check happens in _get_model_form_field()
                        elif f_name not in form_base_fields \
                         and (fields is None or f_name in fields) \
                         and f_name not in exclude \
                         and not f_name in attrs:
                            # Get declared widget kwargs
                            if f_name in widgets:
                                # Not combined with declared fields (e.g. the TranslatedField placeholder)
                                kwargs = {'widget': widgets[f_name]}
                            else:
                                kwargs = {}

                            # See if this formfield was previously defined using a TranslatedField placeholder.
                            placeholder = _get_mro_attribute(bases, f_name)
                            if placeholder and isinstance(placeholder, TranslatedField):
                                kwargs.update(placeholder.kwargs)

                            # Add the form field as attribute to the class.
                            formfield = _get_model_form_field(translations_model, f_name, formfield_callback=formfield_callback, **kwargs)
                            if formfield is not None:
                                attrs[f_name] = formfield

        # Call the super class with updated `attrs` dict.
        return super(TranslatableModelFormMetaclass, mcs).__new__(mcs, name, bases, attrs)

def _get_mro_attribute(bases, name, default=None):
    for base in bases:
        try:
            return getattr(base, name)
        except AttributeError:
            continue
    return default

def _get_model_form_field(model, name, formfield_callback=None, **kwargs):
    """
    Utility to create the formfield from a model field.
    When a field is not editable, a ``None`` will be returned.
    """
    field = model._meta.get_field(name)
    if not field.editable:  # see fields_for_model() logic in Django.
        return None

    # Apply admin formfield_overrides
    if formfield_callback is None:
        formfield = field.formfield(**kwargs)
    elif not callable(formfield_callback):
        raise TypeError('formfield_callback must be a function or callable')
    else:
        formfield = formfield_callback(field, **kwargs)

    return formfield

class TranslatableModelForm(compat.with_metaclass(TranslatableModelFormMetaclass, TranslatableModelFormMixin, forms.ModelForm)):
    """
    The model form to use for translated models.
    """
    # six.with_metaclass does not handle more than 2 parent classes for django < 1.6
    # but we need all of them in django 1.7 to pass check admin.E016:
    #       "The value of 'form' must inherit from 'BaseModelForm'"
    # so we use our copied version in parler.utils.compat

class TranslatableBaseInlineFormSet(BaseInlineFormSet):
    """
    The formset base for creating inlines with translatable models.
    """
    language_code = None

    def _construct_form(self, i, **kwargs):
        form = super(TranslatableBaseInlineFormSet, self)._construct_form(i, **kwargs)
        form.language_code = self.language_code   # Pass the language code for new objects!
        return form

    def save_new(self, form, commit=True):
        obj = super(TranslatableBaseInlineFormSet, self).save_new(form, commit)
        return obj
<|endoftext|>


<commit_before>import sys

# to use this script, first run 'sample' to sample your libtorrent based process
# the output can then be passed to this script to auto-fold call stacks at
# relevant depths and to filter out low sample counts
f = open(sys.argv[1])

def parse_line(l):
	indentation = 0
	while indentation < len(l) and l[indentation] == ' ':
		indentation += 1
	if indentation == 0:
		return (0, 0, '')

	l = l.strip().split(' ')
	samples = int(l[0])
	fun = ' '.join(l[1:])

	return (indentation, samples, fun)

fold = -1

try:
	sample_limit = int(sys.argv[2])
except:
	sample_limit = 5

fun_samples = {}

for l in f:
	if 'Sort by top of stack' in l: break

	indentation, samples, fun = parse_line(l)
	if samples < sample_limit: continue
	if fold != -1 and indentation > fold: continue
	fold = -1

	if '__gnu_cxx::__normal_iterator<' in fun:
		fold = indentation - 1
		continue

	if 'boost::_bi::bind_t' in fun: continue
	if 'boost::_bi::list' in fun: continue
	if 'boost::_mfi::mf' in fun: continue
	if 'boost::_bi::storage' in fun: continue

# should only add leaves
	if fun in fun_samples: fun_samples[fun] += samples
	else: fun_samples[fun] = samples

	output = '%s%-4d %s' % (' ' * (indentation/2), samples, fun)
	if len(output) > 200: output = output[0:200]
	print output

	if 'invariant_checker_impl' in fun: fold = indentation
	if '::find_POD' in fun: fold = indentation
	if 'SHA1_Update' in fun: fold = indentation
	if 'boost::detail::function::basic_vtable' in fun: fold = indentation
	if 'operator new' in fun: fold = indentation
	if 'malloc' == fun: fold = indentation
	if 'free' == fun: fold = indentation
	if 'std::_Rb_tree' in fun: fold = indentation
	if 'pthread_cond_wait' in fun: fold = indentation
	if 'mp_exptmod' == fun: fold = indentation
	if '::check_invariant()' in fun: fold = indentation
	if 'libtorrent::condition::wait' in fun: fold = indentation
	if 'libtorrent::sleep' in fun: fold = indentation
	if 'puts' == fun: fold = indentation
	if 'boost::asio::basic_stream_socket' in fun: fold = indentation
	if 'recvmsg' == fun: fold = indentation
	if 'sendmsg' == fun: fold = indentation
	if 'semaphore_signal_trap' == fun: fold = indentation
	if 'boost::detail::atomic_count::operator' in fun: fold = indentation
	if 'pthread_mutex_lock' == fun: fold = indentation
	if 'pthread_mutex_unlock' == fun: fold = indentation
	if '>::~vector()' == fun: fold = indentation
	if 'szone_free_definite_size' == fun: fold = indentation
	if 'snprintf' == fun: fold = indentation
	if 'usleep' == fun: fold = indentation
	if 'pthread_mutex_lock' == fun: fold = indentation
	if 'pthread_mutex_unlock' == fun: fold = indentation
	if 'std::string::append' in fun: fold = indentation
	if 'getipnodebyname' == fun: fold = indentation
	if '__gnu_debug::_Safe_iterator<std::' in fun: fold = indentation
	if 'fflush' == fun: fold = indentation
	if 'vfprintf' == fun: fold = indentation
	if 'fprintf' == fun: fold = indentation

list = []
for k in fun_samples:
	list.append((fun_samples[k], k))

list = sorted(list, reverse=True)

for i in list:
	print '%-4d %s' % (i[0], i[1])

<commit_msg>parse_sample update<commit_after>import sys

# to use this script, first run 'sample' to sample your libtorrent based process
# the output can then be passed to this script to auto-fold call stacks at
# relevant depths and to filter out low sample counts
f = open(sys.argv[1])

def parse_line(l):
	indentation = 0
	while indentation < len(l) and l[indentation] == ' ':
		indentation += 1
	if indentation == 0:
		return (0, 0, '')

	l = l.strip().split(' ')
	samples = int(l[0])
	fun = ' '.join(l[1:])

	return (indentation, samples, fun)

fold = -1

try:
	sample_limit = int(sys.argv[2])
except:
	sample_limit = 5

fun_samples = {}

for l in f:
	if 'Sort by top of stack' in l: break

	indentation, samples, fun = parse_line(l)
	if samples < sample_limit: continue
	if fold != -1 and indentation > fold: continue
	fold = -1

	if '__gnu_cxx::__normal_iterator<' in fun:
		fold = indentation - 1
		continue

	if 'boost::_bi::bind_t' in fun: continue
	if 'boost::_bi::list' in fun: continue
	if 'boost::_mfi::mf' in fun: continue
	if 'boost::_bi::storage' in fun: continue

# should only add leaves
	if fun in fun_samples: fun_samples[fun] += samples
	else: fun_samples[fun] = samples

	output = '%s%-4d %s' % (' ' * (indentation/2), samples, fun)
	if len(output) > 200: output = output[0:200]
	print output

	if 'invariant_checker_impl' in fun: fold = indentation
	if '::find_POD' in fun: fold = indentation
	if 'SHA1_Update' in fun: fold = indentation
	if 'boost::detail::function::basic_vtable' in fun: fold = indentation
	if 'operator new' in fun: fold = indentation
	if 'malloc' == fun: fold = indentation
	if 'free' == fun: fold = indentation
	if 'std::_Rb_tree' in fun: fold = indentation
	if 'pthread_cond_wait' in fun: fold = indentation
	if 'mp_exptmod' == fun: fold = indentation
	if '::check_invariant()' in fun: fold = indentation
	if 'libtorrent::condition::wait' in fun: fold = indentation
	if 'libtorrent::sleep' in fun: fold = indentation
	if 'puts' == fun: fold = indentation
	if 'boost::asio::basic_stream_socket' in fun: fold = indentation
	if 'recvmsg' == fun: fold = indentation
	if 'sendmsg' == fun: fold = indentation
	if 'semaphore_signal_trap' == fun: fold = indentation
	if 'boost::detail::atomic_count::operator' in fun: fold = indentation
	if 'pthread_mutex_lock' == fun: fold = indentation
	if 'pthread_mutex_unlock' == fun: fold = indentation
	if '>::~vector()' == fun: fold = indentation
	if 'szone_free_definite_size' == fun: fold = indentation
	if 'snprintf' == fun: fold = indentation
	if 'usleep' == fun: fold = indentation
	if 'pthread_mutex_lock' == fun: fold = indentation
	if 'pthread_mutex_unlock' == fun: fold = indentation
	if 'std::string::append' in fun: fold = indentation
	if 'getipnodebyname' == fun: fold = indentation
	if '__gnu_debug::_Safe_iterator<std::' in fun: fold = indentation
	if 'fflush' == fun: fold = indentation
	if 'vfprintf' == fun: fold = indentation
	if 'fprintf' == fun: fold = indentation
	if 'BN_mod_exp' == fun: fold = indentation
	if 'BN_CTX_free' == fun: fold = indentation
	if 'cerror' == fun: fold = indentation

list = []
for k in fun_samples:
	list.append((fun_samples[k], k))

list = sorted(list, reverse=True)

for i in list:
	print '%-4d %s' % (i[0], i[1])

<|endoftext|>


<commit_before>#!/bin/env python3
# -*- coding: utf-8 -*-

import socket
import parse_msg
import os
import sys
import re
import logging
import csv
from threading import Thread

"""
EmailRelay eml parse threaded server process
date 170607
"""
__author__ = "Koray YILMAZ"
__email__ = 'kyilmaz80@gmail.com'
__version__ = "1.13"

# config constants
HOST = 'parser.test.local'
PORT = 1235
MAX_REQUESTS = 10
PATTERN_FILE = './PATTERNS'
EML_MOUNT = '/mnt/nfs'

class SocketThread(Thread):
    """
    A Listening Socket Thread Per Client
    """
    def __init__(self, name, clientsocket, addr):
        super().__init__()
        self.name = name
        self.clientsocket = clientsocket
        self.addr = addr

    def recieve(self):
        def get_msg_fpath(str_path, nfs_mount):
            '''
            str_path: string return from emailrelay
            nfs_mount: email relay spool out folder
            '''
            eml_name = str_path.split('/')[-1]
            return os.path.join(nfs_mount, eml_name)
        SOCKET_BUFFER = 1024
        data = clientsocket.recv(SOCKET_BUFFER).decode()
        logger.info("Data: %s" % str(data))
        msg_fpath = str(data)
        msg_fpath = msg_fpath.rstrip()
        self.msg_path = get_msg_fpath(msg_fpath, EML_MOUNT)

    def send(self):
        ret_code = parse_msg.main(self.msg_path, patterns, regexes)
        # no match or match free to go
        msg = ''
        if ret_code == 0 or ret_code == 100:
            msg = 'ok' + '\r\n'
        elif ret_code == 1:
            msg = 'Mail Cannot Forward!' + '\r\n'
        else:
            pass
        logger.info('Sending the parse result to EmailRelay server...')
        self.clientsocket.send(msg.encode('ascii'))
        self.clientsocket.close()

    def run(self):
        self.recieve()
        self.send()

def load_patterns(file_path):
    """
    loads the patterns from csv file
    """
    patternsDict = {}
    try:
        with open(file_path, mode='r') as csvfile:
            reader = csv.reader(csvfile)
            patternsDict = {rows[0]: (rows[1], rows[2]) for rows in reader}
    except FileNotFoundError:
        logging.error("patterns file not found!")
    return patternsDict

logging.basicConfig(format='%(asctime)s:%(name)s:%(threadName)s:' +
                    '%(levelname)s:%(message)s', filemode='a',
                    level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.info('Pattern tree generating only once')

# load patterns
patterns = load_patterns(PATTERN_FILE)
patternsList = [t[0] for t in patterns.values()]
regexes = [re.compile(p) for p in patternsList]

# create a socket object
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
logger.info('Server socket opened...')

# bind to the port
try:
    serversocket.bind((HOST, PORT))
    logger.info('%s:%s bind done', HOST, PORT)
except Exception as e:
    logger.error('%s:%s bind failure!', HOST, PORT)
    sys.exit(1)

# queue up to 10 requests
serversocket.listen(MAX_REQUESTS)
while True:
    logger.info('------------------------------------------')
    logger.info('Server is waiting for data on port: %s', PORT)
    clientsocket, addr = serversocket.accept()
    logger.info("Got a connection from %s" % str(addr))
    sock_thread = SocketThread("Thread-" + str(addr[1]),
                               clientsocket, addr)
    sock_thread.start()
<commit_msg>Configuration parameters separated from code<commit_after>#!/bin/env python3
# -*- coding: utf-8 -*-

import socket
import parse_msg
import os
import sys
import re
import logging
import csv
import params
from threading import Thread

"""
EmailRelay eml parse threaded server process
date 170707
"""
__author__ = "Koray YILMAZ"
__email__ = 'kyilmaz80@gmail.com'
__version__ = "1.14"

class SocketThread(Thread):
    """
    A Listening Socket Thread Per Client
    """
    def __init__(self, name, clientsocket, addr):
        super().__init__()
        self.name = name
        self.clientsocket = clientsocket
        self.addr = addr

    def recieve(self):
        def get_msg_fpath(str_path, nfs_mount):
            '''
            str_path: string return from emailrelay
            nfs_mount: email relay spool out folder
            '''
            eml_name = str_path.split('/')[-1]
            return os.path.join(nfs_mount, eml_name)
        SOCKET_BUFFER = 1024
        data = clientsocket.recv(SOCKET_BUFFER).decode()
        logger.info("Data: %s" % str(data))
        msg_fpath = str(data)
        msg_fpath = msg_fpath.rstrip()
        self.msg_path = get_msg_fpath(msg_fpath, params.EML_MOUNT)

    def send(self):
        ret_code = parse_msg.main(self.msg_path, patterns, regexes)
        # no match or match free to go
        msg = ''
        if ret_code == 0 or ret_code == 100:
            msg = 'ok' + '\r\n'
        elif ret_code == 1:
            msg = 'Mail Cannot Forward!' + '\r\n'
        else:
            pass
        logger.info('Sending the parse result to EmailRelay server...')
        self.clientsocket.send(msg.encode('ascii'))
        self.clientsocket.close()

    def run(self):
        self.recieve()
        self.send()

def load_patterns(file_path):
    """
    loads the patterns from csv file
    """
    patternsDict = {}
    try:
        with open(file_path, mode='r') as csvfile:
            reader = csv.reader(csvfile)
            patternsDict = {rows[0]: (rows[1], rows[2]) for rows in reader}
    except FileNotFoundError:
        logging.error("patterns file not found!")
    return patternsDict

logging.basicConfig(format='%(asctime)s:%(name)s:%(threadName)s:' +
                    '%(levelname)s:%(message)s', filemode='a',
                    level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.info('Pattern tree generating only once')

# load patterns
patterns = load_patterns(params.PATTERN_FILE)
patternsList = [t[0] for t in patterns.values()]
regexes = [re.compile(p) for p in patternsList]

# create a socket object
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
logger.info('Server socket opened...')

# bind to the port
try:
    serversocket.bind((params.HOST, params.PORT))
    logger.info('%s:%s bind done', params.HOST, params.PORT)
except Exception as e:
    logger.error('%s:%s bind failure!', params.HOST, params.PORT)
    sys.exit(1)

# queue up to 10 requests
serversocket.listen(params.MAX_REQUESTS)
while True:
    logger.info('------------------------------------------')
    logger.info('Server is waiting for data on port: %s', params.PORT)
    clientsocket, addr = serversocket.accept()
    logger.info("Got a connection from %s" % str(addr))
    sock_thread = SocketThread("Thread-" + str(addr[1]),
                               clientsocket, addr)
    sock_thread.start()
<|endoftext|>


<commit_before>from .core import Interface
from threading import Lock
from toolz import merge_with, topk, accumulate, pluck
from operator import add
from bisect import bisect
from collections import defaultdict
from .compatibility import Queue, Empty

def zero():
    return 0

class Buffer(Interface):
    def __init__(self, fast, slow, available_memory=1e9):
        self.lock = Lock()
        self.fast = fast
        self.slow = slow
        self.available_memory = available_memory
        self.lengths = defaultdict(zero)
        self.memory_usage = 0
        Interface.__init__(self)

    def __getstate__(self):
        return {'fast': self.fast,
                'slow': self.slow,
                'total_memory': self.memory_usage,
                'lengths': self.lengths,
                'available_memory': self.available_memory}

    def __setstate__(self, state):
        Interface.__setstate__(self, state)
        self.lock = Lock()
        self.__dict__.update(state)

    def append(self, data, lock=True, **kwargs):
        if lock: self.lock.acquire()
        try:
            for k, v in data.items():
                self.lengths[k] += len(v)
                self.memory_usage += len(v)
            self.fast.append(data, lock=False, **kwargs)
        finally:
            if lock: self.lock.release()

        while self.memory_usage > self.available_memory:
            keys = keys_to_flush(self.lengths, 0.1, maxcount=20)
            self.flush(keys)

    def _get(self, keys, lock=True, **kwargs):
        if lock: self.lock.acquire()
        try:
            result = list(map(add, self.fast.get(keys, lock=False),
                                   self.slow.get(keys, lock=False)))
        finally:
            if lock: self.lock.release()
        return result

    def _iset(self, key, value, lock=True):
        """ Idempotent set """
        if lock: self.lock.acquire()
        try:
            self.fast.iset(key, value, lock=False)
        finally:
            if lock: self.lock.release()

    def _delete(self, keys, lock=True):
        if lock: self.lock.acquire()
        try:
            self.fast.delete(keys, lock=False)
            self.slow.delete(keys, lock=False)
        finally:
            if lock: self.lock.release()

    def drop(self):
        self._iset_seen.clear()
        self.fast.drop()
        self.slow.drop()

    def __exit__(self, *args):
        self.drop()

    def flush(self, keys=None, block=None):
        """ Flush keys to disk

        Parameters
        ----------

        keys: list or None
            list of keys to flush
        block: bool (defaults to None)
            Whether or not to block until all writing is complete

        If no keys are given then flush all keys
        """
        if keys is None:
            keys = list(self.lengths)

        self.slow.append(dict(zip(keys, self.fast.get(keys))))
        self.fast.delete(keys)

        for key in keys:
            self.memory_usage -= self.lengths[key]
            del self.lengths[key]

def keys_to_flush(lengths, fraction=0.1, maxcount=100000):
    """ Which keys to remove

    >>> lengths = {'a': 20, 'b': 10, 'c': 15, 'd': 15,
    ...            'e': 10, 'f': 25, 'g': 5}
    >>> keys_to_flush(lengths, 0.5)
    ['f', 'a']
    """
    top = topk(max(len(lengths) // 2, 1),
               lengths.items(),
               key=1)
    total = sum(lengths.values())
    cutoff = min(maxcount, max(1,
                   bisect(list(accumulate(add, pluck(1, top))),
                          total * fraction)))
    result = [k for k, v in top[:cutoff]]
    assert result
    return result
<commit_msg>FIX: memory_usage not propagated through serialization.<commit_after>from .core import Interface
from threading import Lock
from toolz import merge_with, topk, accumulate, pluck
from operator import add
from bisect import bisect
from collections import defaultdict
from .compatibility import Queue, Empty

def zero():
    return 0

class Buffer(Interface):
    def __init__(self, fast, slow, available_memory=1e9):
        self.lock = Lock()
        self.fast = fast
        self.slow = slow
        self.available_memory = available_memory
        self.lengths = defaultdict(zero)
        self.memory_usage = 0
        Interface.__init__(self)

    def __getstate__(self):
        return {'fast': self.fast,
                'slow': self.slow,
                'memory_usage': self.memory_usage,
                'lengths': self.lengths,
                'available_memory': self.available_memory}

    def __setstate__(self, state):
        Interface.__setstate__(self, state)
        self.lock = Lock()
        self.__dict__.update(state)

    def append(self, data, lock=True, **kwargs):
        if lock: self.lock.acquire()
        try:
            for k, v in data.items():
                self.lengths[k] += len(v)
                self.memory_usage += len(v)
            self.fast.append(data, lock=False, **kwargs)
        finally:
            if lock: self.lock.release()

        while self.memory_usage > self.available_memory:
            keys = keys_to_flush(self.lengths, 0.1, maxcount=20)
            self.flush(keys)

    def _get(self, keys, lock=True, **kwargs):
        if lock: self.lock.acquire()
        try:
            result = list(map(add, self.fast.get(keys, lock=False),
                                   self.slow.get(keys, lock=False)))
        finally:
            if lock: self.lock.release()
        return result

    def _iset(self, key, value, lock=True):
        """ Idempotent set """
        if lock: self.lock.acquire()
        try:
            self.fast.iset(key, value, lock=False)
        finally:
            if lock: self.lock.release()

    def _delete(self, keys, lock=True):
        if lock: self.lock.acquire()
        try:
            self.fast.delete(keys, lock=False)
            self.slow.delete(keys, lock=False)
        finally:
            if lock: self.lock.release()

    def drop(self):
        self._iset_seen.clear()
        self.fast.drop()
        self.slow.drop()

    def __exit__(self, *args):
        self.drop()

    def flush(self, keys=None, block=None):
        """ Flush keys to disk

        Parameters
        ----------

        keys: list or None
            list of keys to flush
        block: bool (defaults to None)
            Whether or not to block until all writing is complete

        If no keys are given then flush all keys
        """
        if keys is None:
            keys = list(self.lengths)

        self.slow.append(dict(zip(keys, self.fast.get(keys))))
        self.fast.delete(keys)

        for key in keys:
            self.memory_usage -= self.lengths[key]
            del self.lengths[key]

def keys_to_flush(lengths, fraction=0.1, maxcount=100000):
    """ Which keys to remove

    >>> lengths = {'a': 20, 'b': 10, 'c': 15, 'd': 15,
    ...            'e': 10, 'f': 25, 'g': 5}
    >>> keys_to_flush(lengths, 0.5)
    ['f', 'a']
    """
    top = topk(max(len(lengths) // 2, 1),
               lengths.items(),
               key=1)
    total = sum(lengths.values())
    cutoff = min(maxcount, max(1,
                   bisect(list(accumulate(add, pluck(1, top))),
                          total * fraction)))
    result = [k for k, v in top[:cutoff]]
    assert result
    return result
<|endoftext|>


<commit_before>from django.contrib import admin

from models import Poll, Choice

class ChoiceInline(admin.TabularInline):
    model = Choice
    extra = 1

class PollAdmin(admin.ModelAdmin):
    model = Poll
    inlines = (ChoiceInline,)
    list_display = ('question', 'count_choices', 'count_total_votes')

admin.site.register(Poll, PollAdmin)
<commit_msg>register votes at admin and define list display<commit_after>from django.contrib import admin

from models import Poll, Choice, Vote

class ChoiceInline(admin.TabularInline):
    model = Choice
    extra = 1

class PollAdmin(admin.ModelAdmin):
    model = Poll
    inlines = (ChoiceInline,)
    list_display = ('question', 'count_choices', 'count_total_votes')

class VoteAdmin(admin.ModelAdmin):
    model = Vote
    list_display = ('choice', 'user', 'poll')

admin.site.register(Poll, PollAdmin)
admin.site.register(Vote, VoteAdmin)
<|endoftext|>