<commit_before>import sys
from functools import wraps

if sys.version_info < (3, 0):
    # noinspection PyCompatibility
    from ConfigParser import ConfigParser, NoOptionError, NoSectionError
else:
    # noinspection PyUnresolvedReferences
    from configparser import ConfigParser, NoOptionError, NoSectionError
from baron import parse, dumps
from tri.declarative import evaluate, dispatch, Namespace

__version__ = '0.0.11'

ALL = 'all'

# decorator
def config_from_setup_cfg(**defaults):
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            config_parser = ConfigParser()
            config_parser.read('setup.cfg')

            def s(key, default):
                try:
                    return config_parser.get('mutmut', key)
                except (NoOptionError, NoSectionError):
                    return default

            for k in list(kwargs.keys()):
                if not kwargs[k]:
                    kwargs[k] = s(k, defaults.get(k))
            f(*args, **kwargs)

        return wrapper
    return decorator

if sys.version_info < (3, 0):
    text_types = (str, unicode)
else:
    text_types = (str,)

def int_mutation(value, **_):
    suffix = ''
    if value.upper().endswith('L'):
        value = value[:-1]
        suffix = 'L'

    if value.startswith('0o'):
        base = 8
        value = value[2:]
    elif value.startswith('0x'):
        base = 16
        value = value[2:]
    elif value.startswith('0b'):
        base = 2
        value = value[2:]
    elif value.startswith('0') and len(value) > 1:
        base = 8
        value = value[1:]
    else:
        base = 10

    result = repr(int(value, base=base) + 1)
    if not result.endswith(suffix):
        result += suffix
    return result

def comparison_mutation(first, **_):
    return {
        '<': '<=',
        '<=': '<',
        '>': '>=',
        '>=': '>',
        '==': '!=',
        '!=': '==',
        '<>': '==',
        'in': 'not in',
        'not': '',
        'is': 'is not',  # this will cause "is not not" sometimes, so there's a hack to fix that later
    }[first]

def float_exponant_mutation(value, **_):
    a, b = value.lower().split('e')
    return '%se%s' % (a, (int(b) if b else 0) + 1)

def complex_mutation(value, **_):
    if '.' in value:
        return '%sj' % (float(value[:-1])+1)
    else:
        return '%sj' % (int(value[:-1])+1)

def string_mutation(value, context, **_):
    context.current_line += value.count('\n')  # Advance line count!
    if value.startswith('"""') or value.startswith("'''"):
        return value  # We assume here that triple-quoted stuff are docs or other things that mutation is meaningless for
    return value[0] + 'XX' + value[1:-1] + 'XX' + value[-1]

def call_argument_mutation(target, context, **_):
    if context.stack[-2]['type'] == 'call' and context.stack[-3]['value'][0]['type'] == 'name' and context.stack[-3]['value'][0]['value'] in context.dict_synonyms and 'value' in target:
        target = target.copy()
        target['value'] += 'XX'
        return target
    else:
        return target

def lambda_mutation(value, **_):
    if (value.get('type'), value.get('value')) == ('name', 'None'):
        return {'section': 'number', 'type': 'int', 'value': '0'}
    else:
        return {'type': 'name', 'value': 'None'}

NEWLINE = {'formatting': [], 'indent': '', 'type': 'endl', 'value': ''}

mutations_by_type = {
    'binary_operator': dict(
        value=lambda value, **_: {
            '+': '-',
            '-': '+',
            '*': '/',
            '/': '*',
            '//': '/',
            '%': '/',
            '<<': '>>',
            '>>': '<<',
            '&': '|',
            '|': '&',
            '^': '&',
            '**': '*',
        }[value],
    ),
    'unitary_operator': dict(
        value=lambda value, **_: {
            'not': 'not not',
            '-': '+',
            '+': '-',
            '~': '',
        }[value],
    ),
    'int': dict(value=int_mutation),
    'long': dict(value=int_mutation),
    'octa': dict(value=int_mutation),
    'hexa': dict(value=int_mutation),
    'binary': dict(value=int_mutation),
    # 'float': dict(value=lambda value, **_: repr(numpy.nextafter(float(value), float(value) + 1000.0))),  # this might be a bit brutal :P
    'float': dict(value=lambda value, **_: repr(float(value) + 100.0)),
    'float_exponant': dict(value=float_exponant_mutation),
    'string': dict(value=string_mutation),
    'unicode_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'binary_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'raw_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'unicode_raw_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'binary_raw_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'complex': dict(value=complex_mutation),
    'continue': dict(type='break'),
    'break': dict(type='continue'),
    'name': dict(
        value=lambda value, **_: {
            'True': 'False',
            'False': 'True',
            'deepcopy': 'copy',
            # TODO: probably need to add a lot of things here... None, some builtins maybe, what more?
        }.get(value, value)),
    'comparison_operator': dict(
        first=comparison_mutation,
    ),
    'boolean_operator': dict(
        value=lambda value, **_: {
            'or': 'and',
            'and': 'or',
        }[value],
    ),
    'decorator': dict(replace_entire_node_with=NEWLINE),
    'call_argument': dict(target=call_argument_mutation),
    'lambda': dict(value=lambda_mutation),

    # Don't mutate:
    'tuple': {},
    'list': {},
    'dict': {},
    'set': {},
    'comma': {},
    'from_import': {},
    'import': {},
    'assignment': {},
    'ifelseblock': {},
    'if': {},
    'elif': {},
    'else': {},
    'atomtrailers': {},  # http://redbaron.readthedocs.io/en/latest/nodes_reference.html#atomtrailersnode
    'dict_comprehension': {},
    'list_comprehension': {},
    'set_comprehension': {},
    'generator_comprehension': {},
    'comprehension_loop': {},
    'comprehension_if': {},
    'dictitem': {},
    'for': {},
    'try': {},
    'finally': {},
    'while': {},
    'class': {},
    'comment': {},
    'del': {},
    'assert': {},
    'raise': {},
    'dotted_name': {},
    'global': {},
    'print': {},
    'ternary_operator': {},
    'call': {},
    'def_argument': {},
    'dict_argument': {},
    'with': {},
    'with_context_item': {},
    'associative_parenthesis': {},
    'left_parenthesis': {},
    'right_parenthesis': {},
    'pass': {},
    'semicolon': {},
    'string_chain': {},
    'exec': {},
    'endl': {},
    'def': {},
    'getitem': {},
    'slice': {},
    'dot': {},
    'list_argument': {},
    'ellipsis': {},
    'argument_generator_comprehension': {},
    'float_exponant_complex': {},  # TODO
    'yield_atom': {},
    'space': {},
    'name_as_name': {},
    'dotted_as_name': {},
    'comparison': {},
    'except': {},
    'star': {},
    'return': {},  # TODO: we should mutate "return foo" -> "return None"
    'yield': {},  # TODO: we should mutate "yield foo" -> "yield None"
}

# TODO: detect regexes and mutate them in nasty ways? Maybe mutate all strings as if they are regexes

class Context(object):
    def __init__(self, mutate_index, dict_synonyms=None, filename=None, exclude=lambda context: False):
        self.index = 0
        self.performed_mutations = 0
        self.mutate_index = mutate_index
        self.current_line = 1
        self.pragma_no_mutate_lines = set()
        self.filename = filename
        self.exclude = exclude
        self.stack = []
        self.dict_synonyms = (dict_synonyms or []) + ['dict']

    def exclude_line(self):
        return self.current_line in self.pragma_no_mutate_lines or self.exclude(context=self)

@dispatch(
    context=Namespace(),
)
def mutate(source, mutate_index, context):
    """
    :param source: source code
    :param mutate_index: the index of the mutation to be performed, if ALL mutates all available places
    :return: tuple: mutated source code, number of mutations performed
    """
    result = parse(source)
    context = Context(mutate_index=mutate_index, **context)
    context.pragma_no_mutate_lines = {i+1 for i, line in enumerate(source.split('\n')) if '# pragma: no mutate' in line}  # lines are 1 based indexed
    mutate_list_of_nodes(result, context=context)
    result_source = dumps(result).replace(' not not ', ' ')
    if context.performed_mutations:
        assert source != result_source
    return result_source, context.performed_mutations

def mutate_node(i, context):
    if not i:
        return

    context.stack.append(i)
    try:

        t = i['type']

        if t == 'endl':
            context.current_line += 1

        assert t in mutations_by_type, (t, i.keys(), dumps(i))
        m = mutations_by_type[t]

        if 'replace_entire_node_with' in m:
            if context.exclude_line():
                return

            if context.mutate_index in (ALL, context.index):
                i.clear()
                for k, v in m['replace_entire_node_with'].items():
                    i[k] = v
                context.performed_mutations += 1
            context.index += 1
            return

        for _, x in sorted(i.items()):
            if x is None:
                continue  # pragma: no cover

            if isinstance(x, list):
                if x:
                    mutate_list_of_nodes(x, context=context)
            elif isinstance(x, dict):
                mutate_node(x, context=context)
            else:
                assert isinstance(x, text_types + (bool,))

            if context.performed_mutations and context.mutate_index != ALL:
                return

        for key, value in sorted(m.items()):
            old = i[key]
            if context.exclude_line():
                continue

            new = evaluate(value, context=context, node=i, **i)
            assert not callable(new)
            if new != old:
                if context.mutate_index in (ALL, context.index):
                    context.performed_mutations += 1
                    i[key] = new
                context.index += 1

            if context.performed_mutations and context.mutate_index != ALL:
                return
    finally:
        context.stack.pop()

def mutate_list_of_nodes(result, context):
    for i in result:
        mutate_node(i, context=context)

        if context.performed_mutations and context.mutate_index != ALL:
            return

@dispatch(
    context=Namespace(),
)
def count_mutations(source, context):
    return mutate(source, ALL, context=context)[1]

@dispatch(
    context=Namespace(),
)
def mutate_file(backup, mutation, filename, context):  # pragma: no cover
    code = open(filename).read()
    if backup:
        open(filename + '.bak', 'w').write(code)
    context.filename = filename
    result, mutations_performed = mutate(code, mutation, context=context)
    open(filename, 'w').write(result)
    return mutations_performed
<commit_msg>Release<commit_after>import sys
from functools import wraps

if sys.version_info < (3, 0):
    # noinspection PyCompatibility
    from ConfigParser import ConfigParser, NoOptionError, NoSectionError
else:
    # noinspection PyUnresolvedReferences
    from configparser import ConfigParser, NoOptionError, NoSectionError
from baron import parse, dumps
from tri.declarative import evaluate, dispatch, Namespace

__version__ = '0.0.12'

ALL = 'all'

# decorator
def config_from_setup_cfg(**defaults):
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            config_parser = ConfigParser()
            config_parser.read('setup.cfg')

            def s(key, default):
                try:
                    return config_parser.get('mutmut', key)
                except (NoOptionError, NoSectionError):
                    return default

            for k in list(kwargs.keys()):
                if not kwargs[k]:
                    kwargs[k] = s(k, defaults.get(k))
            f(*args, **kwargs)

        return wrapper
    return decorator

if sys.version_info < (3, 0):
    text_types = (str, unicode)
else:
    text_types = (str,)

def int_mutation(value, **_):
    suffix = ''
    if value.upper().endswith('L'):
        value = value[:-1]
        suffix = 'L'

    if value.startswith('0o'):
        base = 8
        value = value[2:]
    elif value.startswith('0x'):
        base = 16
        value = value[2:]
    elif value.startswith('0b'):
        base = 2
        value = value[2:]
    elif value.startswith('0') and len(value) > 1:
        base = 8
        value = value[1:]
    else:
        base = 10

    result = repr(int(value, base=base) + 1)
    if not result.endswith(suffix):
        result += suffix
    return result

def comparison_mutation(first, **_):
    return {
        '<': '<=',
        '<=': '<',
        '>': '>=',
        '>=': '>',
        '==': '!=',
        '!=': '==',
        '<>': '==',
        'in': 'not in',
        'not': '',
        'is': 'is not',  # this will cause "is not not" sometimes, so there's a hack to fix that later
    }[first]

def float_exponant_mutation(value, **_):
    a, b = value.lower().split('e')
    return '%se%s' % (a, (int(b) if b else 0) + 1)

def complex_mutation(value, **_):
    if '.' in value:
        return '%sj' % (float(value[:-1])+1)
    else:
        return '%sj' % (int(value[:-1])+1)

def string_mutation(value, context, **_):
    context.current_line += value.count('\n')  # Advance line count!
    if value.startswith('"""') or value.startswith("'''"):
        return value  # We assume here that triple-quoted stuff are docs or other things that mutation is meaningless for
    return value[0] + 'XX' + value[1:-1] + 'XX' + value[-1]

def call_argument_mutation(target, context, **_):
    if context.stack[-2]['type'] == 'call' and context.stack[-3]['value'][0]['type'] == 'name' and context.stack[-3]['value'][0]['value'] in context.dict_synonyms and 'value' in target:
        target = target.copy()
        target['value'] += 'XX'
        return target
    else:
        return target

def lambda_mutation(value, **_):
    if (value.get('type'), value.get('value')) == ('name', 'None'):
        return {'section': 'number', 'type': 'int', 'value': '0'}
    else:
        return {'type': 'name', 'value': 'None'}

NEWLINE = {'formatting': [], 'indent': '', 'type': 'endl', 'value': ''}

mutations_by_type = {
    'binary_operator': dict(
        value=lambda value, **_: {
            '+': '-',
            '-': '+',
            '*': '/',
            '/': '*',
            '//': '/',
            '%': '/',
            '<<': '>>',
            '>>': '<<',
            '&': '|',
            '|': '&',
            '^': '&',
            '**': '*',
        }[value],
    ),
    'unitary_operator': dict(
        value=lambda value, **_: {
            'not': 'not not',
            '-': '+',
            '+': '-',
            '~': '',
        }[value],
    ),
    'int': dict(value=int_mutation),
    'long': dict(value=int_mutation),
    'octa': dict(value=int_mutation),
    'hexa': dict(value=int_mutation),
    'binary': dict(value=int_mutation),
    # 'float': dict(value=lambda value, **_: repr(numpy.nextafter(float(value), float(value) + 1000.0))),  # this might be a bit brutal :P
    'float': dict(value=lambda value, **_: repr(float(value) + 100.0)),
    'float_exponant': dict(value=float_exponant_mutation),
    'string': dict(value=string_mutation),
    'unicode_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'binary_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'raw_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'unicode_raw_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'binary_raw_string': dict(value=lambda value, **_: value[0:2] + 'XX' + value[2:-1] + 'XX' + value[-1]),
    'complex': dict(value=complex_mutation),
    'continue': dict(type='break'),
    'break': dict(type='continue'),
    'name': dict(
        value=lambda value, **_: {
            'True': 'False',
            'False': 'True',
            'deepcopy': 'copy',
            # TODO: probably need to add a lot of things here... None, some builtins maybe, what more?
        }.get(value, value)),
    'comparison_operator': dict(
        first=comparison_mutation,
    ),
    'boolean_operator': dict(
        value=lambda value, **_: {
            'or': 'and',
            'and': 'or',
        }[value],
    ),
    'decorator': dict(replace_entire_node_with=NEWLINE),
    'call_argument': dict(target=call_argument_mutation),
    'lambda': dict(value=lambda_mutation),

    # Don't mutate:
    'tuple': {},
    'list': {},
    'dict': {},
    'set': {},
    'comma': {},
    'from_import': {},
    'import': {},
    'assignment': {},
    'ifelseblock': {},
    'if': {},
    'elif': {},
    'else': {},
    'atomtrailers': {},  # http://redbaron.readthedocs.io/en/latest/nodes_reference.html#atomtrailersnode
    'dict_comprehension': {},
    'list_comprehension': {},
    'set_comprehension': {},
    'generator_comprehension': {},
    'comprehension_loop': {},
    'comprehension_if': {},
    'dictitem': {},
    'for': {},
    'try': {},
    'finally': {},
    'while': {},
    'class': {},
    'comment': {},
    'del': {},
    'assert': {},
    'raise': {},
    'dotted_name': {},
    'global': {},
    'print': {},
    'ternary_operator': {},
    'call': {},
    'def_argument': {},
    'dict_argument': {},
    'with': {},
    'with_context_item': {},
    'associative_parenthesis': {},
    'left_parenthesis': {},
    'right_parenthesis': {},
    'pass': {},
    'semicolon': {},
    'string_chain': {},
    'exec': {},
    'endl': {},
    'def': {},
    'getitem': {},
    'slice': {},
    'dot': {},
    'list_argument': {},
    'ellipsis': {},
    'argument_generator_comprehension': {},
    'float_exponant_complex': {},  # TODO
    'yield_atom': {},
    'space': {},
    'name_as_name': {},
    'dotted_as_name': {},
    'comparison': {},
    'except': {},
    'star': {},
    'return': {},  # TODO: we should mutate "return foo" -> "return None"
    'yield': {},  # TODO: we should mutate "yield foo" -> "yield None"
}

# TODO: detect regexes and mutate them in nasty ways? Maybe mutate all strings as if they are regexes

class Context(object):
    def __init__(self, mutate_index, dict_synonyms=None, filename=None, exclude=lambda context: False):
        self.index = 0
        self.performed_mutations = 0
        self.mutate_index = mutate_index
        self.current_line = 1
        self.pragma_no_mutate_lines = set()
        self.filename = filename
        self.exclude = exclude
        self.stack = []
        self.dict_synonyms = (dict_synonyms or []) + ['dict']

    def exclude_line(self):
        return self.current_line in self.pragma_no_mutate_lines or self.exclude(context=self)

@dispatch(
    context=Namespace(),
)
def mutate(source, mutate_index, context):
    """
    :param source: source code
    :param mutate_index: the index of the mutation to be performed, if ALL mutates all available places
    :return: tuple: mutated source code, number of mutations performed
    """
    result = parse(source)
    context = Context(mutate_index=mutate_index, **context)
    context.pragma_no_mutate_lines = {i+1 for i, line in enumerate(source.split('\n')) if '# pragma: no mutate' in line}  # lines are 1 based indexed
    mutate_list_of_nodes(result, context=context)
    result_source = dumps(result).replace(' not not ', ' ')
    if context.performed_mutations:
        assert source != result_source
    return result_source, context.performed_mutations

def mutate_node(i, context):
    if not i:
        return

    context.stack.append(i)
    try:

        t = i['type']

        if t == 'endl':
            context.current_line += 1

        assert t in mutations_by_type, (t, i.keys(), dumps(i))
        m = mutations_by_type[t]

        if 'replace_entire_node_with' in m:
            if context.exclude_line():
                return

            if context.mutate_index in (ALL, context.index):
                i.clear()
                for k, v in m['replace_entire_node_with'].items():
                    i[k] = v
                context.performed_mutations += 1
            context.index += 1
            return

        for _, x in sorted(i.items()):
            if x is None:
                continue  # pragma: no cover

            if isinstance(x, list):
                if x:
                    mutate_list_of_nodes(x, context=context)
            elif isinstance(x, dict):
                mutate_node(x, context=context)
            else:
                assert isinstance(x, text_types + (bool,))

            if context.performed_mutations and context.mutate_index != ALL:
                return

        for key, value in sorted(m.items()):
            old = i[key]
            if context.exclude_line():
                continue

            new = evaluate(value, context=context, node=i, **i)
            assert not callable(new)
            if new != old:
                if context.mutate_index in (ALL, context.index):
                    context.performed_mutations += 1
                    i[key] = new
                context.index += 1

            if context.performed_mutations and context.mutate_index != ALL:
                return
    finally:
        context.stack.pop()

def mutate_list_of_nodes(result, context):
    for i in result:
        mutate_node(i, context=context)

        if context.performed_mutations and context.mutate_index != ALL:
            return

@dispatch(
    context=Namespace(),
)
def count_mutations(source, context):
    return mutate(source, ALL, context=context)[1]

@dispatch(
    context=Namespace(),
)
def mutate_file(backup, mutation, filename, context):  # pragma: no cover
    code = open(filename).read()
    if backup:
        open(filename + '.bak', 'w').write(code)
    context.filename = filename
    result, mutations_performed = mutate(code, mutation, context=context)
    open(filename, 'w').write(result)
    return mutations_performed
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import contextlib
import errno
import glob
import json
import multiprocessing
import os
import subprocess
import urllib2

class Vex (object):
    VERSIONS = {
        'erlang': [
            ('17.3',   'https://github.com/erlang/otp/archive/OTP-17.3.tar.gz'),
            ('17.3.1', 'https://github.com/erlang/otp/archive/OTP-17.3.1.tar.gz'),
            ('17.3.2', 'https://github.com/erlang/otp/archive/OTP-17.3.2.tar.gz'),
            ('17.3.3', 'https://github.com/erlang/otp/archive/OTP-17.3.3.tar.gz'),
            ('17.3.4', 'https://github.com/erlang/otp/archive/OTP-17.3.4.tar.gz'),
            ('17.4',   'https://github.com/erlang/otp/archive/OTP-17.4.tar.gz'),
            ('17.4.1', 'https://github.com/erlang/otp/archive/OTP-17.4.1.tar.gz'),
            ('17.5',   'https://github.com/erlang/otp/archive/OTP-17.5.tar.gz'),
            ('17.5.1', 'https://github.com/erlang/otp/archive/OTP-17.5.1.tar.gz'),
            ('17.5.2', 'https://github.com/erlang/otp/archive/OTP-17.5.2.tar.gz'),
        ],

        'elixir': [
            ('1.0.0', 'https://github.com/elixir-lang/elixir/archive/v1.0.0.tar.gz'),
            ('1.0.1', 'https://github.com/elixir-lang/elixir/archive/v1.0.1.tar.gz'),
            ('1.0.2', 'https://github.com/elixir-lang/elixir/archive/v1.0.2.tar.gz'),
            ('1.0.3', 'https://github.com/elixir-lang/elixir/archive/v1.0.3.tar.gz'),
            ('1.0.4', 'https://github.com/elixir-lang/elixir/archive/v1.0.4.tar.gz'),
        ]
    }

    LATEST = 'latest'

    HOME = os.path.join(os.path.expanduser('~'), '.vex')

    def __init__(self):
        self.make_vexdir()
        self.dnlds = os.path.join(self.HOME, 'downloads')
        self.build = os.path.join(self.HOME, 'build')
        self.bin = os.path.join(self.HOME, 'bin')
        self.otpver, self.exver = self.read_version_file()

    def set_executable_links(self, otpver, exver):
        with self.change_directory(self.bin):
            self.symlink_executables(os.path.join(self.build, otpver, 'local', 'bin'))
            self.symlink_executables(os.path.join(self.build, otpver, exver, 'local', 'bin'))
        self.write_version_file(otpver, exver)

    ## ERLANG METHODS

    def download_erlang(self, version=LATEST):
        outfile, url = self.interpret_version('erlang', version)
        if os.path.isfile(outfile):
            return outfile
        tarfile = self.download(outfile, url)
        return tarfile

    def build_erlang(self, version, jfactor):
        tarfile = os.path.join(self.dnlds, 'otp-OTP-' + version + '.tar')
        outdir = os.path.join(self.build, version)
        try:
            os.mkdir(outdir)
            os.mkdir(os.path.join(outdir, 'local'))
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

        # return if Erlang has already been installed
        if glob.glob(os.path.join(outdir, 'local', '*')):
            return

        subprocess.check_call(['tar', 'xzvf', tarfile, '--directory', outdir])
        build_dir = os.path.join(outdir, 'otp-OTP-' + version)
        prefix_path = os.path.join(outdir, 'local')
        with self.change_directory(build_dir):
            os.environ['ERL_TOP'] = build_dir
            subprocess.check_call(['./otp_build', 'autoconf'])
            subprocess.check_call(['./configure', '--prefix=%s' % prefix_path])
            subprocess.check_call(['gmake', '-j%d' % jfactor])
            subprocess.check_call(['gmake', 'install'])

    ## ELIXIR METHODS

    def download_elixir(self, version=LATEST):
        outfile, url = self.interpret_version('elixir', version)
        if os.path.isfile(outfile):
            return outfile
        tarfile = self.download(outfile, url)
        return tarfile

    def build_elixir(self, version, otp_version, jfactor):
        self.ensure_erlang(otp_version, jfactor)
        tarfile = os.path.join(self.dnlds, 'elixir-' + version + '.tar')
        outdir = os.path.join(self.build, otp_version, version)
        try:
            os.mkdir(outdir)
            os.mkdir(os.path.join(outdir, 'local'))
            os.mkdir(os.path.join(outdir, 'local', 'bin'))
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

        # return if Elixir has already been installed
        if glob.glob(os.path.join(outdir, 'local', 'bin', '*')):
            return

        subprocess.check_call(['tar', 'xzvf', tarfile, '--directory', outdir])
        build_dir = os.path.join(outdir, 'elixir-' + version)
        otp_path = os.path.join(self.build, otp_version, 'local', 'bin')
        with self.change_directory(build_dir):
            os.environ['PATH'] = otp_path + ':' + os.environ['PATH']
            subprocess.check_call(['gmake', '-j%d' % jfactor])

        with self.change_directory(os.path.join(outdir, 'local', 'bin')):
            for tool in ('elixir', 'elixirc', 'iex', 'mix'):
                os.symlink(os.path.join(build_dir, 'bin', tool), tool)

    ## PRIVATE

    def symlink_executables(self, directory):
        for exe in glob.glob(os.path.join(directory, '*')):
            name = os.path.basename(exe)
            if os.path.islink(name):
                os.remove(name)
            os.symlink(exe, name)

    def make_vexdir(self):
        try:
            os.mkdir(self.HOME)
            os.mkdir(os.path.join(self.HOME, 'downloads'))
            os.mkdir(os.path.join(self.HOME, 'build'))
            os.mkdir(os.path.join(self.HOME, 'bin'))

        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

    def read_version_file(self):
        version_file = os.path.join(self.HOME, 'version.json')
        try:
            with open(version_file, 'r') as f:
                result = json.load(f)
            return result['erlang'], result['elixir']

        except IOError:
            return None, None

    def write_version_file(self, erlang, elixir):
        version_file = os.path.join(self.HOME, 'version.json')
        with open(version_file, 'w') as f:
            json.dump({'erlang': erlang, 'elixir': elixir}, f)

    def interpret_version(self, tool, version):
        if version == self.LATEST:
            res = [self.VERSIONS[tool][-1]]
        else:
            res = list(filter(lambda x: x[0] == version, self.VERSIONS[tool]))
            if not res:
                raise BadVersion('%s %s is not valid' % (tool, version))

        version, url = res[0]
        if tool == 'erlang':
            tarfile = 'otp-OTP-' + version + '.tar'
        else:
            tarfile = 'elixir-' + version + '.tar'
        return os.path.join(self.dnlds, tarfile), url

    def download(self, outfile, url):
        response = urllib2.urlopen(url, cafile='/usr/local/share/certs/ca-root-nss.crt')
        with open(outfile, 'w') as f:
            f.write(response.read())
        return outfile

    @contextlib.contextmanager
    def change_directory(self, new_dir):
        prev = os.getcwd()
        os.chdir(new_dir)
        try:
            yield
        finally:
            os.chdir(prev)

    def ensure_erlang(self, version, jfactor):
        self.download_erlang(version)
        self.build_erlang(version, jfactor)

def main():
    Vex().download_erlang('17.5.1')
    Vex().download_elixir(Vex.LATEST)
    Vex().build_erlang('17.5.1', multiprocessing.cpu_count())
    Vex().build_elixir('1.0.4', '17.5.1', multiprocessing.cpu_count())
    Vex().set_executable_links('17.5.1', '1.0.4')

if __name__ == '__main__':
    main()
<commit_msg>supports command line options<commit_after>#!/usr/bin/env python

from __future__ import print_function

import contextlib
import errno
import glob
import json
import os
import subprocess
import urllib2

class Vex (object):
    VERSIONS = {
        'erlang': [
            ('17.3',   'https://github.com/erlang/otp/archive/OTP-17.3.tar.gz'),
            ('17.3.1', 'https://github.com/erlang/otp/archive/OTP-17.3.1.tar.gz'),
            ('17.3.2', 'https://github.com/erlang/otp/archive/OTP-17.3.2.tar.gz'),
            ('17.3.3', 'https://github.com/erlang/otp/archive/OTP-17.3.3.tar.gz'),
            ('17.3.4', 'https://github.com/erlang/otp/archive/OTP-17.3.4.tar.gz'),
            ('17.4',   'https://github.com/erlang/otp/archive/OTP-17.4.tar.gz'),
            ('17.4.1', 'https://github.com/erlang/otp/archive/OTP-17.4.1.tar.gz'),
            ('17.5',   'https://github.com/erlang/otp/archive/OTP-17.5.tar.gz'),
            ('17.5.1', 'https://github.com/erlang/otp/archive/OTP-17.5.1.tar.gz'),
            ('17.5.2', 'https://github.com/erlang/otp/archive/OTP-17.5.2.tar.gz'),
        ],

        'elixir': [
            ('1.0.0', 'https://github.com/elixir-lang/elixir/archive/v1.0.0.tar.gz'),
            ('1.0.1', 'https://github.com/elixir-lang/elixir/archive/v1.0.1.tar.gz'),
            ('1.0.2', 'https://github.com/elixir-lang/elixir/archive/v1.0.2.tar.gz'),
            ('1.0.3', 'https://github.com/elixir-lang/elixir/archive/v1.0.3.tar.gz'),
            ('1.0.4', 'https://github.com/elixir-lang/elixir/archive/v1.0.4.tar.gz'),
        ]
    }

    LATEST = 'latest'

    HOME = os.path.join(os.path.expanduser('~'), '.vex')

    def __init__(self):
        self.make_vexdir()
        self.dnlds = os.path.join(self.HOME, 'downloads')
        self.build = os.path.join(self.HOME, 'build')
        self.bin = os.path.join(self.HOME, 'bin')
        self.otpver, self.exver = self.read_version_file()

    def set_executable_links(self, otpver, exver):
        with self.change_directory(self.bin):
            self.symlink_executables(os.path.join(self.build, otpver, 'local', 'bin'))
            self.symlink_executables(os.path.join(self.build, otpver, exver, 'local', 'bin'))
        self.write_version_file(otpver, exver)

    def list_versions(self):
        print(' \tERLANG\t|\tELIXIR')
        print(' \t------\t|\t------')
        otpvers = set(map(lambda x: x[0], self.VERSIONS['erlang']))
        exvers = set(map(lambda x: x[0], self.VERSIONS['elixir']))
        for otp_path in sorted(glob.glob(os.path.join(self.build, '*'))):
            otpver = os.path.basename(otp_path)
            if otpver in otpvers:
                for elixir in sorted(glob.glob(os.path.join(otp_path, '*'))):
                    exver = os.path.basename(elixir)
                    if exver in exvers:
                        if exver == self.exver and otpver == self.otpver:
                            flag = '*'
                        else:
                            flag = ' '
                        print('%s\t%s\t|\t%s' % (flag, otpver, exver))

    ## ERLANG METHODS

    def download_erlang(self, version=LATEST):
        outfile, url = self.interpret_version('erlang', version)
        if os.path.isfile(outfile):
            return outfile
        tarfile = self.download(outfile, url)
        return tarfile

    def build_erlang(self, version, jfactor):
        if version == self.LATEST:
            version = self.VERSIONS['erlang'][-1][0]
        tarfile = os.path.join(self.dnlds, 'otp-OTP-' + version + '.tar')
        outdir = os.path.join(self.build, version)
        try:
            os.mkdir(outdir)
            os.mkdir(os.path.join(outdir, 'local'))
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

        # return if Erlang has already been installed
        if glob.glob(os.path.join(outdir, 'local', '*')):
            return

        subprocess.check_call(['tar', 'xzvf', tarfile, '--directory', outdir])
        build_dir = os.path.join(outdir, 'otp-OTP-' + version)
        prefix_path = os.path.join(outdir, 'local')
        with self.change_directory(build_dir):
            os.environ['ERL_TOP'] = build_dir
            subprocess.check_call(['./otp_build', 'autoconf'])
            subprocess.check_call(['./configure', '--prefix=%s' % prefix_path])
            subprocess.check_call(['gmake', '-j%d' % jfactor])
            subprocess.check_call(['gmake', 'install'])

    ## ELIXIR METHODS

    def download_elixir(self, version=LATEST):
        outfile, url = self.interpret_version('elixir', version)
        if os.path.isfile(outfile):
            return outfile
        tarfile = self.download(outfile, url)
        return tarfile

    def build_elixir(self, version, otp_version, jfactor):
        if version == self.LATEST:
            version = self.VERSIONS['elixir'][-1][0]
        if otp_version == self.LATEST:
            otp_version = self.VERSIONS['erlang'][-1][0]

        self.ensure_erlang(otp_version, jfactor)
        tarfile = os.path.join(self.dnlds, 'elixir-' + version + '.tar')
        outdir = os.path.join(self.build, otp_version, version)
        try:
            os.mkdir(outdir)
            os.mkdir(os.path.join(outdir, 'local'))
            os.mkdir(os.path.join(outdir, 'local', 'bin'))
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

        # return if Elixir has already been installed
        if glob.glob(os.path.join(outdir, 'local', 'bin', '*')):
            return

        subprocess.check_call(['tar', 'xzvf', tarfile, '--directory', outdir])
        build_dir = os.path.join(outdir, 'elixir-' + version)
        otp_path = os.path.join(self.build, otp_version, 'local', 'bin')
        with self.change_directory(build_dir):
            os.environ['PATH'] = otp_path + ':' + os.environ['PATH']
            subprocess.check_call(['gmake', '-j%d' % jfactor])

        with self.change_directory(os.path.join(outdir, 'local', 'bin')):
            for tool in ('elixir', 'elixirc', 'iex', 'mix'):
                os.symlink(os.path.join(build_dir, 'bin', tool), tool)

    ## PRIVATE

    def symlink_executables(self, directory):
        for exe in glob.glob(os.path.join(directory, '*')):
            name = os.path.basename(exe)
            if os.path.islink(name):
                os.remove(name)
            os.symlink(exe, name)

    def make_vexdir(self):
        try:
            os.mkdir(self.HOME)
            os.mkdir(os.path.join(self.HOME, 'downloads'))
            os.mkdir(os.path.join(self.HOME, 'build'))
            os.mkdir(os.path.join(self.HOME, 'bin'))

        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

    def read_version_file(self):
        version_file = os.path.join(self.HOME, 'version.json')
        try:
            with open(version_file, 'r') as f:
                result = json.load(f)
            return result['erlang'], result['elixir']

        except IOError:
            return None, None

    def write_version_file(self, erlang, elixir):
        version_file = os.path.join(self.HOME, 'version.json')
        with open(version_file, 'w') as f:
            json.dump({'erlang': erlang, 'elixir': elixir}, f)

    def interpret_version(self, tool, version):
        if version == self.LATEST:
            res = [self.VERSIONS[tool][-1]]
        else:
            res = list(filter(lambda x: x[0] == version, self.VERSIONS[tool]))
            if not res:
                raise BadVersion('%s %s is not valid' % (tool, version))

        version, url = res[0]
        if tool == 'erlang':
            tarfile = 'otp-OTP-' + version + '.tar'
        else:
            tarfile = 'elixir-' + version + '.tar'
        return os.path.join(self.dnlds, tarfile), url

    def download(self, outfile, url):
        response = urllib2.urlopen(url, cafile='/usr/local/share/certs/ca-root-nss.crt')
        with open(outfile, 'w') as f:
            f.write(response.read())
        return outfile

    @contextlib.contextmanager
    def change_directory(self, new_dir):
        prev = os.getcwd()
        os.chdir(new_dir)
        try:
            yield
        finally:
            os.chdir(prev)

    def ensure_erlang(self, version, jfactor):
        self.download_erlang(version)
        self.build_erlang(version, jfactor)

def options():
    usage = '%(prog)s [help | list | install | use] <arguments>'
    parser = argparse.ArgumentParser(usage=usage)
    parser.add_argument('--otp-version', default='latest', help='Erlang version')
    parser.add_argument('--elixir-version', default='latest', help='Elixir version')
    parser.add_argument('--jobs', default=multiprocessing.cpu_count(), type=int, help='Number of build jobs to run in parallel')
    return parser

def usage(parser):
    parser.print_help()

def list_versions():
    Vex().list_versions()

def install(otp_version, elixir_version, jobs):
    vex = Vex()
    vex.download_erlang(otp_version)
    vex.download_elixir(elixir_version)
    vex.build_elixir(elixir_version, otp_version, jobs)

def use(otp_version, elixir_version):
    vex = Vex()
    vex.set_executable_links(otp_version, elixir_version)

def main(command, argv):
    parser = options()
    args = parser.parse_args(argv[1:])

    if command == 'help':
        usage(parser)
    elif command == 'list':
        list_versions()
    elif command == 'install':
        install(args.otp_version, args.elixir_version, args.jobs)
    elif command == 'use':
        use(args.otp_version, args.elixir_version)
    else:
        print('%s: error: unknown command', sys.argv[0])
        usage()
        sys.exit(1)

if __name__ == '__main__':
    import argparse
    import multiprocessing
    import sys

    if len(sys.argv) < 2:
        print('%s: error: must use one of the following commands' % sys.argv[0])
        print('    help\n'
              '    list\n'
              '    install\n'
              '    use')
        sys.exit(1)

    main(sys.argv.pop(1), sys.argv)
<|endoftext|>


<commit_before>#!flask/bin/python
from flask import abort, Flask, jsonify
import happybase
import json
import time

API_VERSION = 'v0.2'

app = Flask(__name__)
connection = happybase.Connection('54.183.87.221')

@app.route('/api/' + API_VERSION + '/brand_metrics', methods = ['GET'])
def get_brand_metrics():
    results = []
    table = connection.table('isura_brand_metrics')
    for key, data in table.scan():
        entry = {}
        entry['brand'] = key
        entry['average_score'] = data['cf1:average_score']
        entry['review_count'] = data['cf1:review_count']
        results.append(entry)
    return jsonify( { 'brand_metrics' : results } )

@app.route('/api/' + API_VERSION + '/brand_metrics/<string:brand>', methods = ['GET'])
def get_brand_metrics_for(brand):
    table = connection.table('isura_brand_metrics')
    row = [r for r in table.scan() if r[0] == brand]
    if not row:
        abort(404)
    return jsonify( { 'brand_metrics' : row[0] } )

@app.route('/api/' + API_VERSION + '/reviews/<string:product>', methods = ['GET'])
def get_review_for(product):
    start = time.time()
    table = connection.table('isura_reviews')
    # Keys are stored in lowercase in HBase for more flexible searching.
    row = table.row(product.lower())
    reviews = []
    if not row:
        abort(404)
    for val in row.itervalues():
        reviews.append(json.loads(val))
    reviews = sorted(reviews, key=lambda k: k['timestamp'])
    reviews = { 'reviews' : reviews }
    reviews['meta'] = { 'count' : len(row), 'responseTime' : time.time() - start }
    return jsonify(reviews)

if __name__ == '__main__':
    app.run(debug = True)
<commit_msg>Implement simple keyword search on flask side talking to HBase<commit_after>#!flask/bin/python
from flask import abort, Flask, jsonify
import happybase
import json
import re
import time

API_VERSION = 'v0.3'

app = Flask(__name__)
connection = happybase.Connection('54.183.87.221')

@app.route('/api/' + API_VERSION + '/brand_metrics', methods = ['GET'])
def get_brand_metrics():
    results = []
    table = connection.table('isura_brand_metrics')
    for key, data in table.scan():
        entry = {}
        entry['brand'] = key
        entry['average_score'] = data['cf1:average_score']
        entry['review_count'] = data['cf1:review_count']
        results.append(entry)
    return jsonify( { 'brand_metrics' : results } )

@app.route('/api/' + API_VERSION + '/brand_metrics/<string:brand>', methods = ['GET'])
def get_brand_metrics_for(brand):
    table = connection.table('isura_brand_metrics')
    row = [r for r in table.scan() if r[0] == brand]
    if not row:
        abort(404)
    return jsonify( { 'brand_metrics' : row[0] } )

@app.route('/api/' + API_VERSION + '/reviews/<string:product>', methods = ['GET'])
def get_review_for(product):
    start = time.time()
    table = connection.table('isura_reviews')
    # Keys are stored in lowercase in HBase for more flexible searching.
    row = table.row(product.lower())
    reviews = []
    if not row:
        abort(404)
    for val in row.itervalues():
        reviews.append(json.loads(val))
    reviews = sorted(reviews, key=lambda k: k['timestamp'])
    reviews = { 'reviews' : reviews }
    reviews['meta'] = { 'count' : len(row), 'responseTime' : time.time() - start }
    return jsonify(reviews)

@app.route('/api/' + API_VERSION + '/reviews/by-keyword/<string:query>', methods = ['GET'])
def get_reviews_by_keyword(query):
    keyword_table = connection.table('isura_products_by_keyword')
    product_table = connection.table('isura_reviews_by_product_id')
    query = query.lower()
    keywords = re.split("[^a-z0-9]+", query)
    hits = {}
    for w in keywords:
        keyword_row = keyword_table.row(w)
        if keyword_row:
            for product_id in keyword_row.itervalues():
                if product_id in hits:
                    hits[product_id] += 1
                else:
                    hits[product_id] = 1
    hits = sorted(hits.items(), key=lambda x:x[1])
    hits = [h[0] for h in hits]
    results = {}
    for product_id in hits:
        product_row = product_table.row(product_id)
        if product_row:
            results[product_id] = product_row.values()
        else:
            raise Exception("The product_id should be in the table!")
    return jsonify(results)

if __name__ == '__main__':
    app.run(debug = True)
<|endoftext|>


<commit_before># Django settings for fecreader project.
import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': '', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.4/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/New_York'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'compressor.finders.CompressorFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'mw#t^1(&-#pb8(p#g#&tqbv7v!mj3hr@cc4e6e8ybn5v#4$b4@'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'debug_toolbar.middleware.DebugToolbarMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    # need to point at other one
    #'summary_data.brisket_context_processors.custom_context'
    'dryrub.context_processors.custom_context'
)

ROOT_URLCONF = 'fecreader.urls'

IE_MEDIA_URL = 'http://s3.amazonaws.com/assets.sunlightfoundation.com/brisket/1.0/'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'fecreader.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.template.TemplateDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # compressor
    'compressor',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'django.contrib.humanize',
    'formdata',
    'ftpdata',
    'fec_alerts',
    'legislators',
    'summary_data',
    'rest_framework',
    'debug_toolbar',
    'south',
    'dryrub',
    'downloads',
)

REST_FRAMEWORK = {

#    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.AllowAny',),
    'DEFAULT_AUTHENTICATION_CLASSES': ('django.contrib.auth.models.AnonymousUser',),
    'PAGINATE_BY': 100,
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
    'DEFAULT_RENDERER_CLASSES': (
        'rest_framework.renderers.JSONRenderer',
        'rest_framework.renderers.BrowsableAPIRenderer',
    )

}

## Timestamp keys

FILING_SCRAPE_KEY = 'scrape_electronic_filings'
COMMITTEES_SCRAPE_KEY = 'committee_scrape'
BULK_EXPORT_KEY = 'bulk_export'
SUMMARY_EXPORT_KEY = 'summary_export'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.

#if DEBUG:
#    import logging
#    l = logging.getLogger('django.db.backends')
#    l.setLevel(logging.DEBUG)
#    l.addHandler(logging.StreamHandler())

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        #'console': {
        #    'level': 'DEBUG',
        #    'class': 'logging.StreamHandler',
        #},
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        }
        #,'django.db.backends.psycopg2': {
        #    'level': 'DEBUG',
        #    'handers': ['console'],
        #},
    }
}

try:
    from local_settings import *
except ImportError, exp:
    pass
<commit_msg>settings diff<commit_after># Django settings for fecreader project.
import os

DEBUG = False
TEMPLATE_DEBUG = False

PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': '', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.4/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['*']

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/New_York'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'compressor.finders.CompressorFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'mw#t^1(&-#pb8(p#g#&tqbv7v!mj3hr@cc4e6e8ybn5v#4$b4@'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'debug_toolbar.middleware.DebugToolbarMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    # need to point at other one
    #'summary_data.brisket_context_processors.custom_context'
    'dryrub.context_processors.custom_context'
)

ROOT_URLCONF = 'fecreader.urls'

IE_MEDIA_URL = 'http://s3.amazonaws.com/assets.sunlightfoundation.com/brisket/1.0/'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'fecreader.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.template.TemplateDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # compressor
    'compressor',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'django.contrib.humanize',
    'formdata',
    'ftpdata',
    'fec_alerts',
    'legislators',
    'summary_data',
    'rest_framework',
    'debug_toolbar',
    'south',
    'dryrub',
    'downloads',
)

REST_FRAMEWORK = {

#    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.AllowAny',),
    'DEFAULT_AUTHENTICATION_CLASSES': ('django.contrib.auth.models.AnonymousUser',),
    'PAGINATE_BY': 100,
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
    'DEFAULT_RENDERER_CLASSES': (
        'rest_framework.renderers.JSONRenderer',
        'rest_framework.renderers.BrowsableAPIRenderer',
    )

}

## Timestamp keys

FILING_SCRAPE_KEY = 'scrape_electronic_filings'
COMMITTEES_SCRAPE_KEY = 'committee_scrape'
BULK_EXPORT_KEY = 'bulk_export'
SUMMARY_EXPORT_KEY = 'summary_export'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.

#if DEBUG:
#    import logging
#    l = logging.getLogger('django.db.backends')
#    l.setLevel(logging.DEBUG)
#    l.addHandler(logging.StreamHandler())

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        #'console': {
        #    'level': 'DEBUG',
        #    'class': 'logging.StreamHandler',
        #},
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        }
        #,'django.db.backends.psycopg2': {
        #    'level': 'DEBUG',
        #    'handers': ['console'],
        #},
    }
}

try:
    from local_settings import *
except ImportError, exp:
    pass
<|endoftext|>


<commit_before>from math import ceil
import numpy as np
import matplotlib
plt = matplotlib.pyplot

###########################
# VISUALIZATION FUNCTIONS #
###########################

def imshow_grey(im):
    return plt.imshow(im, cmap=plt.cm.gray, interpolation='nearest')

def imshow_jet(im):
    return plt.imshow(im, cmap=plt.cm.jet, interpolation='nearest')

def imshow_rand(im):
    rcmap = matplotlib.colors.ListedColormap(np.concatenate(
        (np.zeros((1,3)), np.random.rand(ceil(im.max()), 3))
    ))
    return plt.imshow(im, cmap=rcmap, interpolation='nearest')

def inspect_segs_3D(*args, **kwargs):
    """Show corresponding slices side by side in multiple segmentations."""
    z = 0
    if kwargs.has_key('z'):
        z = kwargs['z']
    axis=-1
    if kwargs.has_key('axis'):
        axis = kwargs['axis']
    numplots = 0
    im = None
    if kwargs.has_key('image'):
        im = kwargs['image']
        numplots += 1
    fignum = 1
    if kwargs.has_key('fignum'):
        fignum = kwargs['fignum']
    prob = None
    if kwargs.has_key('prob'):
        prob = kwargs['prob']
        numplots += 1
    numplots += len(args)
    plot_arrangements = []
    for i in range(1,4):
        for j in range(i,4):
            plot_arrangements.append((i*j, i,j))
    # first plot arrangement
    plot_arrangement = [(i,j) for p,i,j in plot_arrangements
                                                    if p >= numplots][0]
    fig = plt.figure(fignum)
    current_subplot = 1
    if im is not None:
        plt.subplot(*plot_arrangement+(current_subplot,))
        imshow_grey(im.swapaxes(0,axis)[z])
        current_subplot += 1
    if prob is not None:
        plt.subplot(*plot_arrangement+(current_subplot,))
        imshow_jet(prob.swapaxes(0,axis)[z])
        current_subplot += 1
    for i, j in enumerate(range(current_subplot, numplots+1)):
        plt.subplot(*plot_arrangement+(j,))
        imshow_rand(args[i].swapaxes(0,axis)[z])
    return fig
<commit_msg>added visualizations for the output of agglom.agglom_segmentations()<commit_after>from math import ceil
import numpy as np
import evaluate
import matplotlib
plt = matplotlib.pyplot

###########################
# VISUALIZATION FUNCTIONS #
###########################

def imshow_grey(im):
    return plt.imshow(im, cmap=plt.cm.gray, interpolation='nearest')

def imshow_jet(im):
    return plt.imshow(im, cmap=plt.cm.jet, interpolation='nearest')

def imshow_rand(im):
    rcmap = matplotlib.colors.ListedColormap(np.concatenate(
        (np.zeros((1,3)), np.random.rand(ceil(im.max()), 3))
    ))
    return plt.imshow(im, cmap=rcmap, interpolation='nearest')

def inspect_segs_3D(*args, **kwargs):
    """Show corresponding slices side by side in multiple segmentations."""
    z = 0
    if kwargs.has_key('z'):
        z = kwargs['z']
    axis=-1
    if kwargs.has_key('axis'):
        axis = kwargs['axis']
    numplots = 0
    im = None
    if kwargs.has_key('image'):
        im = kwargs['image']
        numplots += 1
    fignum = 1
    if kwargs.has_key('fignum'):
        fignum = kwargs['fignum']
    prob = None
    if kwargs.has_key('prob'):
        prob = kwargs['prob']
        numplots += 1
    numplots += len(args)
    plot_arrangements = []
    for i in range(1,4):
        for j in range(i,4):
            plot_arrangements.append((i*j, i,j))
    # first plot arrangement
    plot_arrangement = [(i,j) for p,i,j in plot_arrangements
                                                    if p >= numplots][0]
    fig = plt.figure(fignum)
    current_subplot = 1
    if im is not None:
        plt.subplot(*plot_arrangement+(current_subplot,))
        imshow_grey(im.swapaxes(0,axis)[z])
        current_subplot += 1
    if prob is not None:
        plt.subplot(*plot_arrangement+(current_subplot,))
        imshow_jet(prob.swapaxes(0,axis)[z])
        current_subplot += 1
    for i, j in enumerate(range(current_subplot, numplots+1)):
        plt.subplot(*plot_arrangement+(j,))
        imshow_rand(args[i].swapaxes(0,axis)[z])
    return fig

def plot_voi(r, gt, fig=None):
    """ Plot the voi from segmentations output from agglo.agglom_segmentations(). """
    v = []
    n = []
    for i in r:
        v.append(evaluate.voi(i[0], gt))
        n.append(len(np.unique(i[0])-1))
    if fig is None:
        fig = plt.figure()
    plt.plot(n, v, figure = fig)
    plt.xlabel('Number of segments', figure = fig)
    plt.ylabel('VOI', figure = fig)

def plot_num_segments(r, gt, num, fig=None):
    """ Plot a segmentation with a specific number of semgents.

        The first input should be the output from agglo.agglom_segmentations().

    """
    i = len(r)-1
    n = len(np.unique(r[i])-1)
    while n > num:
        i -= 1
        n = len(np.unique(r[i])-1)
    v = evaluate.voi(r[i][0], gt)
    if fig is None:
        fig = plt.figure()
    plt.imshow(r[i][0], figure = fig)
    plt.title('VOI: ' + str(v) + '\t Number of segments: ' + str(n), figure = fig)

<|endoftext|>


<commit_before>"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

Written (W) 2013 Heiko Strathmann
"""
from main.distribution.Distribution import Distribution
from numpy.ma.core import shape, zeros, exp
from shogun.Features import BinaryLabels, RealFeatures
from shogun.Classifier import LaplacianInferenceMethod, LogitLikelihood, \
    ZeroMean
from shogun.Kernel import GaussianARDKernel

class PseudoMarginalHyperparameterDistribution(Distribution):
    """
    Class to represent a GP's marginal posterior distribution of hyperparameters

    p(theta|y) \propto p(y|theta) p(theta)

    as an MCMC target. The p(y|theta) function is an unbiased estimate.
    Hyperparameters are the length scales of a Gaussian ARD kernel.

    Uses the Shogun machine learning toolbox for GP inference.
    """
    def __init__(self, X, y, n_importance, prior, ridge=None):
        Distribution.__init__(self, dimension=shape(X)[1])

        self.n_importance=n_importance
        self.prior=prior
        self.ridge=ridge
        self.X=X
        self.y=y

    def log_pdf(self, thetas):
        assert(len(shape(thetas)) == 2)
        assert(shape(thetas)[1] == self.dimension)

        result=zeros(len(thetas))
        for i in range(len(thetas)):
            labels=BinaryLabels(self.y)
            feats_train=RealFeatures(self.X.T)

            # ARD: set set theta, which is in log-scale, as kernel weights
            kernel=GaussianARDKernel(10,1)
            kernel.set_weights(exp(thetas[i]))

            mean=ZeroMean()
            likelihood=LogitLikelihood()
            inference=LaplacianInferenceMethod(kernel, feats_train, mean, labels, likelihood)

            # fix kernel scaling for now
            inference.set_scale(exp(0))

            if self.ridge is not None:
                log_ml_estimate=inference.get_marginal_likelihood_estimate(self.n_importance, self.ridge)
            else:
                log_ml_estimate=inference.get_marginal_likelihood_estimate(self.n_importance)

            # prior is also in log-domain, so no exp of theta
            log_prior=self.prior.log_pdf(thetas[i].reshape(1,len(thetas[i])))
            result[i]=log_ml_estimate+log_prior

        return result

<commit_msg>tell shogun to use one thread only<commit_after>"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

Written (W) 2013 Heiko Strathmann
"""
from main.distribution.Distribution import Distribution
from numpy.ma.core import shape, zeros, exp
from shogun.Features import BinaryLabels, RealFeatures
from shogun.Classifier import LaplacianInferenceMethod, LogitLikelihood, \
    ZeroMean
from shogun.Kernel import GaussianARDKernel

# tell shogun to use 1 thread only (extra to python)
num_threads=1
print "Using Shogun with %d threads" % num_threads
ZeroMean().parallel.set_num_threads(num_threads)

class PseudoMarginalHyperparameterDistribution(Distribution):
    """
    Class to represent a GP's marginal posterior distribution of hyperparameters

    p(theta|y) \propto p(y|theta) p(theta)

    as an MCMC target. The p(y|theta) function is an unbiased estimate.
    Hyperparameters are the length scales of a Gaussian ARD kernel.

    Uses the Shogun machine learning toolbox for GP inference.
    """
    def __init__(self, X, y, n_importance, prior, ridge=None):
        Distribution.__init__(self, dimension=shape(X)[1])

        self.n_importance=n_importance
        self.prior=prior
        self.ridge=ridge
        self.X=X
        self.y=y

    def log_pdf(self, thetas):
        assert(len(shape(thetas)) == 2)
        assert(shape(thetas)[1] == self.dimension)

        result=zeros(len(thetas))
        for i in range(len(thetas)):
            labels=BinaryLabels(self.y)
            feats_train=RealFeatures(self.X.T)

            # ARD: set set theta, which is in log-scale, as kernel weights
            kernel=GaussianARDKernel(10,1)
            kernel.set_weights(exp(thetas[i]))

            mean=ZeroMean()
            likelihood=LogitLikelihood()
            inference=LaplacianInferenceMethod(kernel, feats_train, mean, labels, likelihood)

            # fix kernel scaling for now
            inference.set_scale(exp(0))

            if self.ridge is not None:
                log_ml_estimate=inference.get_marginal_likelihood_estimate(self.n_importance, self.ridge)
            else:
                log_ml_estimate=inference.get_marginal_likelihood_estimate(self.n_importance)

            # prior is also in log-domain, so no exp of theta
            log_prior=self.prior.log_pdf(thetas[i].reshape(1,len(thetas[i])))
            result[i]=log_ml_estimate+log_prior

        return result

<|endoftext|>


<commit_before>from mygpo.api.models import Podcast, EpisodeAction
from babel import Locale, UnknownLocaleError
import re

def get_accepted_lang(request):
    return list(set([s[:2] for s in request.META.get('HTTP_ACCEPT_LANGUAGE', '').split(',')]))

def get_podcast_languages():
    """
    Returns all 2-letter language codes that are used by podcasts.

    It filters obviously invalid strings, but does not check if any
    of these codes is contained in ISO 639.
    """

    langs = [x['language'] for x in Podcast.objects.values('language').distinct()]
    sane_lang = sanitize_language_codes(langs)

    sane_lang.sort()

    return sane_lang

def sanitize_language_codes(langs):
    """
    expects a list of language codes and returns a unique lost of the first
    part of all items. obviously invalid entries are skipped

    >>> sanitize_language_codes(['de-at', 'de-ch'])
    ['de']

    >>> sanitize_language_codes(['de-at', 'en', 'en-gb', '(asdf', 'Deutsch'])
    ['de', 'en]
    """

    r = '^[a-zA-Z]{2}[-_]?.*$'
    return list(set([l[:2] for l in langs if l and re.match(r, l)]))

def get_language_names(lang):
    """
    Takes a list of language codes and returns a list of tuples
    with (code, name)
    """
    res = {}
    for l in lang:
        try:
            locale = Locale(l)
        except UnknownLocaleError:
            continue

        if locale.display_name:
            res[l] = locale.display_name

    return res

class UpdatedException(Exception):
    """Base exception with additional payload"""
    def __init__(self, data):
        Exception.__init__(self)
        self.data = data

def get_played_parts(user, episode):
    """
    return a list of length of alternating unplayed, played parts of the given
    episode for the given user and the resulting duration of the episode

    If no information is available, None and the stored duration of the episode
    are returned
    """
    actions = EpisodeAction.objects.filter(episode=episode, user=user, action='play', playmark__isnull=False, started__isnull=False)

    if actions.count() == 0:
        return None, episode.duration

    lengths = [x.total for x in actions]
    median_length = lengths[len(lengths)/2]

    # flatten (merge) all play-parts
    played_parts = flatten_intervals(actions)

    # if end of last played part exceeds median length, extend episode
    length = max(median_length, played_parts[len(played_parts)-1]['end'])

    #split up the played parts in alternating 'unplayed' and 'played'
    #sections, starting with an unplayed
    sections = []

    lastpos = 0
    for played_part in played_parts:
        sections.append(played_part['start'] - lastpos)
        sections.append(played_part['end'] - played_part['start'])
        lastpos = played_part['end']

    intsections = [int(s) for s in sections]

    return intsections, length

def flatten_intervals(actions):
    """
    takes a list of EpisodeActions and returns a sorted
    list of hashtables with start end elements of the flattened
    play intervals.
    """
    actions = filter(lambda x: x.started and x.playmark, actions)
    actions.sort(key=lambda x: x.started)
    played_parts = []
    if len(actions) == 0:
        return []

    first = actions[0]
    flat_date = {'start': first.started, 'end': first.playmark}
    for action in actions:
        if action.started <= flat_date['end'] and action.playmark >= flat_date['end']:
            flat_date['end'] = action.playmark
        elif action.started >= flat_date['start'] and action.playmark <= flat_date['end']:
            # part already contained
            continue
        else:
            played_parts.append(flat_date)
            flat_date = {'start': action.started, 'end': action.playmark}
    played_parts.append(flat_date)
    return played_parts

<commit_msg>fix bug in get_played_parts()<commit_after>from mygpo.api.models import Podcast, EpisodeAction
from babel import Locale, UnknownLocaleError
import re

def get_accepted_lang(request):
    return list(set([s[:2] for s in request.META.get('HTTP_ACCEPT_LANGUAGE', '').split(',')]))

def get_podcast_languages():
    """
    Returns all 2-letter language codes that are used by podcasts.

    It filters obviously invalid strings, but does not check if any
    of these codes is contained in ISO 639.
    """

    langs = [x['language'] for x in Podcast.objects.values('language').distinct()]
    sane_lang = sanitize_language_codes(langs)

    sane_lang.sort()

    return sane_lang

def sanitize_language_codes(langs):
    """
    expects a list of language codes and returns a unique lost of the first
    part of all items. obviously invalid entries are skipped

    >>> sanitize_language_codes(['de-at', 'de-ch'])
    ['de']

    >>> sanitize_language_codes(['de-at', 'en', 'en-gb', '(asdf', 'Deutsch'])
    ['de', 'en]
    """

    r = '^[a-zA-Z]{2}[-_]?.*$'
    return list(set([l[:2] for l in langs if l and re.match(r, l)]))

def get_language_names(lang):
    """
    Takes a list of language codes and returns a list of tuples
    with (code, name)
    """
    res = {}
    for l in lang:
        try:
            locale = Locale(l)
        except UnknownLocaleError:
            continue

        if locale.display_name:
            res[l] = locale.display_name

    return res

class UpdatedException(Exception):
    """Base exception with additional payload"""
    def __init__(self, data):
        Exception.__init__(self)
        self.data = data

def get_played_parts(user, episode):
    """
    return a list of length of alternating unplayed, played parts of the given
    episode for the given user and the resulting duration of the episode

    If no information is available, None and the stored duration of the episode
    are returned
    """
    actions = EpisodeAction.objects.filter(episode=episode, user=user, action='play', playmark__isnull=False, started__isnull=False)

    if actions.count() == 0:
        return None, episode.duration

    lengths = [x.total for x in actions]
    median_length = lengths[len(lengths)/2]

    # flatten (merge) all play-parts
    played_parts = flatten_intervals(actions)

    # if end of last played part exceeds median length, extend episode
    if played_parts:
        length = max(median_length, played_parts[len(played_parts)-1]['end'])
    else:
        return None, episode.duration

    #split up the played parts in alternating 'unplayed' and 'played'
    #sections, starting with an unplayed
    sections = []

    lastpos = 0
    for played_part in played_parts:
        sections.append(played_part['start'] - lastpos)
        sections.append(played_part['end'] - played_part['start'])
        lastpos = played_part['end']

    intsections = [int(s) for s in sections]

    return intsections, length

def flatten_intervals(actions):
    """
    takes a list of EpisodeActions and returns a sorted
    list of hashtables with start end elements of the flattened
    play intervals.
    """
    actions = filter(lambda x: x.started and x.playmark, actions)
    actions.sort(key=lambda x: x.started)
    played_parts = []
    if len(actions) == 0:
        return []

    first = actions[0]
    flat_date = {'start': first.started, 'end': first.playmark}
    for action in actions:
        if action.started <= flat_date['end'] and action.playmark >= flat_date['end']:
            flat_date['end'] = action.playmark
        elif action.started >= flat_date['start'] and action.playmark <= flat_date['end']:
            # part already contained
            continue
        else:
            played_parts.append(flat_date)
            flat_date = {'start': action.started, 'end': action.playmark}
    played_parts.append(flat_date)
    return played_parts

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# RaiBlocks Telegram bot
# @RaiWalletBot https://t.me/RaiWalletBot
#
# Source code:
# https://github.com/SergiySW/RaiWalletBot
#
# Released under the BSD 3-Clause License
#
#
# Run by cron every minute
#

from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
from telegram import Bot, ParseMode
import logging
import urllib3, socket, json
import time, math
import os, sys

# Parse config
import ConfigParser
config = ConfigParser.ConfigParser()
config.read('bot.cfg')
api_key = config.get('main', 'api_key')
log_file_frontiers = config.get('main', 'log_file_frontiers')
wallet = config.get('main', 'wallet')
fee_account = config.get('main', 'fee_account')
fee_amount = int(config.get('main', 'fee_amount'))
raw_fee_amount = fee_amount * (10 ** 30)
incoming_fee = int(config.get('main', 'incoming_fee'))
raw_incoming_fee = incoming_fee * (10 ** 30)

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
					level=logging.INFO, filename=log_file_frontiers)

logger = logging.getLogger(__name__)

account_url = 'https://raiblockscommunity.net/account/index.php?acc='
hash_url = 'https://raiblockscommunity.net/block/index.php?h='
faucet_url = 'https://raiblockscommunity.net/faucet/mobile.php?a='

# MySQL requests
from common_mysql import mysql_update_balance, mysql_update_frontier, mysql_select_accounts_list, mysql_set_price

# Request to node
from common_rpc import *

# Common functions
from common import push

# frontiers check
def frontiers():
	time_start = int(time.time())
	# set bot
	bot = Bot(api_key)
	# list from MySQL
	accounts_list = mysql_select_accounts_list()
	# list from node
	frontiers = rpc({"action":"frontier_list","wallet":wallet}, 'frontiers')

	faucet_account = 'xrb_13ezf4od79h1tgj9aiu4djzcmmguendtjfuhwfukhuucboua8cpoihmh8byo'

	for account in accounts_list:
		try:
			frontier = frontiers[account[1]]
		#if ((account[1] == r.keys()[0]) and (account[2] != frontier)):
			if (account[2] != frontier):
				# update frontier
				balance = account_balance(account[1])
				# check if balance changed
				mysql_update_frontier(account[1], frontier)
				#print('{0} --> {1}	{2}'.format(account[3], balance, frontier))
				logging.info('{0} --> {1}	{2}'.format(account[3], balance, frontier))
				#print(balance)
				if (int(account[3]) < int(balance)):
					received_amount = int(balance) - int(account[3])
					max_send = int(balance) - int(fee_amount)
					if (max_send < 0):
						max_send = 0
					# retrieve sender
					send_tx = json.loads(rpc({"action":"block","hash":frontier}, 'contents'))
					#print (send_tx)
					send_source = send_tx['source']
					#print (send_source)
					block_account = rpc({"action":"block_account","hash":send_source}, 'account')
					#print (block_account)
					sender = ''
					if (block_account == faucet_account):
						sender = ' from Faucet/Landing'
					elif (block_account == fee_account):
						sender = ' from @RaiWalletBot itself'
					elif (block_account == account[1]):
						sender = ' from you'
					else:
						for sender_account in accounts_list:
							if (sender_account[1] == block_account):
								if ((sender_account[4] is not None) and (sender_account[4])):
									sender = ' from @{0}'.format(sender_account[4])
								else:
									sender = ' from one of our users'
					#print (sender)
					logging.info(sender)
					logging.info(block_account)
					# receive fee protection
					mysql_update_balance(account[1], int(balance))
					logging.info('{0} Mrai (XRB) received by {1}, hash: {2}'.format(received_amount, account[0], frontier))
					if (incoming_fee >= 1):
						fee = rpc({"action": "send", "wallet": wallet, "source": account[1], "destination": fee_account, "amount": raw_incoming_fee}, 'block')
						balance = account_balance(account[1])
						push(bot, account[0], '*{0} Mrai (XRB)* received{7}. Transaction hash: [{5}]({6}{5})\nIncoming fee: *{4} Mrai (XRB)*. Your current balance: {1} Mrai (XRB). Send limit: {2} Mrai (XRB)'.format("{:,}".format(received_amount), "{:,}".format(balance), "{:,}".format(max_send), incoming_fee, frontier, hash_url, sender))
						logging.info('Incoming fee deducted')
					else:
						#print(account[0])
						#print('*{0} Mrai (XRB)* received{5}. Transaction hash: [{3}]({4}{3})\nYour current balance: *{1} Mrai (XRB)*. Send limit: {2} Mrai (XRB)'.format("{:,}".format(received_amount), "{:,}".format(balance), "{:,}".format(max_send), frontier, hash_url, sender))
						push(bot, account[0], '*{0} Mrai (XRB)* received{5}. Transaction hash: [{3}]({4}{3})\nYour current balance: *{1} Mrai (XRB)*. Send limit: {2} Mrai (XRB)'.format("{:,}".format(received_amount), "{:,}".format(balance), "{:,}".format(max_send), frontier, hash_url, sender))
					time.sleep(0.1)
		# no frontier. No transactions
		except KeyError:
			# doesn't exist
			x = 0 # do something
	time_end = int(time.time())
	total_time = time_end - time_start
	#print(total_time)
	if (total_time > 20):
		logging.warning(('WARNING!!! \nMore than 20 seconds execution time!!!'))
	return total_time

'''
def cryptopia():
	http = urllib3.PoolManager()
	url = 'https://www.cryptopia.co.nz/api/GetMarket/4874'
	response = http.request('GET', url)
	json_cryptopia = json.loads(response.data)
	json_array = json_cryptopia['Data']
	last_price = int(json_array['LastPrice'] * (10 ** 8))
	high_price = int(json_array['High'] * (10 ** 8))
	low_price = int(json_array['Low'] * (10 ** 8))
	ask_price = int(json_array['AskPrice'] * (10 ** 8))
	bid_price = int(json_array['BidPrice'] * (10 ** 8))
	volume = int(json_array['Volume'])

	mysql_set_price(last_price, high_price, low_price, ask_price, bid_price, volume)
'''

def mercatox():
	http = urllib3.PoolManager()
	url = 'https://mercatox.com/public/json24'
	response = http.request('GET', url)
	json_mercatox = json.loads(response.data)
	json_array = json_mercatox['pairs']['XRB_BTC']
	last_price = int(float(json_array['last']) * (10 ** 8))
	high_price = int(float(json_array['high24hr']) * (10 ** 8))
	low_price = int(float(json_array['low24hr']) * (10 ** 8))
	ask_price = int(float(json_array['lowestAsk']) * (10 ** 8))
	bid_price = int(float(json_array['highestBid']) * (10 ** 8))
	volume = int(float(json_array['baseVolume']))

	mysql_set_price(last_price, high_price, low_price, ask_price, bid_price, volume)

def frontiers_usual():
	run_time = frontiers()
	if (run_time < 15):
		time.sleep(15-run_time)
		run_time = frontiers()
		if (run_time < 15):
			time.sleep(15-run_time)
			run_time = frontiers()
			if (run_time < 15):
				time.sleep(15-run_time)
				frontiers()
	elif (run_time <= 30):
		time.sleep(30-run_time)
		frontiers()

	#cryptopia()
	mercatox()

# check if already running
def frontiers_proc_check():
	for dirname in os.listdir('/proc'):
		if dirname == 'curproc':
			continue

		try:
			with open('/proc/{}/cmdline'.format(dirname), mode='rb') as fd:
				content = fd.read().decode().split('\x00')
		except Exception:
			continue

		cont3 = ''
		if (len(content) == 3):
			cont3 = content[2]
		if (('python' in content[0]) or ('python' in cont3)):
			if (((content[1] in sys.argv[0]) or (sys.argv[0] in cont3)) and ('self' not in dirname)):
				frontiers_usual()

#frontiers_proc_check()
frontiers_usual()
<commit_msg>frontier_list --> wallet_frontiers<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# RaiBlocks Telegram bot
# @RaiWalletBot https://t.me/RaiWalletBot
#
# Source code:
# https://github.com/SergiySW/RaiWalletBot
#
# Released under the BSD 3-Clause License
#
#
# Run by cron every minute
#

from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
from telegram import Bot, ParseMode
import logging
import urllib3, socket, json
import time, math
import os, sys

# Parse config
import ConfigParser
config = ConfigParser.ConfigParser()
config.read('bot.cfg')
api_key = config.get('main', 'api_key')
log_file_frontiers = config.get('main', 'log_file_frontiers')
wallet = config.get('main', 'wallet')
fee_account = config.get('main', 'fee_account')
fee_amount = int(config.get('main', 'fee_amount'))
raw_fee_amount = fee_amount * (10 ** 30)
incoming_fee = int(config.get('main', 'incoming_fee'))
raw_incoming_fee = incoming_fee * (10 ** 30)

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
					level=logging.INFO, filename=log_file_frontiers)

logger = logging.getLogger(__name__)

account_url = 'https://raiblockscommunity.net/account/index.php?acc='
hash_url = 'https://raiblockscommunity.net/block/index.php?h='
faucet_url = 'https://raiblockscommunity.net/faucet/mobile.php?a='

# MySQL requests
from common_mysql import mysql_update_balance, mysql_update_frontier, mysql_select_accounts_list, mysql_set_price

# Request to node
from common_rpc import *

# Common functions
from common import push

# frontiers check
def frontiers():
	time_start = int(time.time())
	# set bot
	bot = Bot(api_key)
	# list from MySQL
	accounts_list = mysql_select_accounts_list()
	# list from node
	frontiers = rpc({"action":"wallet_frontiers","wallet":wallet}, 'frontiers')

	faucet_account = 'xrb_13ezf4od79h1tgj9aiu4djzcmmguendtjfuhwfukhuucboua8cpoihmh8byo'

	for account in accounts_list:
		try:
			frontier = frontiers[account[1]]
		#if ((account[1] == r.keys()[0]) and (account[2] != frontier)):
			if (account[2] != frontier):
				# update frontier
				balance = account_balance(account[1])
				# check if balance changed
				mysql_update_frontier(account[1], frontier)
				#print('{0} --> {1}	{2}'.format(account[3], balance, frontier))
				logging.info('{0} --> {1}	{2}'.format(account[3], balance, frontier))
				#print(balance)
				if (int(account[3]) < int(balance)):
					received_amount = int(balance) - int(account[3])
					max_send = int(balance) - int(fee_amount)
					if (max_send < 0):
						max_send = 0
					# retrieve sender
					send_tx = json.loads(rpc({"action":"block","hash":frontier}, 'contents'))
					#print (send_tx)
					send_source = send_tx['source']
					#print (send_source)
					block_account = rpc({"action":"block_account","hash":send_source}, 'account')
					#print (block_account)
					sender = ''
					if (block_account == faucet_account):
						sender = ' from Faucet/Landing'
					elif (block_account == fee_account):
						sender = ' from @RaiWalletBot itself'
					elif (block_account == account[1]):
						sender = ' from you'
					else:
						for sender_account in accounts_list:
							if (sender_account[1] == block_account):
								if ((sender_account[4] is not None) and (sender_account[4])):
									sender = ' from @{0}'.format(sender_account[4])
								else:
									sender = ' from one of our users'
					#print (sender)
					logging.info(sender)
					logging.info(block_account)
					# receive fee protection
					mysql_update_balance(account[1], int(balance))
					logging.info('{0} Mrai (XRB) received by {1}, hash: {2}'.format(received_amount, account[0], frontier))
					if (incoming_fee >= 1):
						fee = rpc({"action": "send", "wallet": wallet, "source": account[1], "destination": fee_account, "amount": raw_incoming_fee}, 'block')
						balance = account_balance(account[1])
						push(bot, account[0], '*{0} Mrai (XRB)* received{7}. Transaction hash: [{5}]({6}{5})\nIncoming fee: *{4} Mrai (XRB)*. Your current balance: {1} Mrai (XRB). Send limit: {2} Mrai (XRB)'.format("{:,}".format(received_amount), "{:,}".format(balance), "{:,}".format(max_send), incoming_fee, frontier, hash_url, sender))
						logging.info('Incoming fee deducted')
					else:
						#print(account[0])
						#print('*{0} Mrai (XRB)* received{5}. Transaction hash: [{3}]({4}{3})\nYour current balance: *{1} Mrai (XRB)*. Send limit: {2} Mrai (XRB)'.format("{:,}".format(received_amount), "{:,}".format(balance), "{:,}".format(max_send), frontier, hash_url, sender))
						push(bot, account[0], '*{0} Mrai (XRB)* received{5}. Transaction hash: [{3}]({4}{3})\nYour current balance: *{1} Mrai (XRB)*. Send limit: {2} Mrai (XRB)'.format("{:,}".format(received_amount), "{:,}".format(balance), "{:,}".format(max_send), frontier, hash_url, sender))
					time.sleep(0.1)
		# no frontier. No transactions
		except KeyError:
			# doesn't exist
			x = 0 # do something
	time_end = int(time.time())
	total_time = time_end - time_start
	#print(total_time)
	if (total_time > 20):
		logging.warning(('WARNING!!! \nMore than 20 seconds execution time!!!'))
	return total_time

'''
def cryptopia():
	http = urllib3.PoolManager()
	url = 'https://www.cryptopia.co.nz/api/GetMarket/4874'
	response = http.request('GET', url)
	json_cryptopia = json.loads(response.data)
	json_array = json_cryptopia['Data']
	last_price = int(json_array['LastPrice'] * (10 ** 8))
	high_price = int(json_array['High'] * (10 ** 8))
	low_price = int(json_array['Low'] * (10 ** 8))
	ask_price = int(json_array['AskPrice'] * (10 ** 8))
	bid_price = int(json_array['BidPrice'] * (10 ** 8))
	volume = int(json_array['Volume'])

	mysql_set_price(last_price, high_price, low_price, ask_price, bid_price, volume)
'''

def mercatox():
	http = urllib3.PoolManager()
	url = 'https://mercatox.com/public/json24'
	response = http.request('GET', url)
	json_mercatox = json.loads(response.data)
	json_array = json_mercatox['pairs']['XRB_BTC']
	last_price = int(float(json_array['last']) * (10 ** 8))
	high_price = int(float(json_array['high24hr']) * (10 ** 8))
	low_price = int(float(json_array['low24hr']) * (10 ** 8))
	ask_price = int(float(json_array['lowestAsk']) * (10 ** 8))
	bid_price = int(float(json_array['highestBid']) * (10 ** 8))
	volume = int(float(json_array['baseVolume']))

	mysql_set_price(last_price, high_price, low_price, ask_price, bid_price, volume)

def frontiers_usual():
	run_time = frontiers()
	if (run_time < 15):
		time.sleep(15-run_time)
		run_time = frontiers()
		if (run_time < 15):
			time.sleep(15-run_time)
			run_time = frontiers()
			if (run_time < 15):
				time.sleep(15-run_time)
				frontiers()
	elif (run_time <= 30):
		time.sleep(30-run_time)
		frontiers()

	#cryptopia()
	mercatox()

# check if already running
def frontiers_proc_check():
	for dirname in os.listdir('/proc'):
		if dirname == 'curproc':
			continue

		try:
			with open('/proc/{}/cmdline'.format(dirname), mode='rb') as fd:
				content = fd.read().decode().split('\x00')
		except Exception:
			continue

		cont3 = ''
		if (len(content) == 3):
			cont3 = content[2]
		if (('python' in content[0]) or ('python' in cont3)):
			if (((content[1] in sys.argv[0]) or (sys.argv[0] in cont3)) and ('self' not in dirname)):
				frontiers_usual()

#frontiers_proc_check()
frontiers_usual()
<|endoftext|>