<commit_before>#!/usr/bin/env python3
# Copyright 2010-2021 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# [START program]
"""From Bradley, H. and M., 'Applied Mathematical Programming', figure 8.1."""
# [START import]
from ortools.graph import pywrapgraph
# [END import]

def main():
    """MinCostFlow simple interface example."""
    # [START data]
    # Define four parallel arrays: sources, destinations, capacities,
    # and unit costs between each pair. For instance, the arc from node 0
    # to node 1 has a capacity of 15.
    start_nodes = [0, 0, 1, 1, 1, 2, 2, 3, 4]
    end_nodes = [1, 2, 2, 3, 4, 3, 4, 4, 2]
    capacities = [15, 8, 20, 4, 10, 15, 4, 20, 5]
    unit_costs = [4, 4, 2, 2, 6, 1, 3, 2, 3]

    # Define an array of supplies at each node.
    supplies = [20, 0, 0, -5, -15]
    # [END data]

    # [START constraints]
    # Instantiate a SimpleMinCostFlow solver.
    min_cost_flow = pywrapgraph.SimpleMinCostFlow()

    # Add each arc.
    for arc in zip(start_nodes, end_nodes, capacities, unit_costs):
        min_cost_flow.AddArcWithCapacityAndUnitCost(arc[0], arc[1], arc[2],
                                                    arc[3])

    # Add node supplies.
    for count, supply in enumerate(supplies):
        min_cost_flow.SetNodeSupply(count, supply)
    # [END constraints]

    # [START solve]
    # Find the min cost flow.
    solve_status = min_cost_flow.Solve()
    # [END solve]

    # [START print_solution]
    if solve_status == min_cost_flow.OPTIMAL:
        print('Minimum cost: ', min_cost_flow.OptimalCost())
        print('')
        print(' Arc   Flow / Capacity  Cost')
        for i in range(min_cost_flow.NumArcs()):
            cost = min_cost_flow.Flow(i) * min_cost_flow.UnitCost(i)
            print('%1s -> %1s    %3s   / %3s   %3s' %
                  (min_cost_flow.Tail(i), min_cost_flow.Head(i),
                   min_cost_flow.Flow(i), min_cost_flow.Capacity(i), cost))
    else:
        print('Solving the min cost flow problem failed. Solver status: ',
              solve_status)
    # [END print_solution]

if __name__ == '__main__':
    main()
# [END program]
<commit_msg>graph: Update Python simple_min_cost_flow<commit_after>#!/usr/bin/env python3
# Copyright 2010-2021 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# [START program]
"""From Bradley, Hax and Maganti, 'Applied Mathematical Programming', figure 8.1."""
# [START import]
from ortools.graph import pywrapgraph
# [END import]

def main():
    """MinCostFlow simple interface example."""
    # [START solver]
    # Instantiate a SimpleMinCostFlow solver.
    min_cost_flow = pywrapgraph.SimpleMinCostFlow()
    # [END solver]

    # [START data]
    # Define four parallel arrays: sources, destinations, capacities,
    # and unit costs between each pair. For instance, the arc from node 0
    # to node 1 has a capacity of 15.
    start_nodes = [0, 0, 1, 1, 1, 2, 2, 3, 4]
    end_nodes = [1, 2, 2, 3, 4, 3, 4, 4, 2]
    capacities = [15, 8, 20, 4, 10, 15, 4, 20, 5]
    unit_costs = [4, 4, 2, 2, 6, 1, 3, 2, 3]

    # Define an array of supplies at each node.
    supplies = [20, 0, 0, -5, -15]
    # [END data]

    # [START constraints]
    # Add each arc.
    for arc in zip(start_nodes, end_nodes, capacities, unit_costs):
        min_cost_flow.AddArcWithCapacityAndUnitCost(arc[0], arc[1], arc[2],
                                                    arc[3])

    # Add node supply.
    for count, supply in enumerate(supplies):
        min_cost_flow.SetNodeSupply(count, supply)
    # [END constraints]

    # [START solve]
    # Find the min cost flow.
    status = min_cost_flow.Solve()
    # [END solve]

    # [START print_solution]
    if status != min_cost_flow.OPTIMAL:
        print('There was an issue with the min cost flow input.')
        print(f'Status: {status}')
        exit(1)
    print('Minimum cost: ', min_cost_flow.OptimalCost())
    print('')
    print(' Arc   Flow / Capacity  Cost')
    for i in range(min_cost_flow.NumArcs()):
        cost = min_cost_flow.Flow(i) * min_cost_flow.UnitCost(i)
        print('%1s -> %1s    %3s   / %3s   %3s' %
              (min_cost_flow.Tail(i), min_cost_flow.Head(i),
               min_cost_flow.Flow(i), min_cost_flow.Capacity(i), cost))
    # [END print_solution]

if __name__ == '__main__':
    main()
# [END program]
<|endoftext|>


<commit_before>from scanner import scanner
from token import Token

class ParserError(Exception):
    def __init__(self, token_stream, *expected_tokens):
        template = "Received {}, expected {{{}}}"
        if token_stream.lookahead():
            token_msg = "{} ('{}')".format(*token_stream.current)
        else:
            token_msg = 'out of tokens'
        self.message = template.format(token_msg, ', '.join(expected_tokens))

    def __str__(self):
        return self.message

class TokenStream:
    """Stream of tokens, with features needed by a parser."""

    def __init__(self, source):
        self.source = source
        self.current = next(self.source)

    def __iter__(self):
        yield self.current
        yield from self.source

    def lookahead(self):
        """See the token type at the front of the stream"""
        if self.current:
            return self.current[0]

    def consume(self, *expected):
        """Consume the current token, return the next token's literal
        representation and type"""
        print("Consuming,", self.current)
        if self.lookahead() not in expected:
            raise ParserError(self, *expected)
        consumed_type, consumed_literal = self.current
        try:
            self.current = next(self.source)
        except StopIteration:
            self.current = None
        return consumed_literal, consumed_type

    def try_consume(self, *expected):
        if self.lookahead() in expected:
            return self.consume(*expected)

def parser(tokens):
    stream = TokenStream(tokens)
    return statements(stream)

class statements_ast:
    def __init__(self, statements):
        self.statements = statements

    def tree(self, indent=0):
        return '\n'.join(st.tree(indent+1) for st in self.statements)

def statements(stream):
    statements_list = []
    while stream.lookahead():
        statements_list.append(statement(stream))
    return statements_ast(statements_list)

def statement(stream):
    if stream.lookahead() == Token.LET:
        return let(stream)
    elif stream.lookahead() == Token.MACRO:
        return macro(stream)
    else:
        raise ParserError(stream, Token.LET)

class let_ast:
    def __init__(self, var, size):
        self.var = var
        self.size = size

    def tree(self, indent=0):
        return '\t' * indent + f'Let {self.var}[{self.size}]'

def let(stream):
    stream.consume(Token.LET)
    var, _ = stream.consume(Token.ID)
    if stream.try_consume(Token.LSBRA):
        size_tok, _ = stream.consume(Token.NUM)
        stream.consume(Token.RSBRA)
        size = int(size_tok)
    else:
        size = 1
    if stream.try_consume(Token.ASSIGN):
        expr = expression()
        return statements_ast(let_ast(var, size), expr)
    else:
        return let_ast(var, size)

def macro(stream):
    stream.consume(Token.MACRO)
    name, _ = stream.consume(Token.MACROID)
    if stream.consume(Token.LPAR):
        params = [stream.consume(Token.MACROPARAM)]
        while stream.try_consume(Token.COMMA):
            params.append(stream.consume(Token.MACROPARAM))
        stream.consume(Token.RPAR)
    else:
        params = []
    stream.consume(Token.LBRAC)
    contents = statements(stream)
    stream.consume(Token.RBRAC)
    return macro_ast(name, params, contents)

with open('example.mc') as f:
    ast = parser(scanner(f.read(), Token.regexes))
    print(ast.tree())

<commit_msg>Move macro, add value<commit_after>from scanner import scanner
from token import Token

class ParserError(Exception):
    def __init__(self, token_stream, *expected_tokens):
        template = "Received {}, expected {{{}}}"
        if token_stream.lookahead():
            token_msg = "{} ('{}')".format(*token_stream.current)
        else:
            token_msg = 'out of tokens'
        self.message = template.format(token_msg, ', '.join(expected_tokens))

    def __str__(self):
        return self.message

class TokenStream:
    """Stream of tokens, with features needed by a parser."""

    def __init__(self, source):
        self.source = source
        self.current = next(self.source)

    def __iter__(self):
        yield self.current
        yield from self.source

    def lookahead(self):
        """See the token type at the front of the stream"""
        if self.current:
            return self.current[0]

    def consume(self, *expected):
        """Consume the current token, return the next token's literal
        representation and type"""
        print("Consuming,", self.current)
        if self.lookahead() not in expected:
            raise ParserError(self, *expected)
        consumed_type, consumed_literal = self.current
        try:
            self.current = next(self.source)
        except StopIteration:
            self.current = None
        return consumed_literal, consumed_type

    def try_consume(self, *expected):
        if self.lookahead() in expected:
            return self.consume(*expected)
# Macros should be preprocessed
macros = {}

def macrodef(stream):
    stream.consume(Token.MACRO)
    name, _ = stream.consume(Token.MACROID)
    if stream.consume(Token.LPAR):
        params = [stream.consume(Token.MACROPARAM)[0]]
        while stream.try_consume(Token.COMMA):
            params.append(stream.consume(Token.MACROPARAM)[0])
        stream.consume(Token.RPAR)
    else:
        params = []
    stream.consume(Token.LBRAC)
    contents = expression(stream)
    stream.consume(Token.RBRAC)
    macros[name] = (params, contents)

def parser(tokens):
    stream = TokenStream(tokens)
    return statements(stream)

class statements_ast:
    def __init__(self, statements):
        self.statements = statements

    def tree(self, indent=0):
        return '\n'.join(st.tree(indent+1) for st in self.statements)

def statements(stream):
    statements_list = []
    while stream.lookahead():
        statements_list.append(statement(stream))
    return statements_ast(statements_list)

def statement(stream):
    if stream.lookahead() == Token.LET:
        return let(stream)
    else:
        raise ParserError(stream, Token.LET)

class let_ast:
    def __init__(self, var, size):
        self.var = var
        self.size = size

    def tree(self, indent=0):
        return '\t' * indent + f'Let {self.var}[{self.size}]'

def let(stream):
    stream.consume(Token.LET)
    if stream.try_consume(Token.LSBRA):
        size_tok, _ = stream.consume(Token.NUM)
        stream.consume(Token.RSBRA)
        size = int(size_tok)
    else:
        size = 1
    var, _ = stream.consume(Token.ID)
    if stream.try_consume(Token.ASSIGN):
        expr = expression()
        return statements_ast(let_ast(var, size), expr)
    else:
        return let_ast(var, size)

def value(stream):
    if stream.lookahead() == Token.NUM:
        return literal_ast(int(stream.consume(Token.NUM)[0]))
    elif stream.lookahead() == Token.CHR:
        return literal_ast(ord(stream.consume(Token.CHR)[0][1]))
    else:
        raise ParserError(stream, Token.NUM, Token.CHR)

with open('example.mc') as f:
    ast = parser(scanner(f.read(), Token.regexes))
    print(ast.tree())

<|endoftext|>


<commit_before>import math

import numpy

import chainer
from chainer.backends import cuda
from chainer import distribution
from chainer.functions.array import expand_dims
from chainer.functions.array import repeat
from chainer.functions.math import erfc
from chainer.functions.math import erfcinv
from chainer.functions.math import exponential
from chainer.utils import argument

ENTROPYC = 0.5 * math.log(2 * math.pi * math.e)
LOGPROBC = - 0.5 * math.log(2 * math.pi)
PROBC = 1. / (2 * math.pi) ** 0.5

class Normal(distribution.Distribution):

    """Normal Distribution.

    The probability density function of the distribution is expressed as

    .. math::
        p(x;\\mu,\\sigma) = \\frac{1}{\\sqrt{2\\pi\\sigma^2}}
            \\exp\\left(-\\frac{(x-\\mu)^2}{2\\sigma^2}\\right)

    Args:
        loc(:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`): Parameter of distribution representing the \
        location :math:`\\mu`. This is the mean parameter.
        scale(:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`): Parameter of distribution representing the \
        scale :math:`\\sigma`. Either `scale` or `log_scale` (not both) must \
        have a value.
        log_scale(:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`): Parameter of distribution representing the \
        scale :math:`\\log(\\sigma)`. Either `scale` or `log_scale` (not \
        both) must have a value.

    """

    def __init__(self, loc, scale=None, **kwargs):
        super(Normal, self).__init__()
        log_scale = None
        if kwargs:
            log_scale, = argument.parse_kwargs(
                kwargs, ('log_scale', log_scale))
        if not (scale is None) ^ (log_scale is None):
            raise ValueError(
                "Either `scale` or `log_scale` (not both) must have a value.")
        self.loc = chainer.as_variable(loc)

        with chainer.using_config('enable_backprop', True):
            if scale is None:
                self.__log_scale = chainer.as_variable(log_scale)
                self.__scale = exponential.exp(self.log_scale)
            else:
                self.__scale = chainer.as_variable(scale)
                self.__log_scale = exponential.log(self.scale)

    @property
    def scale(self):
        return self.__scale

    @property
    def log_scale(self):
        return self.__log_scale

    @property
    def batch_shape(self):
        return self.loc.shape

    def cdf(self, x):
        return 0.5 * erfc.erfc((self.loc - x) / (2 ** 0.5 * self.scale))

    @property
    def entropy(self):
        return self.log_scale + ENTROPYC

    @property
    def event_shape(self):
        return ()

    def icdf(self, x):
        return self.loc - (
            erfcinv.erfcinv(2. * chainer.as_variable(x))
            * (2 ** 0.5 * self.scale))

    @property
    def _is_gpu(self):
        return isinstance(self.loc.data, cuda.ndarray)

    def log_cdf(self, x):
        return exponential.log(self.cdf(x))

    def log_prob(self, x):
        return LOGPROBC - self.log_scale \
            - 0.5 * (x - self.loc) ** 2 / self.scale ** 2

    def log_survival_function(self, x):
        return exponential.log(self.survival_function(x))

    @property
    def mean(self):
        return self.loc

    def prob(self, x):
        return (PROBC / self.scale) * exponential.exp(
            - 0.5 * (x - self.loc) ** 2 / self.scale ** 2)

    def sample_n(self, n):
        if self._is_gpu:
            eps = cuda.cupy.random.standard_normal(
                (n,)+self.loc.shape, dtype=self.loc.dtype)
        else:
            eps = numpy.random.standard_normal(
                (n,)+self.loc.shape).astype(numpy.float32)
        noise = repeat.repeat(
            expand_dims.expand_dims(self.scale, axis=0), n, axis=0) * eps
        noise += repeat.repeat(expand_dims.expand_dims(
            self.loc, axis=0), n, axis=0)

        return noise

    @property
    def stddev(self):
        return self.scale

    @property
    def support(self):
        return 'real'

    def survival_function(self, x):
        return 0.5 * erfc.erfc((x - self.loc) / (2 ** 0.5 * self.scale))

    @property
    def variance(self):
        return self.scale ** 2

@distribution.register_kl(Normal, Normal)
def _kl_normal_normal(dist1, dist2):
    return dist2.log_scale - dist1.log_scale \
        + 0.5 * (dist1.scale ** 2 + (dist1.loc - dist2.loc) ** 2) \
        / dist2.scale ** 2 - 0.5
<commit_msg>Use ndtr and log_ndtr in normal distribution<commit_after>import math

import numpy

import chainer
from chainer.backends import cuda
from chainer import distribution
from chainer.functions.array import expand_dims
from chainer.functions.array import repeat
from chainer.functions.math import erfcinv
from chainer.functions.math import exponential
from chainer.functions.math import log_ndtr
from chainer.functions.math import ndtr
from chainer.utils import argument

ENTROPYC = 0.5 * math.log(2 * math.pi * math.e)
LOGPROBC = - 0.5 * math.log(2 * math.pi)
PROBC = 1. / (2 * math.pi) ** 0.5

class Normal(distribution.Distribution):

    """Normal Distribution.

    The probability density function of the distribution is expressed as

    .. math::
        p(x;\\mu,\\sigma) = \\frac{1}{\\sqrt{2\\pi\\sigma^2}}
            \\exp\\left(-\\frac{(x-\\mu)^2}{2\\sigma^2}\\right)

    Args:
        loc(:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`): Parameter of distribution representing the \
        location :math:`\\mu`. This is the mean parameter.
        scale(:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`): Parameter of distribution representing the \
        scale :math:`\\sigma`. Either `scale` or `log_scale` (not both) must \
        have a value.
        log_scale(:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`): Parameter of distribution representing the \
        scale :math:`\\log(\\sigma)`. Either `scale` or `log_scale` (not \
        both) must have a value.

    """

    def __init__(self, loc, scale=None, **kwargs):
        super(Normal, self).__init__()
        log_scale = None
        if kwargs:
            log_scale, = argument.parse_kwargs(
                kwargs, ('log_scale', log_scale))
        if not (scale is None) ^ (log_scale is None):
            raise ValueError(
                "Either `scale` or `log_scale` (not both) must have a value.")
        self.loc = chainer.as_variable(loc)

        with chainer.using_config('enable_backprop', True):
            if scale is None:
                self.__log_scale = chainer.as_variable(log_scale)
                self.__scale = exponential.exp(self.log_scale)
            else:
                self.__scale = chainer.as_variable(scale)
                self.__log_scale = exponential.log(self.scale)

    @property
    def scale(self):
        return self.__scale

    @property
    def log_scale(self):
        return self.__log_scale

    @property
    def batch_shape(self):
        return self.loc.shape

    def cdf(self, x):
        return ndtr.ndtr((x - self.loc) / self.scale)

    @property
    def entropy(self):
        return self.log_scale + ENTROPYC

    @property
    def event_shape(self):
        return ()

    def icdf(self, x):
        return self.loc - (
            erfcinv.erfcinv(2. * chainer.as_variable(x))
            * (2 ** 0.5 * self.scale))

    @property
    def _is_gpu(self):
        return isinstance(self.loc.data, cuda.ndarray)

    def log_cdf(self, x):
        return log_ndtr.log_ndtr((x - self.loc) / self.scale)

    def log_prob(self, x):
        return LOGPROBC - self.log_scale \
            - 0.5 * (x - self.loc) ** 2 / self.scale ** 2

    def log_survival_function(self, x):
        return log_ndtr.log_ndtr((self.loc - x) / self.scale)

    @property
    def mean(self):
        return self.loc

    def prob(self, x):
        return (PROBC / self.scale) * exponential.exp(
            - 0.5 * (x - self.loc) ** 2 / self.scale ** 2)

    def sample_n(self, n):
        if self._is_gpu:
            eps = cuda.cupy.random.standard_normal(
                (n,)+self.loc.shape, dtype=self.loc.dtype)
        else:
            eps = numpy.random.standard_normal(
                (n,)+self.loc.shape).astype(numpy.float32)
        noise = repeat.repeat(
            expand_dims.expand_dims(self.scale, axis=0), n, axis=0) * eps
        noise += repeat.repeat(expand_dims.expand_dims(
            self.loc, axis=0), n, axis=0)

        return noise

    @property
    def stddev(self):
        return self.scale

    @property
    def support(self):
        return 'real'

    def survival_function(self, x):
        return ndtr.ndtr((self.loc - x) / self.scale)

    @property
    def variance(self):
        return self.scale ** 2

@distribution.register_kl(Normal, Normal)
def _kl_normal_normal(dist1, dist2):
    return dist2.log_scale - dist1.log_scale \
        + 0.5 * (dist1.scale ** 2 + (dist1.loc - dist2.loc) ** 2) \
        / dist2.scale ** 2 - 0.5
<|endoftext|>


<commit_before>import argparse
import sys

from parser.helper import infer_edition_from_url, get_html_tree_from_string, get_html_tree_from_url
import importlib

if sys.version_info[0:3] >= (3, 0, 0):  # python 3 (tested)
    from zim.zimpy_p3 import ZimFile
else:  # python 2 (not tested)
    from zim.zimpy_p2 import ZimFile

# key: Wiktionary edition code
# value: parser class (not parser instance)
parsers = {}

headers = ['edition', 'headword', 'head_lang', 'translation', 'trans_lang', 'trans_lang_code', 'pos', 'pronunciation']

# dynamically loading all modules
def import_all_parsers():
    parser_list = ['ja', 'vi', 'tr', 'fr', 'ru', 'uz', 'de', 'az']
    for parser_name in parser_list:
        module_to_import = '.parse_' + parser_name
        module = importlib.import_module(module_to_import, package='parser')
        parsers[parser_name] = getattr(module, parser_name.capitalize() + "Parser")
        # tested_url.extend(module.tested_url)

def get_parser(edition):
    if edition not in parsers:
        module_to_import = '.parse_' + edition
        try:
            module = importlib.import_module(module_to_import, package='parser')
            parsers[edition] = getattr(module, edition.capitalize() + "Parser")
        except Exception as e:
            print(e)
            return None
    # instantiate the class
    return parsers[edition]()

def read_zim_file(file):
    # print(file.metadata())
    # we only need main articles. They are in namespace 'A'.
    namespace = b'A'
    for article in file.articles():
        if article['namespace'] != namespace:
            continue
        body = file.get_article_by_index(
            article['index'], follow_redirect=False)[0]
        if not body:
            continue
        else:
            yield (body.decode('utf-8'))

def test_zim(filename, edition=None):
    file = ZimFile(filename=filename)
    # file.list_articles_by_url()
    edition_lang_code = file.metadata()['language'].decode('utf-8')
    # print(edition_lang_code)

    if edition:
        edition_wikt_code = edition
        # print(edition_wikt_code)
    else:
        import parser.lang_code_conversion as languages
        edition_wikt_code = languages.get_wikt_code_from_iso639_3(edition_lang_code)
        # print(edition_wikt_code)

    print(','.join(headers))
    # get the parser class
    parser = get_parser(edition_wikt_code)
    if parser is None:
        print("We don't have a parser for {}/{} language yet.".format(edition_lang_code, edition_wikt_code))
        return

    # instantiate the parser
    page_generator = read_zim_file(file)
    for page in page_generator:
        soup = get_html_tree_from_string(page)
        for tup in parser.generate_translation_tuples(soup):
            print(",".join(tup))

def test_html(edition=None):
    if edition is None:
        import_all_parsers()
        print(','.join(headers))
        urls_lists = [parser.tested_url for parser in parsers.values()]
        test_urls = [url for sub_list in urls_lists for url in sub_list]
        for url in test_urls:
            edition = infer_edition_from_url(url)
            soup = get_html_tree_from_url(url)
            parser = get_parser(edition)
            for tup in parser.generate_translation_tuples(soup):
                print(",".join(tup))
    else:
        parser = get_parser(edition)
        print(','.join(headers))
        for url in parser.tested_url:
            soup = get_html_tree_from_url(url)
            for tup in parser.generate_translation_tuples(soup):
                print(','.join(tup))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--zim', '-z', help='use zim file instead of html')
    parser.add_argument('--edition', '-e', help='explicitly specify the language edition, for either html or zim')

    args = parser.parse_args()
    if args.zim:
        test_zim(args.zim, args.edition)
    else:
        test_html(args.edition)

if __name__ == '__main__':
    main()
<commit_msg>log any type error and debug info without stopping the program<commit_after>import argparse
import sys
import logging

from parser.helper import infer_edition_from_url, get_html_tree_from_string, get_html_tree_from_url
import importlib

if sys.version_info[0:3] >= (3, 0, 0):  # python 3 (tested)
    from zim.zimpy_p3 import ZimFile
else:  # python 2 (not tested)
    from zim.zimpy_p2 import ZimFile

LOG_FILENAME = "log/parser.log"
logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG)

# key: Wiktionary edition code
# value: parser class (not parser instance)
parsers = {}

headers = ['edition', 'headword', 'head_lang', 'translation', 'trans_lang', 'trans_lang_code', 'pos', 'pronunciation']

# dynamically loading all modules
def import_all_parsers():
    parser_list = ['ja', 'vi', 'tr', 'fr', 'ru', 'uz', 'de', 'az']
    for parser_name in parser_list:
        module_to_import = '.parse_' + parser_name
        module = importlib.import_module(module_to_import, package='parser')
        parsers[parser_name] = getattr(module, parser_name.capitalize() + "Parser")
        # tested_url.extend(module.tested_url)

def get_parser(edition):
    if edition not in parsers:
        module_to_import = '.parse_' + edition
        try:
            module = importlib.import_module(module_to_import, package='parser')
            parsers[edition] = getattr(module, edition.capitalize() + "Parser")
        except Exception as e:
            print(e)
            return None
    # instantiate the class
    return parsers[edition]()

def read_zim_file(file):
    # print(file.metadata())
    # we only need main articles. They are in namespace 'A'.
    namespace = b'A'
    for article in file.articles():
        if article['namespace'] != namespace:
            continue
        body = file.get_article_by_index(
            article['index'], follow_redirect=False)[0]
        if not body:
            continue
        else:
            yield (body.decode('utf-8'))

def test_zim(filename, edition=None):
    file = ZimFile(filename=filename)
    # file.list_articles_by_url()
    edition_lang_code = file.metadata()['language'].decode('utf-8')
    # print(edition_lang_code)

    if edition:
        edition_wikt_code = edition
        # print(edition_wikt_code)
    else:
        import parser.lang_code_conversion as languages
        edition_wikt_code = languages.get_wikt_code_from_iso639_3(edition_lang_code)
        # print(edition_wikt_code)

    print(','.join(headers))
    # get the parser class
    parser = get_parser(edition_wikt_code)
    if parser is None:
        print("We don't have a parser for {}/{} language yet.".format(edition_lang_code, edition_wikt_code))
        return

    # instantiate the parser
    page_generator = read_zim_file(file)
    for page in page_generator:
        soup = get_html_tree_from_string(page)
        for tup in parser.generate_translation_tuples(soup):
            try:
                print(','.join(tup))
            except TypeError as e:
                logging.debug(e)
                logging.debug(tup)
                logging.debug(soup)
                continue

def test_html(edition=None):
    if edition is None:
        import_all_parsers()
        print(','.join(headers))
        urls_lists = [parser.tested_url for parser in parsers.values()]
        test_urls = [url for sub_list in urls_lists for url in sub_list]
        for url in test_urls:
            edition = infer_edition_from_url(url)
            soup = get_html_tree_from_url(url)
            parser = get_parser(edition)
            for tup in parser.generate_translation_tuples(soup):
                print(",".join(tup))
    else:
        parser = get_parser(edition)
        print(','.join(headers))
        for url in parser.tested_url:
            soup = get_html_tree_from_url(url)
            for tup in parser.generate_translation_tuples(soup):
                print(','.join(tup))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--zim', '-z', help='use zim file instead of html')
    parser.add_argument('--edition', '-e', help='explicitly specify the language edition, for either html or zim')

    args = parser.parse_args()
    if args.zim:
        test_zim(args.zim, args.edition)
    else:
        test_html(args.edition)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#encoding:utf-8

from utils import (get_url, download_file, telegram_autoplay_limit,
                    just_send_an_album)

subreddit = 'behindthegifs'
# t_channel = '@r_behindthegifs'
t_channel = '@r_channels_test'

def send_post(submission, bot):
    what, story = get_url(submission)
    if what != 'album':
        return False

    title = submission.title
    url = submission.url
    link = submission.short_link
    text = '{}\n{}\n\n{}'.format(title, url, link)
    bot.sendMessage(t_channel, text)

    just_send_an_album(t_channel, story, bot)
    return True
<commit_msg>TEST CHANNEL FIXED.<commit_after>#encoding:utf-8

from utils import (get_url, download_file, telegram_autoplay_limit,
                    just_send_an_album)

subreddit = 'behindthegifs'
t_channel = '@r_behindthegifs'

def send_post(submission, bot):
    what, story = get_url(submission)
    if what != 'album':
        return False

    title = submission.title
    url = submission.url
    link = submission.short_link
    text = '{}\n{}\n\n{}'.format(title, url, link)
    bot.sendMessage(t_channel, text)

    just_send_an_album(t_channel, story, bot)
    return True
<|endoftext|>


<commit_before>from django.db import models

class Member(models.Model):
    first_name = models.CharField(max_length=50)
    surname =  models.CharField(max_length=50)
    address = models.CharField(max_length =150, blank=True)
    email = models.EmailField(max_length=100)

<commit_msg>Added function for displaying members<commit_after>from django.db import models

class Member(models.Model):
    first_name = models.CharField(max_length=50)
    surname =  models.CharField(max_length=50)
    address = models.CharField(max_length =150, blank=True)
    email = models.EmailField(max_length=100)

    def __str__(self):
        return self.first_name + ' - ' + self.surname
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ts=2 sw=2 et ai
from compiler.pyassem import CONV

###############################################################################
# Copyright (c) 2012,2013,2014 Andreas Vogel andreas@wellenvogel.net
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included
#  in all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
#  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.
###############################################################################

import os
import struct
import sys
import ctypes
import threading
import sqlite3
import logging
import create_gemf

logger=logging.getLogger(__name__)

def mbtiles_connect(mbtiles_file):
    try:
        con = sqlite3.connect(mbtiles_file)
        return con
    except Exception as e:
        logger.error("Could not connect to database")
        logger.exception(e)
        raise e

#currently we are not prepared for any change of the mbtiles data
#between importing the header and importing the data
#mbtiles follow the tms spec - so we have to compute the gemf y by 2^^z-1-y
def mb2gemf(tile,scheme="tms"):
  if scheme.lower() == "tms":
    return (tile[0],tile[1],pow(2,tile[0])-1-tile[2])
  else:  #scheme.lower() == "xyz":
    return (tile[0],tile[1],tile[2])

def convertMbTiles(nameOut,namesIn):
  gemf=create_gemf.GemfWriter(nameOut);
  source="default"
  for nameIn in namesIn:
    con=mbtiles_connect(nameIn)
    logger.info("importing mbtiles (header) from %s to %s",nameIn,nameOut)
    #write header data
    tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
    t = tiles.fetchone()
    while t:
        z = t[0]
        x = t[1]
        y = t[2]
        logger.debug("tile z=%d,x=%d,y=%d",z,x,y)
        ts=set([mb2gemf((z,x,y)),])
        gemf.addTileSet(source,ts)
        t=tiles.fetchone()
    con.close()
  logger.info("creating gemf header for %s",nameOut)
  gemf.finishHeader()
  logger.info("gemf has %d ranges",len(gemf.ranges))
  for nameIn in namesIn:
    con=mbtiles_connect(nameIn)
    logger.info("importing mbtiles (data) from %s to %s",nameIn,nameOut)
    tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
    t = tiles.fetchone()
    while t:
        z = t[0]
        x = t[1]
        y = t[2]
        data=t[3]
        logger.debug("tile data z=%d,x=%d,y=%d",z,x,y)
        gemf.addTile(source,mb2gemf((z,x,y)),data)
        t=tiles.fetchone()
    con.close()
  logger.info("closing gemf file %s",nameOut)
  gemf.closeFile()

if __name__ == "__main__":
  if len(sys.argv) < 3:
    raise Exception("missing parameter, usage: %s gemfname mbname [mbname]..."%(sys.argv[0]))
  convertMbTiles(sys.argv[1],sys.argv[2:])

<commit_msg>Fix Mbtiles schema<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ts=2 sw=2 et ai
from compiler.pyassem import CONV

###############################################################################
# Copyright (c) 2012,2013,2014 Andreas Vogel andreas@wellenvogel.net
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included
#  in all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
#  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.
###############################################################################

import os
import struct
import sys
import ctypes
import threading
import sqlite3
import logging
import create_gemf

logger=logging.getLogger(__name__)

def mbtiles_connect(mbtiles_file):
    try:
        con = sqlite3.connect(mbtiles_file)
        return con
    except Exception as e:
        logger.error("Could not connect to database")
        logger.exception(e)
        raise e

#currently we are not prepared for any change of the mbtiles data
#between importing the header and importing the data
#mbtiles follow the tms spec - so we have to compute the gemf y by 2^^z-1-y
def mb2gemf(tile,scheme="xyz"):
  if scheme.lower() == "xyz":
    return (tile[0],tile[1],pow(2,tile[0])-1-tile[2])
  else:  #scheme.lower() == "tms":
    return (tile[0],tile[1],tile[2])

def convertMbTiles(nameOut,namesIn):
  gemf=create_gemf.GemfWriter(nameOut);
  source="default"
  for nameIn in namesIn:
    con=mbtiles_connect(nameIn)
    logger.info("importing mbtiles (header) from %s to %s",nameIn,nameOut)
    #write header data
    tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
    t = tiles.fetchone()
    while t:
        z = t[0]
        x = t[1]
        y = t[2]
        logger.debug("tile z=%d,x=%d,y=%d",z,x,y)
        ts=set([mb2gemf((z,x,y)),])
        gemf.addTileSet(source,ts)
        t=tiles.fetchone()
    con.close()
  logger.info("creating gemf header for %s",nameOut)
  gemf.finishHeader()
  logger.info("gemf has %d ranges",len(gemf.ranges))
  for nameIn in namesIn:
    con=mbtiles_connect(nameIn)
    logger.info("importing mbtiles (data) from %s to %s",nameIn,nameOut)
    tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
    t = tiles.fetchone()
    while t:
        z = t[0]
        x = t[1]
        y = t[2]
        data=t[3]
        logger.debug("tile data z=%d,x=%d,y=%d",z,x,y)
        gemf.addTile(source,mb2gemf((z,x,y)),data)
        t=tiles.fetchone()
    con.close()
  logger.info("closing gemf file %s",nameOut)
  gemf.closeFile()

if __name__ == "__main__":
  if len(sys.argv) < 3:
    raise Exception("missing parameter, usage: %s gemfname mbname [mbname]..."%(sys.argv[0]))
  convertMbTiles(sys.argv[1],sys.argv[2:])

<|endoftext|>


<commit_before>__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer',
           'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile',
           'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert',
           'WorkerThread', 'ThreadPool', 'SSLAdapter',
           'CherryPyWSGIServer',
           'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0',
           'WSGIPathInfoDispatcher', 'get_ssl_adapter_class']

import sys
if sys.version_info < (3, 0):
    from wsgiserver2 import *
else:
    # Le sigh. Boo for backward-incompatible syntax.
    exec('from .wsgiserver3 import *')
<commit_msg>Use uniform syntax for wsgiserver imports<commit_after>__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer',
           'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile',
           'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert',
           'WorkerThread', 'ThreadPool', 'SSLAdapter',
           'CherryPyWSGIServer',
           'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0',
           'WSGIPathInfoDispatcher', 'get_ssl_adapter_class']

import sys
if sys.version_info < (3, 0):
    from .wsgiserver2 import *
else:
    from .wsgiserver3 import *
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Anomaly-related algorithms."""

import numpy

from nupic.algorithms.anomaly_likelihood import AnomalyLikelihood
from nupic.research.TP import TP

def pureAnomaly(activeColumns, prevPredictedColumns):
    """the pure anomaly score

    computed as diff of current active columns and columns predicted from previous round
    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step
    @return anomaly score 0..1 (float)
    """
    nActiveColumns = len(activeColumns)
    if nActiveColumns > 0:
      # Test whether each element of a 1-D array is also present in a second
      # array. Sum to get the total # of columns that are active and were
      # predicted.
      score = numpy.sum(numpy.in1d(activeColumns, prevPredictedColumns))
      # Get the percent of active columns that were NOT predicted, that is
      # our anomaly score.
      score = (nActiveColumns - score) / float(nActiveColumns)
    elif len(prevPredictedColumns) > 0:
      # There were predicted columns but none active.
      score = 1.0
    else:
      # There were no predicted or active columns.
      score = 0.0
    return score

class Anomaly(object):
  """basic class that computes anomaly"""

  # anomaly modes supported
  MODE_PURE = "pure"
  MODE_LIKELIHOOD = "likelihood"
  MODE_WEIGHTED = "weighted"
  _supportedModes = [MODE_PURE, MODE_LIKELIHOOD, MODE_WEIGHTED]

  def __init__(self, useTP = None, slidingWindowSize = None, anomalyMode="pure"):
    """
    @param (optional) useTP -- tp temporal pooler instance used
    @param (optional) slidingWindowSize -- enables moving average on final
            anomaly score; how many elements are summed up, sliding window size; int >= 0
    @param (optional) anomalyMode -- (string) which way to use to compute anomaly;
                      possible values are:
                           -- "pure" -- the default, how much anomal the value is;
                                        float 0..1 where 1=totally unexpected
                           -- "likelihood" -- uses the anomaly_likelihood code;
                                        models probability of receiving this value and anomalyScore;
                                        used in Grok
                           -- "weighted" -- "pure" anomaly weighted by "likelihood" (anomaly * likelihood)
    """
#    self.pureAnomaly = staticmethod(pureAnomaly)

    # using TP
    if useTP is not None and isinstance(useTP, TP):
      self._tp = useTP
      self._prevPredictedColumns = numpy.array([])
    else:
      raise Exception("Anomaly: you've provided instance of TP, but it does not look as a correct temporal pooler object: "+str(type(useTP)))

    # using cumulative anomaly , sliding window
    if slidingWindowSize > 0:
      self._windowSize = slidingWindowSize
      self._buf = numpy.array([0] * self._windowSize, dtype=numpy.float) #sliding window buffer
      self._i = 0 # index pointer to actual position
    else:
      raise Exception("Anomaly: if you define slidingWindowSize, it has to be an integer > 0; slidingWindowSize="+str(slidingWindowSize))

    # mode
    self._mode = anomalyMode
    if self._mode == Anomaly.MODE_LIKELIHOOD:
      self._likelihood = AnomalyLikelihood() # probabilistic anomaly
    if not (self._mode in _supportedModes):
      raise ValueError("Invalid anomaly mode; only supported modes are: \"pure\",
                       \"likelihood\", \"weighted\"; you used:"+self._mode)

  def computeAnomalyScore(self, activeColumns, prevPredictedColumns, value=None, timestamp=None):
    """Compute the anomaly score as the percent of active columns not predicted.

    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step (ignored with useTP != None)
    @param value: (optional) input value, that is what activeColumns represent; used in anomaly-likelihood
    @param timestamp: (optional) date timestamp when the sample occured; used in anomaly-likelihood
    @return the computed anomaly score; float 0..1
    """

    # using TP provided during init, _prevPredColumns stored internally here
    if hasattr(self._tp):
      prevPredictedColumns = self._prevPredictedColumns # override the values passed by parameter with the stored value
      self._prevPredictedColumns = self._tp._getTPRegion().getOutputData("topDownOut").nonzero()[0]

    # 1. here is the 'classic' anomaly score
    anomalyScore = pureAnomaly(activeColumns, prevPredictedColumns)

    # compute final anomaly based on selected mode
    if self._mode == Anomaly.MODE_PURE:
      score = anomalyScore
    elif self._mode == Anomaly.MODE_LIKELIHOOD:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = probability
    elif self._mode == Anomaly.MODE_WEIGHTED:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = anomalyScore * probability

    # last, do moving-average if windowSize is set
    if hasattr(self._windowSize):
      score = self._movingAverage(score)

    return score

  def _movingAverage(self, newElement=None):
    """moving average

    @param newValue (optional) add a new element before computing the avg
    @return moving average of self._windowSize last elements
    """
    if newElement is not None:
      self._buf[self._i]= newElement
      self._i = (self._i + 1) % self._windowSize
    return self._buf.sum()/float(self._windowSize) # normalize to 0..1

<commit_msg>fix typo<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Anomaly-related algorithms."""

import numpy

from nupic.algorithms.anomaly_likelihood import AnomalyLikelihood
from nupic.research.TP import TP

def pureAnomaly(activeColumns, prevPredictedColumns):
    """the pure anomaly score

    computed as diff of current active columns and columns predicted from previous round
    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step
    @return anomaly score 0..1 (float)
    """
    nActiveColumns = len(activeColumns)
    if nActiveColumns > 0:
      # Test whether each element of a 1-D array is also present in a second
      # array. Sum to get the total # of columns that are active and were
      # predicted.
      score = numpy.sum(numpy.in1d(activeColumns, prevPredictedColumns))
      # Get the percent of active columns that were NOT predicted, that is
      # our anomaly score.
      score = (nActiveColumns - score) / float(nActiveColumns)
    elif len(prevPredictedColumns) > 0:
      # There were predicted columns but none active.
      score = 1.0
    else:
      # There were no predicted or active columns.
      score = 0.0
    return score

class Anomaly(object):
  """basic class that computes anomaly"""

  # anomaly modes supported
  MODE_PURE = "pure"
  MODE_LIKELIHOOD = "likelihood"
  MODE_WEIGHTED = "weighted"
  _supportedModes = [MODE_PURE, MODE_LIKELIHOOD, MODE_WEIGHTED]

  def __init__(self, useTP = None, slidingWindowSize = None, anomalyMode="pure"):
    """
    @param (optional) useTP -- tp temporal pooler instance used
    @param (optional) slidingWindowSize -- enables moving average on final
            anomaly score; how many elements are summed up, sliding window size; int >= 0
    @param (optional) anomalyMode -- (string) which way to use to compute anomaly;
                      possible values are:
                           -- "pure" -- the default, how much anomal the value is;
                                        float 0..1 where 1=totally unexpected
                           -- "likelihood" -- uses the anomaly_likelihood code;
                                        models probability of receiving this value and anomalyScore;
                                        used in Grok
                           -- "weighted" -- "pure" anomaly weighted by "likelihood" (anomaly * likelihood)
    """
#    self.pureAnomaly = staticmethod(pureAnomaly)

    # using TP
    if useTP is not None and isinstance(useTP, TP):
      self._tp = useTP
      self._prevPredictedColumns = numpy.array([])
    else:
      raise Exception("Anomaly: you've provided instance of TP, but it does not look as a correct temporal pooler object: "+str(type(useTP)))

    # using cumulative anomaly , sliding window
    if slidingWindowSize > 0:
      self._windowSize = slidingWindowSize
      self._buf = numpy.array([0] * self._windowSize, dtype=numpy.float) #sliding window buffer
      self._i = 0 # index pointer to actual position
    else:
      raise Exception("Anomaly: if you define slidingWindowSize, it has to be an integer > 0; slidingWindowSize="+str(slidingWindowSize))

    # mode
    self._mode = anomalyMode
    if self._mode == Anomaly.MODE_LIKELIHOOD:
      self._likelihood = AnomalyLikelihood() # probabilistic anomaly
    if not (self._mode in _supportedModes):
      raise ValueError("Invalid anomaly mode; only supported modes are: \"pure\",\
                       \"likelihood\", \"weighted\"; you used:"+self._mode)

  def computeAnomalyScore(self, activeColumns, prevPredictedColumns, value=None, timestamp=None):
    """Compute the anomaly score as the percent of active columns not predicted.

    @param activeColumns: array of active column indices
    @param prevPredictedColumns: array of columns indices predicted in previous step (ignored with useTP != None)
    @param value: (optional) input value, that is what activeColumns represent; used in anomaly-likelihood
    @param timestamp: (optional) date timestamp when the sample occured; used in anomaly-likelihood
    @return the computed anomaly score; float 0..1
    """

    # using TP provided during init, _prevPredColumns stored internally here
    if hasattr(self._tp):
      prevPredictedColumns = self._prevPredictedColumns # override the values passed by parameter with the stored value
      self._prevPredictedColumns = self._tp._getTPRegion().getOutputData("topDownOut").nonzero()[0]

    # 1. here is the 'classic' anomaly score
    anomalyScore = pureAnomaly(activeColumns, prevPredictedColumns)

    # compute final anomaly based on selected mode
    if self._mode == Anomaly.MODE_PURE:
      score = anomalyScore
    elif self._mode == Anomaly.MODE_LIKELIHOOD:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = probability
    elif self._mode == Anomaly.MODE_WEIGHTED:
      probability = self._likelihood.anomalyProbability(value, anomalyScore, timestamp)
      score = anomalyScore * probability

    # last, do moving-average if windowSize is set
    if hasattr(self._windowSize):
      score = self._movingAverage(score)

    return score

  def _movingAverage(self, newElement=None):
    """moving average

    @param newValue (optional) add a new element before computing the avg
    @return moving average of self._windowSize last elements
    """
    if newElement is not None:
      self._buf[self._i]= newElement
      self._i = (self._i + 1) % self._windowSize
    return self._buf.sum()/float(self._windowSize) # normalize to 0..1

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# datasets.py
#
# purpose:  Functions to handle datasets.
# author:   Filipe P. A. Fernandes
# e-mail:   ocefpaf@gmail
# web:      http://ocefpaf.tiddlyspot.com/
# created:  09-Sep-2011
# modified: Wed 12 Sep 2012 12:00:36 PM BRT
#
# obs: some Functions were based on:
# http://www.trondkristiansen.com/?page_id=1071

from __future__ import division

import numpy as np

from netCDF4 import Dataset

from oceans.ff_tools import get_profile

__all__ = [
    'woa_subset',
    'etopo_subset',
    'laplace_X',
    'laplace_Y',
    'laplace_filter',
    'get_depth'
]

def get_indices(min_lat, max_lat, min_lon, max_lon, lons, lats):
    r"""Return the data indices for a lon, lat square."""

    distances1, distances2, indices = [], [], []
    index = 1
    for point in lats:
        s1 = max_lat - point
        s2 = min_lat - point
        distances1.append((np.dot(s1, s1), point, index))
        distances2.append((np.dot(s2, s2), point, index - 1))
        index = index + 1

    distances1.sort()
    distances2.sort()
    indices.append(distances1[0])
    indices.append(distances2[0])

    distances1, distances2, indices = [], [], []
    index = 1
    for point in lons:
        s1 = max_lon - point
        s2 = min_lon - point
        distances1.append((np.dot(s1, s1), point, index))
        distances2.append((np.dot(s2, s2), point, index - 1))
        index = index + 1

    distances1.sort()
    distances2.sort()
    indices.append(distances1[0])
    indices.append(distances2[0])

    # max_lat_indices, min_lat_indices, max_lon_indices, min_lon_indices.
    res = np.zeros((4), dtype=np.float64)
    res[0] = indices[3][2]
    res[1] = indices[2][2]
    res[2] = indices[1][2]
    res[3] = indices[0][2]
    return res

def woa_subset(min_lat, max_lat, min_lon, max_lon, woa_file=None):
    r"""Get a World Ocean Atlas subset.
    Should work on any netCDF with x, y, data.
    `woa_file` can be an OpenDap url."""

    if not woa_file:
        # TODO: Check for a online dap version of this file.
        tfile = '/home/filipe/00-NOBKP/OcFisData/ETOPO2v2g_f4.nc'

    woa = Dataset(tfile, 'r')

    lons = woa.variables["x"][:]
    lats = woa.variables["y"][:]

    res = get_indices(min_lat, max_lat, min_lon, max_lon, lons, lats)

    lon, lat = np.meshgrid(lons[res[0]:res[1]], lats[res[2]:res[3]])

    bathy = woa.variables["temperature"][int(res[2]):int(res[3]),
                                         int(res[0]):int(res[1])]

    return lon, lat, temperature

def etopo_subset(min_lat, max_lat, min_lon, max_lon, tfile=None, smoo=False):
    r"""Get a etopo subset.
    Should work on any netCDF with x, y, data
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2011/07/contourICEMaps.py"""

    if not tfile:
        # TODO: Check for a online dap version of this file.
        tfile = '/home/filipe/00-NOBKP/OcFisData/ETOPO2v2g_f4.nc'

    etopo = Dataset(tfile, 'r')

    lons = etopo.variables["x"][:]
    lats = etopo.variables["y"][:]

    res = get_indices(min_lat, max_lat, min_lon, max_lon, lons, lats)

    lon, lat = np.meshgrid(lons[res[0]:res[1]], lats[res[2]:res[3]])

    bathy = etopo.variables["z"][int(res[2]):int(res[3]),
                                 int(res[0]):int(res[1])]

    if smoo == 'True':
        bathy = laplace_filter(bathy, M=None)

    return lon, lat, bathy

def laplace_X(F, M):
    r"""1D Laplace Filter in X-direction (axis=1)
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2010/09/laplaceFilter.py"""

    jmax, imax = F.shape

    # Add strips of land
    F2 = np.zeros((jmax, imax + 2), dtype=F.dtype)
    F2[:, 1:-1] = F
    M2 = np.zeros((jmax, imax + 2), dtype=M.dtype)
    M2[:, 1:-1] = M

    MS = M2[:, 2:] + M2[:, :-2]
    FS = F2[:, 2:] * M2[:, 2:] + F2[:, :-2] * M2[:, :-2]

    return np.where(M > 0.5, (1 - 0.25 * MS) * F + 0.25 * FS, F)

def laplace_Y(F, M):
    r"""1D Laplace Filter in Y-direction (axis=1)
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2010/09/laplaceFilter.py"""

    jmax, imax = F.shape

    # Add strips of land
    F2 = np.zeros((jmax + 2, imax), dtype=F.dtype)
    F2[1:-1, :] = F
    M2 = np.zeros((jmax + 2, imax), dtype=M.dtype)
    M2[1:-1, :] = M

    MS = M2[2:, :] + M2[:-2, :]
    FS = F2[2:, :] * M2[2:, :] + F2[:-2, :] * M2[:-2, :]

    return np.where(M > 0.5, (1 - 0.25 * MS) * F + 0.25 * FS, F)

def laplace_filter(F, M=None):
    r"""Laplace filter a 2D field with mask.  The mask may cause laplace_X and
    laplace_Y to not commute. Take average of both directions.
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2010/09/laplaceFilter.py"""

    if not M:
        M = np.ones_like(F)

    return 0.5 * (laplace_X(laplace_Y(F, M), M) +
                  laplace_Y(laplace_X(F, M), M))

def get_depth(lon, lat):
    r"""Find the depths for each station on the etopo2 database."""
    lons, lats, bathy = etopo_subset(lat.min() - 5, lat.max() + 5,
                                     lon.min() - 5, lon.max() + 5, smoo=False)

    return get_profile(lons, lats, bathy, lon, lat, mode='nearest', order=3)
<commit_msg>Fixed bug in `etopo_subset`.<commit_after># -*- coding: utf-8 -*-
#
# datasets.py
#
# purpose:  Functions to handle datasets.
# author:   Filipe P. A. Fernandes
# e-mail:   ocefpaf@gmail
# web:      http://ocefpaf.tiddlyspot.com/
# created:  09-Sep-2011
# modified: Wed 12 Sep 2012 12:00:36 PM BRT
#
# obs: some Functions were based on:
# http://www.trondkristiansen.com/?page_id=1071

from __future__ import division

import numpy as np

from netCDF4 import Dataset

from oceans.ff_tools import get_profile

__all__ = [
    'woa_subset',
    'etopo_subset',
    'laplace_X',
    'laplace_Y',
    'laplace_filter',
    'get_depth'
]

def get_indices(min_lat, max_lat, min_lon, max_lon, lons, lats):
    r"""Return the data indices for a lon, lat square."""

    distances1, distances2, indices = [], [], []
    index = 1
    for point in lats:
        s1 = max_lat - point
        s2 = min_lat - point
        distances1.append((np.dot(s1, s1), point, index))
        distances2.append((np.dot(s2, s2), point, index - 1))
        index = index + 1

    distances1.sort()
    distances2.sort()
    indices.append(distances1[0])
    indices.append(distances2[0])

    distances1, distances2 = [], []
    index = 1
    for point in lons:
        s1 = max_lon - point
        s2 = min_lon - point
        distances1.append((np.dot(s1, s1), point, index))
        distances2.append((np.dot(s2, s2), point, index - 1))
        index = index + 1

    distances1.sort()
    distances2.sort()
    indices.append(distances1[0])
    indices.append(distances2[0])

    # max_lat_indices, min_lat_indices, max_lon_indices, min_lon_indices.
    res = np.zeros((4), dtype=np.float64)
    res[0] = indices[3][2]
    res[1] = indices[2][2]
    res[2] = indices[1][2]
    res[3] = indices[0][2]
    return res

def woa_subset(min_lat, max_lat, min_lon, max_lon, woa_file=None):
    r"""Get a World Ocean Atlas subset.
    Should work on any netCDF with x, y, data.
    `woa_file` can be an OpenDap url."""

    if not woa_file:
        # TODO: Check for a online dap version of this file.
        woa_file = None

    woa = Dataset(woa_file, 'r')

    lons = woa.variables["x"][:]
    lats = woa.variables["y"][:]

    res = get_indices(min_lat, max_lat, min_lon, max_lon, lons, lats)

    lon, lat = np.meshgrid(lons[res[0]:res[1]], lats[res[2]:res[3]])

    bathy = woa.variables["temperature"][int(res[2]):int(res[3]),
                                         int(res[0]):int(res[1])]

    return lon, lat, temperature

def etopo_subset(min_lat, max_lat, min_lon, max_lon, tfile=None, smoo=False):
    r"""Get a etopo subset.
    Should work on any netCDF with x, y, data
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2011/07/contourICEMaps.py

    Example
    -------
    >>> offset = 5
    >>> lonStart, lonEnd, latStart, latEnd = -43, -30.0, -22.0, -17.0
    >>> lons, lats, bathy = etopo_subset(latStart - offset, latEnd + offset,
    ...                                  lonStart - offset, lonEnd + offset, smoo=True)
    """

    if not tfile:
        # TODO: Check for a online dap version of this file.
        tfile = '/home/filipe/00-NOBKP/OcFisData/ETOPO2v2g_f4.nc'

    etopo = Dataset(tfile, 'r')

    lons = etopo.variables["x"][:]
    lats = etopo.variables["y"][:]

    res = get_indices(min_lat, max_lat, min_lon, max_lon, lons, lats)

    lon, lat = np.meshgrid(lons[res[0]:res[1]], lats[res[2]:res[3]])

    bathy = etopo.variables["z"][int(res[2]):int(res[3]),
                                 int(res[0]):int(res[1])]

    if smoo == 'True':
        bathy = laplace_filter(bathy, M=None)

    return lon, lat, bathy

def laplace_X(F, M):
    r"""1D Laplace Filter in X-direction (axis=1)
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2010/09/laplaceFilter.py"""

    jmax, imax = F.shape

    # Add strips of land
    F2 = np.zeros((jmax, imax + 2), dtype=F.dtype)
    F2[:, 1:-1] = F
    M2 = np.zeros((jmax, imax + 2), dtype=M.dtype)
    M2[:, 1:-1] = M

    MS = M2[:, 2:] + M2[:, :-2]
    FS = F2[:, 2:] * M2[:, 2:] + F2[:, :-2] * M2[:, :-2]

    return np.where(M > 0.5, (1 - 0.25 * MS) * F + 0.25 * FS, F)

def laplace_Y(F, M):
    r"""1D Laplace Filter in Y-direction (axis=1)
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2010/09/laplaceFilter.py"""

    jmax, imax = F.shape

    # Add strips of land
    F2 = np.zeros((jmax + 2, imax), dtype=F.dtype)
    F2[1:-1, :] = F
    M2 = np.zeros((jmax + 2, imax), dtype=M.dtype)
    M2[1:-1, :] = M

    MS = M2[2:, :] + M2[:-2, :]
    FS = F2[2:, :] * M2[2:, :] + F2[:-2, :] * M2[:-2, :]

    return np.where(M > 0.5, (1 - 0.25 * MS) * F + 0.25 * FS, F)

def laplace_filter(F, M=None):
    r"""Laplace filter a 2D field with mask.  The mask may cause laplace_X and
    laplace_Y to not commute. Take average of both directions.
    http://www.trondkristiansen.com/wp-content/uploads/downloads/
    2010/09/laplaceFilter.py"""

    if not M:
        M = np.ones_like(F)

    return 0.5 * (laplace_X(laplace_Y(F, M), M) +
                  laplace_Y(laplace_X(F, M), M))

def get_depth(lon, lat):
    r"""Find the depths for each station on the etopo2 database."""
    lons, lats, bathy = etopo_subset(lat.min() - 5, lat.max() + 5,
                                     lon.min() - 5, lon.max() + 5, smoo=False)

    return get_profile(lons, lats, bathy, lon, lat, mode='nearest', order=3)
<|endoftext|>


<commit_before>import matplotlib
import matplotlib.pyplot as plt

def pdplot(pddata, kind='bar', xlabel=None, ylabel=None, title=None, **kwargs):
    ax = pddata.plot(kind=kind, **kwargs)
    plt.axhline(0, color='k')
    if xlabel:
        ax.set_xlabel(xlabel)
    if ylabel:
        ax.set_ylabel(ylabel)
    if title:
        ax.set_title(title, fontsize="xx-large", y=1.04)

<commit_msg>Return the axis, will ya?  I want it back!<commit_after>import matplotlib
import matplotlib.pyplot as plt

def pdplot(pddata, kind='bar', xlabel=None, ylabel=None, title=None, **kwargs):
    ax = pddata.plot(kind=kind, **kwargs)
    plt.axhline(0, color='k')
    if xlabel:
        ax.set_xlabel(xlabel)
    if ylabel:
        ax.set_ylabel(ylabel)
    if title:
        ax.set_title(title, fontsize="xx-large", y=1.04)
    return ax<|endoftext|>


<commit_before># coding: utf8

from ckan.logic import side_effect_free

from logging import getLogger
log = getLogger(__name__)

@side_effect_free
def context_user_show(context, data_dict):
    user = envidat_get_user_from_context(context)
    return {'user': user}

def envidat_get_user_from_context(context):
    auth_user_obj = context.get('auth_user_obj', None)

    if auth_user_obj:
        return auth_user_obj.as_dict()
    else:
        return {}
<commit_msg>user_show reutrns empty dict when no user<commit_after># coding: utf8

from ckan.logic import side_effect_free

from logging import getLogger
log = getLogger(__name__)

@side_effect_free
def context_user_show(context, data_dict):
    user = envidat_get_user_from_context(context)
    if user:
        return {'user': user}
    else:
        return {}

def envidat_get_user_from_context(context):
    auth_user_obj = context.get('auth_user_obj', None)

    if auth_user_obj:
        return auth_user_obj.as_dict()
    else:
        return {}
<|endoftext|>


<commit_before>'''OAI-PMH implementation for CKAN datasets and groups.
'''
# pylint: disable=E1101,E1103
import json
import logging

from oaipmh import common
from oaipmh.common import ResumptionOAIPMH
from oaipmh.error import IdDoesNotExistError
from pylons import config
from sqlalchemy import between

from ckan.lib.helpers import url_for
from ckan.logic import get_action
from ckan.model import Package, Session, Group, PackageRevision
from ckanext.dcat.processors import RDFSerializer
from ckanext.kata import helpers
import utils

log = logging.getLogger(__name__)

rdfserializer = RDFSerializer()

class CKANServer(ResumptionOAIPMH):
    '''A OAI-PMH implementation class for CKAN.
    '''
    def identify(self):
        '''Return identification information for this server.
        '''
        return common.Identify(
            repositoryName=config.get('ckan.site_title', 'repository'),
            baseURL=config.get('ckan.site_url', None) + url_for(controller='ckanext.oaipmh.controller:OAIPMHController', action='index'),
            protocolVersion="2.0",
            adminEmails=[config.get('email_to')],
            earliestDatestamp=utils.get_earliest_datestamp(),
            deletedRecord='no',
            granularity='YYYY-MM-DDThh:mm:ssZ',
            compression=['identity'])

    def _get_json_content(self, js):
        '''
        Gets all items from JSON

        :param js: json string
        :return: list of items
        '''

        try:
            json_data = json.loads(js)
            json_titles = list()
            for key, value in json_data.iteritems():
                json_titles.append(value)
            return json_titles
        except:
            return [js]

    def _record_for_dataset_dcat(self, dataset, spec):
        '''Show a tuple of a header and metadata for this dataset.
        Note that dataset_xml (metadata) returned is just a string containing
        ready rdf xml. This is contrary to the common practice of pyoia's
        getRecord method.
        '''
        package = get_action('package_show')({}, {'id': dataset.id})
        dataset_xml = rdfserializer.serialize_dataset(package, _format='xml')
        return (common.Header('', dataset.id, dataset.metadata_created, [spec], False),
                dataset_xml, None)

    def _record_for_dataset(self, dataset, spec):
        '''Show a tuple of a header and metadata for this dataset.
        '''
        package = get_action('package_show')({}, {'id': dataset.id})

        coverage = []
        temporal_begin = package.get('temporal_coverage_begin', '')
        temporal_end = package.get('temporal_coverage_end', '')

        geographic = package.get('geographic_coverage', '')
        if geographic:
            coverage.extend(geographic.split(','))
        if temporal_begin or temporal_end:
            coverage.append("%s/%s" % (temporal_begin, temporal_end))

        pids = [pid.get('id') for pid in package.get('pids', {}) if pid.get('id', False)]
        pids.append(package.get('id'))
        pids.append(config.get('ckan.site_url') + url_for(controller="package", action='read', id=package['name']))

        meta = {'title': self._get_json_content(package.get('title', None) or package.get('name')),
                'creator': [author['name'] for author in helpers.get_authors(package) if 'name' in author],
                'publisher': [agent['name'] for agent in helpers.get_distributors(package) + helpers.get_contacts(package) if 'name' in agent],
                'contributor': [author['name'] for author in helpers.get_contributors(package) if 'name' in author],
                'identifier': pids,
                'type': ['dataset'],
                'language': [l.strip() for l in package.get('language').split(",")] if package.get('language', None) else None,
                'description': self._get_json_content(package.get('notes')) if package.get('notes', None) else None,
                'subject': [tag.get('display_name') for tag in package['tags']] if package.get('tags', None) else None,
                'date': [dataset.metadata_created.strftime('%Y-%m-%d')] if dataset.metadata_created else None,
                'rights': [package['license_title']] if package.get('license_title', None) else None,
                'coverage': coverage if coverage else None, }

        iters = dataset.extras.items()
        meta = dict(iters + meta.items())
        metadata = {}
        # Fixes the bug on having a large dataset being scrambled to individual
        # letters
        for key, value in meta.items():
            if not isinstance(value, list):
                metadata[str(key)] = [value]
            else:
                metadata[str(key)] = value
        return (common.Header('', dataset.id, dataset.metadata_created, [spec], False),
                common.Metadata('', metadata), None)

    def getRecord(self, metadataPrefix, identifier):
        '''Simple getRecord for a dataset.
        '''
        package = Package.get(identifier)
        if not package:
            raise IdDoesNotExistError("No dataset with id %s" % identifier)
        spec = package.name
        if package.owner_org:
            group = Group.get(package.owner_org)
            if group and group.name:
                spec = group.name
        if metadataPrefix == 'oai_dc':
            return self._record_for_dataset(package, spec)
        return self._record_for_dataset_dcat(package, spec)

    def listIdentifiers(self, metadataPrefix=None, set=None, cursor=None,
                        from_=None, until=None, batch_size=None):
        '''List all identifiers for this repository.
        '''
        data = []
        packages = []
        group = None
        if not set:
            if not from_ and not until:
                packages = Session.query(Package).filter(Package.type=='dataset').\
                    filter(Package.private!=True).filter(Package.state=='active').all()
            else:
                if from_ and not until:
                    packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(PackageRevision.revision_timestamp > from_).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if until and not from_:
                    packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(PackageRevision.revision_timestamp < until).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if from_ and until:
                    packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(between(PackageRevision.revision_timestamp, from_, until)).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
        else:
            group = Group.get(set)
            if group:
                packages = group.packages(return_query=True).filter(Package.type=='dataset').\
                    filter(Package.private!=True).filter(Package.state=='active')
                if from_ and not until:
                    packages = packages.filter(PackageRevision.revision_timestamp > from_).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active')
                if until and not from_:
                    packages = packages.filter(PackageRevision.revision_timestamp < until).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active')
                if from_ and until:
                    packages = packages.filter(between(PackageRevision.revision_timestamp, from_, until)).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active')
                packages = packages.all()
        if cursor:
            cursor_end = cursor + batch_size if cursor + batch_size < len(packages) else len(packages)
            packages = packages[cursor:cursor_end]
        for package in packages:
            spec = package.name
            if group:
                spec = group.name
            else:
                if package.owner_org:
                    group = Group.get(package.owner_org)
                    if group and group.name:
                        spec = group.name
                    group = None
            data.append(common.Header('', package.id, package.metadata_created, [spec], False))

        return data

    def listMetadataFormats(self, identifier=None):
        '''List available metadata formats.
        '''
        return [('oai_dc',
                'http://www.openarchives.org/OAI/2.0/oai_dc.xsd',
                'http://www.openarchives.org/OAI/2.0/oai_dc/'),
                ('rdf',
                 'http://www.openarchives.org/OAI/2.0/rdf.xsd',
                 'http://www.openarchives.org/OAI/2.0/rdf/')]

    def listRecords(self, metadataPrefix, set=None, cursor=None, from_=None,
                    until=None, batch_size=None):
        '''Show a selection of records, basically lists all datasets.
        '''
        data = []
        packages = []
        group = None
        if not set:
            if not from_ and not until:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(Package.state=='active').all()
            if from_ and not until:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(PackageRevision.revision_timestamp > from_).filter(Package.name==PackageRevision.name).\
                    filter(Package.state=='active').all()
            if until and not from_:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(PackageRevision.revision_timestamp < until).filter(Package.name==PackageRevision.name).\
                    filter(Package.state=='active').all()
            if from_ and until:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(between(PackageRevision.revision_timestamp, from_, until)).\
                    filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
        else:
            group = Group.get(set)
            if group:
                packages = group.packages(return_query=True)
                if from_ and not until:
                    packages = packages.filter(PackageRevision.revision_timestamp > from_).\
                        filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if until and not from_:
                    packages = packages.filter(PackageRevision.revision_timestamp < until).\
                        filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if from_ and until:
                    packages = packages.filter(between(PackageRevision.revision_timestamp, from_, until)).\
                        filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
        if cursor:
            packages = packages[cursor:]
        for res in packages:
            spec = res.name
            if group:
                spec = group.name
            else:
                if res.owner_org:
                    group = Group.get(res.owner_org)
                    if group and group.name:
                        spec = group.name
                    group = None
            data.append(self._record_for_dataset(res, spec))
        return data

    def listSets(self, cursor=None, batch_size=None):
        '''List all sets in this repository, where sets are groups.
        '''
        data = []
        groups = Session.query(Group).filter(Group.state == 'active')
        if cursor:
            cursor_end = cursor+batch_size if cursor+batch_size < groups.count() else groups.count()
            groups = groups[cursor:cursor_end]
        for dataset in groups:
            data.append((dataset.name, dataset.title, dataset.description))
        return data
<commit_msg>New RDF format for OAI server's "ListRecords" verb.<commit_after>'''OAI-PMH implementation for CKAN datasets and groups.
'''
# pylint: disable=E1101,E1103
import json
import logging

from oaipmh import common
from oaipmh.common import ResumptionOAIPMH
from oaipmh.error import IdDoesNotExistError
from pylons import config
from sqlalchemy import between

from ckan.lib.helpers import url_for
from ckan.logic import get_action
from ckan.model import Package, Session, Group, PackageRevision
from ckanext.dcat.processors import RDFSerializer
from ckanext.kata import helpers
import utils

log = logging.getLogger(__name__)

rdfserializer = RDFSerializer()

class CKANServer(ResumptionOAIPMH):
    '''A OAI-PMH implementation class for CKAN.
    '''
    def identify(self):
        '''Return identification information for this server.
        '''
        return common.Identify(
            repositoryName=config.get('ckan.site_title', 'repository'),
            baseURL=config.get('ckan.site_url', None) + url_for(controller='ckanext.oaipmh.controller:OAIPMHController', action='index'),
            protocolVersion="2.0",
            adminEmails=[config.get('email_to')],
            earliestDatestamp=utils.get_earliest_datestamp(),
            deletedRecord='no',
            granularity='YYYY-MM-DDThh:mm:ssZ',
            compression=['identity'])

    def _get_json_content(self, js):
        '''
        Gets all items from JSON

        :param js: json string
        :return: list of items
        '''

        try:
            json_data = json.loads(js)
            json_titles = list()
            for key, value in json_data.iteritems():
                json_titles.append(value)
            return json_titles
        except:
            return [js]

    def _record_for_dataset_dcat(self, dataset, spec):
        '''Show a tuple of a header and metadata for this dataset.
        Note that dataset_xml (metadata) returned is just a string containing
        ready rdf xml. This is contrary to the common practice of pyoia's
        getRecord method.
        '''
        package = get_action('package_show')({}, {'id': dataset.id})
        dataset_xml = rdfserializer.serialize_dataset(package, _format='xml')
        return (common.Header('', dataset.id, dataset.metadata_created, [spec], False),
                dataset_xml, None)

    def _record_for_dataset(self, dataset, spec):
        '''Show a tuple of a header and metadata for this dataset.
        '''
        package = get_action('package_show')({}, {'id': dataset.id})

        coverage = []
        temporal_begin = package.get('temporal_coverage_begin', '')
        temporal_end = package.get('temporal_coverage_end', '')

        geographic = package.get('geographic_coverage', '')
        if geographic:
            coverage.extend(geographic.split(','))
        if temporal_begin or temporal_end:
            coverage.append("%s/%s" % (temporal_begin, temporal_end))

        pids = [pid.get('id') for pid in package.get('pids', {}) if pid.get('id', False)]
        pids.append(package.get('id'))
        pids.append(config.get('ckan.site_url') + url_for(controller="package", action='read', id=package['name']))

        meta = {'title': self._get_json_content(package.get('title', None) or package.get('name')),
                'creator': [author['name'] for author in helpers.get_authors(package) if 'name' in author],
                'publisher': [agent['name'] for agent in helpers.get_distributors(package) + helpers.get_contacts(package) if 'name' in agent],
                'contributor': [author['name'] for author in helpers.get_contributors(package) if 'name' in author],
                'identifier': pids,
                'type': ['dataset'],
                'language': [l.strip() for l in package.get('language').split(",")] if package.get('language', None) else None,
                'description': self._get_json_content(package.get('notes')) if package.get('notes', None) else None,
                'subject': [tag.get('display_name') for tag in package['tags']] if package.get('tags', None) else None,
                'date': [dataset.metadata_created.strftime('%Y-%m-%d')] if dataset.metadata_created else None,
                'rights': [package['license_title']] if package.get('license_title', None) else None,
                'coverage': coverage if coverage else None, }

        iters = dataset.extras.items()
        meta = dict(iters + meta.items())
        metadata = {}
        # Fixes the bug on having a large dataset being scrambled to individual
        # letters
        for key, value in meta.items():
            if not isinstance(value, list):
                metadata[str(key)] = [value]
            else:
                metadata[str(key)] = value
        return (common.Header('', dataset.id, dataset.metadata_created, [spec], False),
                common.Metadata('', metadata), None)

    def getRecord(self, metadataPrefix, identifier):
        '''Simple getRecord for a dataset.
        '''
        package = Package.get(identifier)
        if not package:
            raise IdDoesNotExistError("No dataset with id %s" % identifier)
        spec = package.name
        if package.owner_org:
            group = Group.get(package.owner_org)
            if group and group.name:
                spec = group.name
        if metadataPrefix == 'oai_dc':
            return self._record_for_dataset(package, spec)
        return self._record_for_dataset_dcat(package, spec)

    def listIdentifiers(self, metadataPrefix=None, set=None, cursor=None,
                        from_=None, until=None, batch_size=None):
        '''List all identifiers for this repository.
        '''
        data = []
        packages = []
        group = None
        if not set:
            if not from_ and not until:
                packages = Session.query(Package).filter(Package.type=='dataset').\
                    filter(Package.private!=True).filter(Package.state=='active').all()
            else:
                if from_ and not until:
                    packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(PackageRevision.revision_timestamp > from_).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if until and not from_:
                    packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(PackageRevision.revision_timestamp < until).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if from_ and until:
                    packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(between(PackageRevision.revision_timestamp, from_, until)).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
        else:
            group = Group.get(set)
            if group:
                packages = group.packages(return_query=True).filter(Package.type=='dataset').\
                    filter(Package.private!=True).filter(Package.state=='active')
                if from_ and not until:
                    packages = packages.filter(PackageRevision.revision_timestamp > from_).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active')
                if until and not from_:
                    packages = packages.filter(PackageRevision.revision_timestamp < until).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active')
                if from_ and until:
                    packages = packages.filter(between(PackageRevision.revision_timestamp, from_, until)).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active')
                packages = packages.all()
        if cursor:
            cursor_end = cursor + batch_size if cursor + batch_size < len(packages) else len(packages)
            packages = packages[cursor:cursor_end]
        for package in packages:
            spec = package.name
            if group:
                spec = group.name
            else:
                if package.owner_org:
                    group = Group.get(package.owner_org)
                    if group and group.name:
                        spec = group.name
                    group = None
            data.append(common.Header('', package.id, package.metadata_created, [spec], False))

        return data

    def listMetadataFormats(self, identifier=None):
        '''List available metadata formats.
        '''
        return [('oai_dc',
                'http://www.openarchives.org/OAI/2.0/oai_dc.xsd',
                'http://www.openarchives.org/OAI/2.0/oai_dc/'),
                ('rdf',
                 'http://www.openarchives.org/OAI/2.0/rdf.xsd',
                 'http://www.openarchives.org/OAI/2.0/rdf/')]

    def listRecords(self, metadataPrefix, set=None, cursor=None, from_=None,
                    until=None, batch_size=None):
        '''Show a selection of records, basically lists all datasets.
        '''
        data = []
        packages = []
        group = None
        if not set:
            if not from_ and not until:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(Package.state=='active').all()
            if from_ and not until:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(PackageRevision.revision_timestamp > from_).filter(Package.name==PackageRevision.name).\
                    filter(Package.state=='active').all()
            if until and not from_:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(PackageRevision.revision_timestamp < until).filter(Package.name==PackageRevision.name).\
                    filter(Package.state=='active').all()
            if from_ and until:
                packages = Session.query(Package).filter(Package.type=='dataset').filter(Package.private!=True).\
                    filter(between(PackageRevision.revision_timestamp, from_, until)).\
                    filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
        else:
            group = Group.get(set)
            if group:
                packages = group.packages(return_query=True)
                if from_ and not until:
                    packages = packages.filter(PackageRevision.revision_timestamp > from_).\
                        filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if until and not from_:
                    packages = packages.filter(PackageRevision.revision_timestamp < until).\
                        filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
                if from_ and until:
                    packages = packages.filter(between(PackageRevision.revision_timestamp, from_, until)).\
                        filter(Package.type=='dataset').filter(Package.private!=True).\
                        filter(Package.name==PackageRevision.name).filter(Package.state=='active').all()
        if cursor:
            packages = packages[cursor:]
        for res in packages:
            spec = res.name
            if group:
                spec = group.name
            else:
                if res.owner_org:
                    group = Group.get(res.owner_org)
                    if group and group.name:
                        spec = group.name
                    group = None
            if metadataPrefix == 'oai_dc':
                data.append(self._record_for_dataset(res, spec))
            data.append(self._record_for_dataset_dcat(res, spec))
        return data

    def listSets(self, cursor=None, batch_size=None):
        '''List all sets in this repository, where sets are groups.
        '''
        data = []
        groups = Session.query(Group).filter(Group.state == 'active')
        if cursor:
            cursor_end = cursor+batch_size if cursor+batch_size < groups.count() else groups.count()
            groups = groups[cursor:cursor_end]
        for dataset in groups:
            data.append((dataset.name, dataset.title, dataset.description))
        return data
<|endoftext|>


<commit_before>import logging
import ckanapi
import os

from pylons import config

import ckan.plugins.toolkit as toolkit
from ckan.lib.celery_app import celery
from ckan.lib.helpers import get_pkg_dict_extra
from ckanext.syndicate.plugin import (
    get_syndicate_flag,
    get_syndicated_id,
    get_syndicated_name_prefix,
    get_syndicated_organization,
)

logger = logging.getLogger(__name__)

@celery.task(name='syndicate.sync_package')
def sync_package_task(package, action, ckan_ini_filepath):
    logger = sync_package_task.get_logger()
    load_config(ckan_ini_filepath)
    register_translator()
    logger.info("Sync package %s, with action %s" % (package, action))
    return sync_package(package, action)

@celery.task(name='syndicate.sync_resource')
def sync_resource_task(package_id, resource_id, action, ckan_ini_filepath):
    logger = sync_resource_task.get_logger()
    load_config(ckan_ini_filepath)
    register_translator()
    logger.info('Sync package {0} resource {1}, with action {2}'.format(
        package_id, resource_id, action))

    return sync_resource(package_id, resource_id, action)

# TODO: why mp this
# enable celery logging for when you run nosetests -s
log = logging.getLogger('ckanext.syndicate.tasks')

def get_logger():
    return log
sync_package_task.get_logger = get_logger
sync_resource_task.get_logger = get_logger

def load_config(ckan_ini_filepath):
    import paste.deploy
    config_abs_path = os.path.abspath(ckan_ini_filepath)
    conf = paste.deploy.appconfig('config:' + config_abs_path)
    import ckan
    ckan.config.environment.load_environment(conf.global_conf,
                                             conf.local_conf)

def register_translator():
    # https://github.com/ckan/ckanext-archiver/blob/master/ckanext/archiver/bin/common.py
    # If not set (in cli access), patch the a translator with a mock, so the
    # _() functions in logic layer don't cause failure.
    from paste.registry import Registry
    from pylons import translator
    from ckan.lib.cli import MockTranslator
    if 'registery' not in globals():
        global registry
        registry = Registry()
        registry.prepare()

    if 'translator_obj' not in globals():
        global translator_obj
        translator_obj = MockTranslator()
        registry.register(translator, translator_obj)

def get_target():
    ckan_url = config.get('ckan.syndicate.ckan_url')
    api_key = config.get('ckan.syndicate.api_key')
    assert ckan_url and api_key, "Task must have ckan_url and api_key"

    ckan = ckanapi.RemoteCKAN(ckan_url, apikey=api_key)
    return ckan

def filter_extras(extras):
    extras_dict = dict([(o['key'], o['value']) for o in extras])
    del extras_dict[get_syndicate_flag()]
    return [{'key': k, 'value': v} for (k, v) in extras_dict.iteritems()]

def filter_resources(resources):
    return [
        {'url': r['url'], 'name': r['name']} for r in resources
    ]

def sync_package(package_id, action, ckan_ini_filepath=None):
    logger.info('sync package {0}'.format(package_id))

    # load the package at run of time task (rather than use package state at
    # time of task creation).
    from ckan import model
    context = {'model': model, 'ignore_auth': True, 'session': model.Session,
               'use_cache': False, 'validate': False}

    params = {
        'id': package_id,
    }
    package = toolkit.get_action('package_show')(
        context,
        params,
    )

    if action == 'dataset/create':
        _create_package(package)

    elif action == 'dataset/update':
        _update_package(package)
    elif action == 'dataset/delete':
        _delete_package(package)
    else:
        raise Exception('Unsupported action {0}'.format(action))

def sync_resource(dataset_id, resource_id, action):
    from ckan import model
    context = {'model': model, 'ignore_auth': True, 'session': model.Session,
               'use_cache': False, 'validate': False}

    dataset_dict = toolkit.get_action('package_show')(
        context,
        data_dict={
            'id': dataset_id,
        }
    )

    _update_package(dataset_dict)

def _create_package(package):
    ckan = get_target()

    # Create a new package based on the local instance
    new_package_data = dict(package)

    del new_package_data['id']
    del new_package_data['tags']

    new_package_data['name'] = "%s-%s" % (
        get_syndicated_name_prefix(),
        new_package_data['name'])

    new_package_data['extras'] = filter_extras(new_package_data['extras'])
    new_package_data['resources'] = filter_resources(package['resources'])
    new_package_data['owner_org'] = get_syndicated_organization()
    del new_package_data['organization']

    context = {}

    try:
        new_package_data = toolkit.get_action('update_dataset_for_syndication')(
            context, {'dataset_dict': new_package_data})
    except KeyError:
        pass

    try:
        remote_package = ckan.action.package_create(**new_package_data)

        set_syndicated_id(package, remote_package['id'])
    except toolkit.ValidationError as e:
        if 'That URL is already in use.' in e.error_dict.get('name', []):
            logger.info("package with name '{0}' already exists".format(
                new_package_data['name']))
        raise e

def _update_package(package):
    syndicated_id = get_pkg_dict_extra(package, get_syndicated_id())

    if syndicated_id is None:
        _create_package(package)
        return

    ckan = get_target()

    try:
        updated_package = dict(package)
        # Keep the existing remote ID and Name
        del updated_package['id']
        del updated_package['name']

        updated_package['extras'] = filter_extras(package['extras'])
        updated_package['resources'] = filter_resources(package['resources'])
        ckan.action.package_update(
            id=syndicated_id,
            **updated_package
        )
    except ckanapi.NotFound:
        _create_package(package)

def _delete_package(package):
    # TODO: Check when this gets called. Deleting a dataset just sets the
    # state to deleted so effectively an update
    syndicated_id = get_pkg_dict_extra(package, get_syndicated_id())

    ckan = get_target()

    ckan.action.package_delete(
        id=syndicated_id)

def set_syndicated_id(local_package, remote_package_id):
    """ Set the remote package id on the local package """
    extras = local_package['extras']
    extras_dict = dict([(o['key'], o['value']) for o in extras])
    extras_dict.update({get_syndicated_id(): remote_package_id})
    extras = [{'key': k, 'value': v} for (k, v) in extras_dict.iteritems()]
    local_package['extras'] = extras
    _update_package_extras(local_package)

def _update_package_extras(package):
    from ckan import model
    from ckan.lib.dictization.model_save import package_extras_save

    package_id = package['id']
    package_obj = model.Package.get(package_id)
    if not package:
        raise Exception('No Package with ID %s found:s' % package_id)

    extra_dicts = package.get("extras")
    context_ = {'model': model, 'session': model.Session}
    model.repo.new_revision()
    package_extras_save(extra_dicts, package_obj, context_)
    model.Session.commit()
    model.Session.flush()

    _update_search_index(package_obj.id, logger)

def _update_search_index(package_id, log):
    '''
    Tells CKAN to update its search index for a given package.
    '''
    from ckan import model
    from ckan.lib.search.index import PackageSearchIndex
    package_index = PackageSearchIndex()
    context_ = {'model': model, 'ignore_auth': True, 'session': model.Session,
                'use_cache': False, 'validate': False}
    package = toolkit.get_action('package_show')(context_, {'id': package_id})
    package_index.index_package(package, defer_commit=False)
    log.info('Search indexed %s', package['name'])
<commit_msg>Remove backup file checked in by error<commit_after><|endoftext|>