<commit_before>from os import path
import os
import shutil
import sys

def _local_path(name):
    return path.join(path.dirname(__file__), name)

def install_headers(package_name, include_dir=None):
    """Install the headers of a known package_name into include_dir. Known
    package names are ['murmurhash', 'numpy'].

    include_dir defaults to path.join(sys.prefix, 'include'):

    >>> install_headers('numpy')
    >>> assert path.exists(path.join(sys.prefix, 'include', 'numpy', 'ndarray.h'))

    If path.join(include_dir, package_name) exists, install_headers will look
    for any headers that are missing, and add them.
    """
    if include_dir is None:
        include_dir = path.join(sys.prefix, 'include')
    assert path.exists(include_dir) and path.isdir(include_dir)
    assert not path.islink(include_dir)
    src_dir = _local_path(package_name)
    dest_dir = path.join(include_dir, package_name)
    if path.exists(dest_dir):
        for filename in os.listdir(src_dir):
            shutil.copy(path.join(src_dir, filename), path.join(dest_dir, filename))
    else:
        shutil.copytree(src_dir, dest_dir)

def fix_venv_pypy_include(include_dir=None):
    """Workaround virtualenv bug for pypy. Virtualenv symlinks the entire include
    directory, instead of creating its own directory and symlinking the contents.
    Unlink the link, create own dir, and link the contents within it.
    """
    if include_dir is None:
        include_dir = path.join(sys.prefix, 'include')
    assert path.exists(include_dir)
    if path.islink(include_dir):
        dest_dir = include_dir
        src_dir = path.realpath(include_dir)
        os.unlink(dest_dir)
        os.mkdir(dest_dir)
        for fn in os.listdir(src_dir):
            shutil.copytree(path.join(src_dir, fn), path.join(dest_dir, fn))
<commit_msg>* Fix venv pypy workaround<commit_after>from os import path
import os
import shutil
import sys

def _local_path(name):
    return path.join(path.dirname(__file__), name)

def install_headers(package_name, include_dir=None):
    """Install the headers of a known package_name into include_dir. Known
    package names are ['murmurhash', 'numpy'].

    include_dir defaults to path.join(sys.prefix, 'include'):

    >>> install_headers('numpy')
    >>> assert path.exists(path.join(sys.prefix, 'include', 'numpy', 'ndarray.h'))

    If path.join(include_dir, package_name) exists, install_headers will look
    for any headers that are missing, and add them.
    """
    if include_dir is None:
        include_dir = path.join(sys.prefix, 'include')
    assert path.exists(include_dir) and path.isdir(include_dir)
    assert not path.islink(include_dir)
    src_dir = _local_path(package_name)
    dest_dir = path.join(include_dir, package_name)
    if path.exists(dest_dir):
        for filename in os.listdir(src_dir):
            shutil.copy(path.join(src_dir, filename), path.join(dest_dir, filename))
    else:
        shutil.copytree(src_dir, dest_dir)

def fix_venv_pypy_include(include_dir=None):
    """Workaround virtualenv bug for pypy. Virtualenv symlinks the entire include
    directory, instead of creating its own directory and symlinking the contents.
    Unlink the link, create own dir, and link the contents within it.
    """
    if include_dir is None:
        include_dir = path.join(sys.prefix, 'include')
    assert path.exists(include_dir)
    if path.islink(include_dir):
        dest_dir = include_dir
        src_dir = path.realpath(include_dir)
        os.unlink(dest_dir)
        os.mkdir(dest_dir)
        for fn in os.listdir(src_dir):
            if path.isdir(path.join(src_dir, fn)):
                shutil.copytree(path.join(src_dir, fn), path.join(dest_dir, fn))
            else:
                shutil.copy(path.join(src_dir, fn), path.join(dest_dir, fn))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

from copy import deepcopy
import os

import pytest

import discoursegraphs as dg

"""
This module contains some tests for the ``discoursegraph`` module.
"""

import pytest

class TestDiscourseDocumentGraph:
    def setup(self):
        """create an empty DiscourseDocumentGraph"""
        self.docgraph = dg.DiscourseDocumentGraph()
        assert isinstance(self.docgraph, dg.DiscourseDocumentGraph)
        assert self.docgraph.name == ''
        assert self.docgraph.nodes() == ['discoursegraph:root_node']

    def test_docgraph_name_namespace(self):
        """create a docgraph with a user-defined name and namespace"""
        name = 'mydoc'
        ns = 'weird'
        ddg = dg.DiscourseDocumentGraph(name=name, namespace=ns)
        assert isinstance(ddg, dg.DiscourseDocumentGraph)
        assert ddg.name == name
        assert ddg.nodes() == ['{}:root_node'.format(ns)]

    def test_add_node(self):
        """add nodes to the graph, with and without layers"""
        # add node without any additional attributes / layers
        self.docgraph.add_node(1)
        assert len(self.docgraph.node) == 2
        assert self.docgraph.node[1] == {'layers': {'discoursegraph'}}

        # add nodes with additional layer(s)
        self.docgraph.add_node(2, layers={'foo'})
        assert len(self.docgraph.node) == 3
        assert self.docgraph.node[2] == {'layers': {'foo'}}

        self.docgraph.add_node(3, layers={'foo', 'bar'})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[3] == {'layers': {'foo', 'bar'}}

        self.docgraph.add_node(4, layers={self.docgraph.ns, 'foo', 'bar'})
        assert len(self.docgraph.node) == 5
        assert self.docgraph.node[4] == \
            {'layers': {'discoursegraph', 'foo', 'bar'}}

        # re-add an already existing node with different layer(s)
        # this will simply add the new layers to the existing set
        self.docgraph.add_node(4, layers={'bla'})
        assert len(self.docgraph.node) == 5
        assert self.docgraph.node[4] == \
            {'layers': {'discoursegraph', 'foo', 'bar', 'bla'}}

        self.docgraph.add_node(4, layers={'xyz', 'abc'})
        assert len(self.docgraph.node) == 5
        assert self.docgraph.node[4] == \
            {'layers': {'discoursegraph', 'foo', 'bar', 'bla', 'xyz', 'abc'}}

        # try to add node with bad layers
        with pytest.raises(AssertionError) as excinfo:
            # 'layers' must be a set of str
            self.docgraph.add_node(666, layers='foo')
            self.docgraph.add_node(666, layers=['foo'])
            self.docgraph.add_node(666, layers={23})
            self.docgraph.add_node(666, layers={'foo', 23})

    def test_add_node_with_attributes(self):
        """add nodes to the graph, with and without attributes"""
        # add node without attributes
        self.docgraph.add_node(1, attr_dict=None)
        assert len(self.docgraph.node) == 2
        assert self.docgraph.node[1] == {'layers': {'discoursegraph'}}

        # add node with attributes
        self.docgraph.add_node(2, attr_dict={'cat': 'NP'})
        assert len(self.docgraph.node) == 3
        assert self.docgraph.node[2] == \
            {'layers': {'discoursegraph'},
             'cat': 'NP'}

        self.docgraph.add_node(3, attr_dict={'cat': 'NP', 'lemma': 'dog'})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[3] == \
            {'layers': {'discoursegraph'},
             'cat': 'NP', 'lemma': 'dog'}

        # overwrite attribute of existing node
        self.docgraph.add_node(2, attr_dict={'cat': 'N'})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[2] == \
            {'layers': {'discoursegraph'},
             'cat': 'N'}

        # overwrite attribute of existing node add additional attrib
        self.docgraph.add_node(2, attr_dict={'cat': 'NP', 'score': 0.5})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[2] == \
            {'layers': {'discoursegraph'},
             'cat': 'NP', 'score': 0.5}

        # try to add node with bad attributes
        with pytest.raises(AttributeError) as excinfo:
            # 'attr_dict' must be a dict
            self.docgraph.add_node(666, attr_dict='foo')
            self.docgraph.add_node(666, attr_dict=['foo'])
            self.docgraph.add_node(666, attr_dict={23})
            self.docgraph.add_node(666, attr_dict={'foo', 23})

    def test_add_nodes_from(self):
        """add multiple nodes at once"""
        assert len(self.docgraph.node) == 1  # the root node is already present

        # add nodes without layers / attributes
        self.docgraph.add_nodes_from(range(5))
        assert len(self.docgraph.node) == 6

        for node_id in range(5):
            assert self.docgraph.node[node_id] == {'layers': {'discoursegraph'}}

        # re-add nodes with an additional layer
        self.docgraph.add_nodes_from([(0, {'layers':{'foo'}}), (1, {'layers':{'bar'}})])
        assert self.docgraph.node[0] == \
                {'layers': {'discoursegraph', 'foo'}}
        assert self.docgraph.node[1] == \
                {'layers': {'discoursegraph', 'bar'}}

    @pytest.mark.xfail
    def test_add_nodes_from_fail(self):
        """this is an implementation error. layers should never be overwritten,
        only appended."""
        self.docgraph.add_nodes_from((0,1))
        # re-add nodes with an additional layer
        self.docgraph.add_nodes_from((0,1), layers={'foo'})

        for node_id in (0,1):
            assert self.docgraph.node[node_id] == \
                {'layers': {'discoursegraph', 'foo'}}

    def test_add_edge(self):
        self.docgraph.add_nodes_from(
            [(1, {'layers':{'token'}, 'word':'hello'}),
             (2, {'layers':{'token'}, 'word':'world'})])

        assert self.docgraph.edges(data=True) == []

        # in a multi-digraph, we can have multiple edges between nodes
        self.docgraph.add_edge(1, 2, layers={'generic'})
        self.docgraph.add_edge(1, 2, layers={'tokens'}, weight=0.7)

        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'generic'}}),
             (1, 2, {'layers': {'tokens'}, 'weight': 0.7})]

        # there are two edges between the nodes 1 and 2
        # this will update only the 'weight' attribute of the second one (key=1)
        self.docgraph.add_edge(1, 2, layers={'tokens'}, key=1, weight=1.0)
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'generic'}}),
             (1, 2, {'layers': {'tokens'}, 'weight': 1.0})]

        # add an additional layer to the second multiedge
        self.docgraph.add_edge(1, 2, layers={'foo'}, key=1)
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'generic'}}),
             (1, 2, {'layers': {'foo', 'tokens'}, 'weight': 1.0})]

        with pytest.raises(AttributeError) as excinfo:
            # 'attr_dict' must be a dict
            self.docgraph.add_edge(1, 2, attr_dict='bar')

    def test_add_edges_from(self):
        """add multiple edges at once"""
        # multiple edges between the same nodes
        self.docgraph.add_edges_from(
            [(1, 2, {'layers': {'int'}, 'weight': 23}),
             (1, 2, {'layers': {'int'}, 'weight': 42})])
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'int'}, 'weight': 23}),
             (1, 2, {'layers': {'int'}, 'weight': 42})]

        # update the first edge (key=0), overwrite its 'weigth'
        # the 'layers' value is not overwritten, but appended
        self.docgraph.add_edges_from([(1, 2, 0, {'layers':{'number'}, 'weight':66})])
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'int', 'number'}, 'weight': 66}),
             (1, 2, {'layers': {'int'}, 'weight': 42})]

        # this combines attributes given in the edge-specific ebunch and
        # the attr_dict, which applies to all edges fed to this method
        # call
        self.docgraph.add_edges_from(
            ebunch=[(3, 4, {'layers': {'ptb'}})],
            attr_dict={'score': 0.5})
        assert self.docgraph[3][4] == {0: {'layers': {'ptb'}, 'score': 0.5}}

        self.docgraph.add_edges_from(
            ebunch=[(4, 5, {'layers': {'ptb'}})],
            attr_dict={'score': 0.5, 'layers': {'fake'}})
        assert self.docgraph[4][5] == {0: {'layers': {'fake', 'ptb'}, 'score': 0.5}}

        with pytest.raises(AttributeError) as excinfo:
            # ebunch must be a (u, v, attribs) or (u, v, key, attribs) tuple
            self.docgraph.add_edges_from(ebunch=[(1, 2)])
            self.docgraph.add_edges_from(ebunch=[(1, 2)], attr_dict='bogus')
            # attr_dict must be a dict
            self.docgraph.add_edges_from(
                ebunch=[(1, 2, {'layers': {'ptb'}})],
                attr_dict='bogus')

        with pytest.raises(AssertionError) as excinfo:
            # (u, v, attribs): attribs must be a dict
            self.docgraph.add_edges_from([(1, 2, 'bar')])
            # (u, v, attribs): attribs must contain a 'layers' key
            self.docgraph.add_edges_from([(1, 2, {'score': 0.5})])

    def test_add_layer(self):
        """add a layer to existing nodes or edges"""
        self.docgraph.add_node(1)
        self.docgraph.add_layer(1, 'fake')
        assert self.docgraph.node[1] == {'layers': {'discoursegraph', 'fake'}}

        self.docgraph.add_edge(1, 2)
        self.docgraph.add_layer((1, 2), 'fake')
        assert self.docgraph[1][2] == {0: {'layers': {'discoursegraph', 'fake'}}}

    def test_add_offsets_get_offsets(self):
        """annotate tokens with offsets and retrieve them."""
        # add a few tokens to the docgraph
        for i, token in enumerate(['Ich', 'bin', 'ein', 'Berliner', '.']):
            self.docgraph.add_node(
                i, layers={self.docgraph.ns+':token'},
                attr_dict={self.docgraph.ns+':token': token})
            self.docgraph.tokens.append(i)
        assert len(self.docgraph.tokens) == 5

        # get_offsets() will trigger add_offsets() on first run
        self.docgraph.get_offsets()

        assert self.docgraph.get_offsets(0) == (0, 3)
        assert self.docgraph.get_offsets(1) == (4, 7)
        assert self.docgraph.get_offsets(2) == (8, 11)
        assert self.docgraph.get_offsets(3) == (12, 20)
        assert self.docgraph.get_offsets(4) == (21, 22)
        assert list(self.docgraph.get_offsets()) == \
            [(0, 0, 3), (1, 4, 7), (2, 8, 11), (3, 12, 20), (4, 21, 22)]

        # adding the offsets again (with the default namespace) must not
        # change anything
        nodes_dict = deepcopy(self.docgraph.node)
        self.docgraph.add_offsets()
        assert self.docgraph.node == nodes_dict

    def test_get_phrases(self):
        """extract all VPs from a document"""
        ptb_filepath = os.path.join(dg.DATA_ROOT_DIR, 'ptb-example.mrg')
        #~ import pudb; pudb.set_trace()
        pdg = dg.read_ptb(ptb_filepath)
        list(pdg.get_phrases(cat_val='PP'))

        verb_phrase_node_ids = list(pdg.get_phrases(cat_val='VP'))
        assert verb_phrase_node_ids == [6, 67, 68, 71, 124, 138, 206]
        assert dg.get_text(pdg, 206) == \
            'aligns with anarcho-syndicalism and libertarian socialism'

    def test_get_tokens(self):
        # add a few tokens to the docgraph
        for i, token in enumerate(['dogs', 'bite']):
            self.docgraph.add_node(
                i, layers={self.docgraph.ns+':token'},
                attr_dict={self.docgraph.ns+':token': token})
            self.docgraph.tokens.append(i)

        assert list(self.docgraph.get_tokens()) == [(0, 'dogs'), (1, 'bite')]
        assert list(self.docgraph.get_tokens(token_strings_only=True)) == ['dogs', 'bite']

def test_is_continuous():
    """tests, if a discontinuous span of tokens is recognised as such.

    The node ``lower`` spans the two non-adjacent tokens 1 and 4.

            root
              |
         ---upper---
         |    |    |
    -----|--lower--|----
    |    |         |   |
    1    2         3   4
    """
    docgraph = dg.DiscourseDocumentGraph()
    docgraph.add_node('1', attr_dict={'discoursegraph:token': '1'})
    docgraph.add_node('2', attr_dict={'discoursegraph:token': '2'})
    docgraph.add_node('3', attr_dict={'discoursegraph:token': '3'})
    docgraph.add_node('4', attr_dict={'discoursegraph:token': '4'})
    docgraph.add_edge(docgraph.root, 'upper', edge_type=dg.EdgeTypes.dominance_relation)
    docgraph.add_edge('upper', 'lower', edge_type=dg.EdgeTypes.dominance_relation)
    docgraph.add_edge('lower', '1', edge_type=dg.EdgeTypes.spanning_relation)
    docgraph.add_edge('upper', '2', edge_type=dg.EdgeTypes.spanning_relation)
    docgraph.add_edge('upper', '3', edge_type=dg.EdgeTypes.spanning_relation)
    docgraph.add_edge('lower', '4', edge_type=dg.EdgeTypes.spanning_relation)
    # determine order of the tokens
    docgraph.tokens = ['1', '2', '3', '4']

    assert dg.is_continuous(docgraph, docgraph.root)
    assert dg.is_continuous(docgraph, 'upper')
    assert not dg.is_continuous(docgraph, 'lower')
    assert dg.is_continuous(docgraph, '1')
    assert dg.is_continuous(docgraph, '2')
    assert dg.is_continuous(docgraph, '3')
    assert dg.is_continuous(docgraph, '4')
<commit_msg>quantifiedcode: removed double import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

from copy import deepcopy
import os

import pytest

import discoursegraphs as dg

"""
This module contains some tests for the ``discoursegraph`` module.
"""

class TestDiscourseDocumentGraph:
    def setup(self):
        """create an empty DiscourseDocumentGraph"""
        self.docgraph = dg.DiscourseDocumentGraph()
        assert isinstance(self.docgraph, dg.DiscourseDocumentGraph)
        assert self.docgraph.name == ''
        assert self.docgraph.nodes() == ['discoursegraph:root_node']

    def test_docgraph_name_namespace(self):
        """create a docgraph with a user-defined name and namespace"""
        name = 'mydoc'
        ns = 'weird'
        ddg = dg.DiscourseDocumentGraph(name=name, namespace=ns)
        assert isinstance(ddg, dg.DiscourseDocumentGraph)
        assert ddg.name == name
        assert ddg.nodes() == ['{}:root_node'.format(ns)]

    def test_add_node(self):
        """add nodes to the graph, with and without layers"""
        # add node without any additional attributes / layers
        self.docgraph.add_node(1)
        assert len(self.docgraph.node) == 2
        assert self.docgraph.node[1] == {'layers': {'discoursegraph'}}

        # add nodes with additional layer(s)
        self.docgraph.add_node(2, layers={'foo'})
        assert len(self.docgraph.node) == 3
        assert self.docgraph.node[2] == {'layers': {'foo'}}

        self.docgraph.add_node(3, layers={'foo', 'bar'})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[3] == {'layers': {'foo', 'bar'}}

        self.docgraph.add_node(4, layers={self.docgraph.ns, 'foo', 'bar'})
        assert len(self.docgraph.node) == 5
        assert self.docgraph.node[4] == \
            {'layers': {'discoursegraph', 'foo', 'bar'}}

        # re-add an already existing node with different layer(s)
        # this will simply add the new layers to the existing set
        self.docgraph.add_node(4, layers={'bla'})
        assert len(self.docgraph.node) == 5
        assert self.docgraph.node[4] == \
            {'layers': {'discoursegraph', 'foo', 'bar', 'bla'}}

        self.docgraph.add_node(4, layers={'xyz', 'abc'})
        assert len(self.docgraph.node) == 5
        assert self.docgraph.node[4] == \
            {'layers': {'discoursegraph', 'foo', 'bar', 'bla', 'xyz', 'abc'}}

        # try to add node with bad layers
        with pytest.raises(AssertionError) as excinfo:
            # 'layers' must be a set of str
            self.docgraph.add_node(666, layers='foo')
            self.docgraph.add_node(666, layers=['foo'])
            self.docgraph.add_node(666, layers={23})
            self.docgraph.add_node(666, layers={'foo', 23})

    def test_add_node_with_attributes(self):
        """add nodes to the graph, with and without attributes"""
        # add node without attributes
        self.docgraph.add_node(1, attr_dict=None)
        assert len(self.docgraph.node) == 2
        assert self.docgraph.node[1] == {'layers': {'discoursegraph'}}

        # add node with attributes
        self.docgraph.add_node(2, attr_dict={'cat': 'NP'})
        assert len(self.docgraph.node) == 3
        assert self.docgraph.node[2] == \
            {'layers': {'discoursegraph'},
             'cat': 'NP'}

        self.docgraph.add_node(3, attr_dict={'cat': 'NP', 'lemma': 'dog'})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[3] == \
            {'layers': {'discoursegraph'},
             'cat': 'NP', 'lemma': 'dog'}

        # overwrite attribute of existing node
        self.docgraph.add_node(2, attr_dict={'cat': 'N'})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[2] == \
            {'layers': {'discoursegraph'},
             'cat': 'N'}

        # overwrite attribute of existing node add additional attrib
        self.docgraph.add_node(2, attr_dict={'cat': 'NP', 'score': 0.5})
        assert len(self.docgraph.node) == 4
        assert self.docgraph.node[2] == \
            {'layers': {'discoursegraph'},
             'cat': 'NP', 'score': 0.5}

        # try to add node with bad attributes
        with pytest.raises(AttributeError) as excinfo:
            # 'attr_dict' must be a dict
            self.docgraph.add_node(666, attr_dict='foo')
            self.docgraph.add_node(666, attr_dict=['foo'])
            self.docgraph.add_node(666, attr_dict={23})
            self.docgraph.add_node(666, attr_dict={'foo', 23})

    def test_add_nodes_from(self):
        """add multiple nodes at once"""
        assert len(self.docgraph.node) == 1  # the root node is already present

        # add nodes without layers / attributes
        self.docgraph.add_nodes_from(range(5))
        assert len(self.docgraph.node) == 6

        for node_id in range(5):
            assert self.docgraph.node[node_id] == {'layers': {'discoursegraph'}}

        # re-add nodes with an additional layer
        self.docgraph.add_nodes_from([(0, {'layers':{'foo'}}), (1, {'layers':{'bar'}})])
        assert self.docgraph.node[0] == \
                {'layers': {'discoursegraph', 'foo'}}
        assert self.docgraph.node[1] == \
                {'layers': {'discoursegraph', 'bar'}}

    @pytest.mark.xfail
    def test_add_nodes_from_fail(self):
        """this is an implementation error. layers should never be overwritten,
        only appended."""
        self.docgraph.add_nodes_from((0,1))
        # re-add nodes with an additional layer
        self.docgraph.add_nodes_from((0,1), layers={'foo'})

        for node_id in (0,1):
            assert self.docgraph.node[node_id] == \
                {'layers': {'discoursegraph', 'foo'}}

    def test_add_edge(self):
        self.docgraph.add_nodes_from(
            [(1, {'layers':{'token'}, 'word':'hello'}),
             (2, {'layers':{'token'}, 'word':'world'})])

        assert self.docgraph.edges(data=True) == []

        # in a multi-digraph, we can have multiple edges between nodes
        self.docgraph.add_edge(1, 2, layers={'generic'})
        self.docgraph.add_edge(1, 2, layers={'tokens'}, weight=0.7)

        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'generic'}}),
             (1, 2, {'layers': {'tokens'}, 'weight': 0.7})]

        # there are two edges between the nodes 1 and 2
        # this will update only the 'weight' attribute of the second one (key=1)
        self.docgraph.add_edge(1, 2, layers={'tokens'}, key=1, weight=1.0)
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'generic'}}),
             (1, 2, {'layers': {'tokens'}, 'weight': 1.0})]

        # add an additional layer to the second multiedge
        self.docgraph.add_edge(1, 2, layers={'foo'}, key=1)
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'generic'}}),
             (1, 2, {'layers': {'foo', 'tokens'}, 'weight': 1.0})]

        with pytest.raises(AttributeError) as excinfo:
            # 'attr_dict' must be a dict
            self.docgraph.add_edge(1, 2, attr_dict='bar')

    def test_add_edges_from(self):
        """add multiple edges at once"""
        # multiple edges between the same nodes
        self.docgraph.add_edges_from(
            [(1, 2, {'layers': {'int'}, 'weight': 23}),
             (1, 2, {'layers': {'int'}, 'weight': 42})])
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'int'}, 'weight': 23}),
             (1, 2, {'layers': {'int'}, 'weight': 42})]

        # update the first edge (key=0), overwrite its 'weigth'
        # the 'layers' value is not overwritten, but appended
        self.docgraph.add_edges_from([(1, 2, 0, {'layers':{'number'}, 'weight':66})])
        assert self.docgraph.edges(data=True) == \
            [(1, 2, {'layers': {'int', 'number'}, 'weight': 66}),
             (1, 2, {'layers': {'int'}, 'weight': 42})]

        # this combines attributes given in the edge-specific ebunch and
        # the attr_dict, which applies to all edges fed to this method
        # call
        self.docgraph.add_edges_from(
            ebunch=[(3, 4, {'layers': {'ptb'}})],
            attr_dict={'score': 0.5})
        assert self.docgraph[3][4] == {0: {'layers': {'ptb'}, 'score': 0.5}}

        self.docgraph.add_edges_from(
            ebunch=[(4, 5, {'layers': {'ptb'}})],
            attr_dict={'score': 0.5, 'layers': {'fake'}})
        assert self.docgraph[4][5] == {0: {'layers': {'fake', 'ptb'}, 'score': 0.5}}

        with pytest.raises(AttributeError) as excinfo:
            # ebunch must be a (u, v, attribs) or (u, v, key, attribs) tuple
            self.docgraph.add_edges_from(ebunch=[(1, 2)])
            self.docgraph.add_edges_from(ebunch=[(1, 2)], attr_dict='bogus')
            # attr_dict must be a dict
            self.docgraph.add_edges_from(
                ebunch=[(1, 2, {'layers': {'ptb'}})],
                attr_dict='bogus')

        with pytest.raises(AssertionError) as excinfo:
            # (u, v, attribs): attribs must be a dict
            self.docgraph.add_edges_from([(1, 2, 'bar')])
            # (u, v, attribs): attribs must contain a 'layers' key
            self.docgraph.add_edges_from([(1, 2, {'score': 0.5})])

    def test_add_layer(self):
        """add a layer to existing nodes or edges"""
        self.docgraph.add_node(1)
        self.docgraph.add_layer(1, 'fake')
        assert self.docgraph.node[1] == {'layers': {'discoursegraph', 'fake'}}

        self.docgraph.add_edge(1, 2)
        self.docgraph.add_layer((1, 2), 'fake')
        assert self.docgraph[1][2] == {0: {'layers': {'discoursegraph', 'fake'}}}

    def test_add_offsets_get_offsets(self):
        """annotate tokens with offsets and retrieve them."""
        # add a few tokens to the docgraph
        for i, token in enumerate(['Ich', 'bin', 'ein', 'Berliner', '.']):
            self.docgraph.add_node(
                i, layers={self.docgraph.ns+':token'},
                attr_dict={self.docgraph.ns+':token': token})
            self.docgraph.tokens.append(i)
        assert len(self.docgraph.tokens) == 5

        # get_offsets() will trigger add_offsets() on first run
        self.docgraph.get_offsets()

        assert self.docgraph.get_offsets(0) == (0, 3)
        assert self.docgraph.get_offsets(1) == (4, 7)
        assert self.docgraph.get_offsets(2) == (8, 11)
        assert self.docgraph.get_offsets(3) == (12, 20)
        assert self.docgraph.get_offsets(4) == (21, 22)
        assert list(self.docgraph.get_offsets()) == \
            [(0, 0, 3), (1, 4, 7), (2, 8, 11), (3, 12, 20), (4, 21, 22)]

        # adding the offsets again (with the default namespace) must not
        # change anything
        nodes_dict = deepcopy(self.docgraph.node)
        self.docgraph.add_offsets()
        assert self.docgraph.node == nodes_dict

    def test_get_phrases(self):
        """extract all VPs from a document"""
        ptb_filepath = os.path.join(dg.DATA_ROOT_DIR, 'ptb-example.mrg')
        #~ import pudb; pudb.set_trace()
        pdg = dg.read_ptb(ptb_filepath)
        list(pdg.get_phrases(cat_val='PP'))

        verb_phrase_node_ids = list(pdg.get_phrases(cat_val='VP'))
        assert verb_phrase_node_ids == [6, 67, 68, 71, 124, 138, 206]
        assert dg.get_text(pdg, 206) == \
            'aligns with anarcho-syndicalism and libertarian socialism'

    def test_get_tokens(self):
        # add a few tokens to the docgraph
        for i, token in enumerate(['dogs', 'bite']):
            self.docgraph.add_node(
                i, layers={self.docgraph.ns+':token'},
                attr_dict={self.docgraph.ns+':token': token})
            self.docgraph.tokens.append(i)

        assert list(self.docgraph.get_tokens()) == [(0, 'dogs'), (1, 'bite')]
        assert list(self.docgraph.get_tokens(token_strings_only=True)) == ['dogs', 'bite']

def test_is_continuous():
    """tests, if a discontinuous span of tokens is recognised as such.

    The node ``lower`` spans the two non-adjacent tokens 1 and 4.

            root
              |
         ---upper---
         |    |    |
    -----|--lower--|----
    |    |         |   |
    1    2         3   4
    """
    docgraph = dg.DiscourseDocumentGraph()
    docgraph.add_node('1', attr_dict={'discoursegraph:token': '1'})
    docgraph.add_node('2', attr_dict={'discoursegraph:token': '2'})
    docgraph.add_node('3', attr_dict={'discoursegraph:token': '3'})
    docgraph.add_node('4', attr_dict={'discoursegraph:token': '4'})
    docgraph.add_edge(docgraph.root, 'upper', edge_type=dg.EdgeTypes.dominance_relation)
    docgraph.add_edge('upper', 'lower', edge_type=dg.EdgeTypes.dominance_relation)
    docgraph.add_edge('lower', '1', edge_type=dg.EdgeTypes.spanning_relation)
    docgraph.add_edge('upper', '2', edge_type=dg.EdgeTypes.spanning_relation)
    docgraph.add_edge('upper', '3', edge_type=dg.EdgeTypes.spanning_relation)
    docgraph.add_edge('lower', '4', edge_type=dg.EdgeTypes.spanning_relation)
    # determine order of the tokens
    docgraph.tokens = ['1', '2', '3', '4']

    assert dg.is_continuous(docgraph, docgraph.root)
    assert dg.is_continuous(docgraph, 'upper')
    assert not dg.is_continuous(docgraph, 'lower')
    assert dg.is_continuous(docgraph, '1')
    assert dg.is_continuous(docgraph, '2')
    assert dg.is_continuous(docgraph, '3')
    assert dg.is_continuous(docgraph, '4')
<|endoftext|>


<commit_before>from collections import namedtuple

import numpy as np
import uuid
from . import config, log, Parser

from .hnumpy import StorageNumpy
from .IStorage import IStorage

from .tools import get_istorage_attrs, build_remotely, storage_id_from_name, basic_types, \
    valid_types

class StorageObj(IStorage):
    args_names = ["name", "tokens", "storage_id", "class_name", "built_remotely"]
    args = namedtuple('StorageObjArgs', args_names)
    _prepared_store_meta = config.session.prepare('INSERT INTO hecuba.istorage'
                                                  '(storage_id, class_name, name, tokens) '
                                                  ' VALUES (?,?,?,?)')

    """
    This class is where information will be stored in Hecuba.
    The information can be in memory, stored in a python dictionary or local variables, or saved in a
    DB(Cassandra), depending on if it's persistent or not.
    """

    @staticmethod
    def _store_meta(storage_args):
        """
            Saves the information of the object in the istorage table.
            Args:
                storage_args (object): contains all data needed to restore the object from the workers
        """
        log.debug("StorageObj: storing media %s", storage_args)
        try:

            config.session.execute(StorageObj._prepared_store_meta,
                                   [storage_args.storage_id,
                                    storage_args.class_name,
                                    storage_args.name,
                                    storage_args.tokens])
        except Exception as ex:
            log.warn("Error creating the StorageDict metadata: %s, %s", str(storage_args), ex)
            raise ex

    @classmethod
    def _parse_comments(cls, comments):
        parser = Parser("ClassField")
        return parser._parse_comments(comments)

    def __init__(self, name=None, storage_id=None, *args, **kwargs):
        """
            Creates a new storageobj.
            Args:
                name (string): the name of the Cassandra Keyspace + table where information can be found
                tokens (list of tuples): token ranges assigned to the new StorageObj
                storage_id (string):  an unique storageobj identifier
                kwargs: more optional parameters
        """

        # Assign private attributes
        self._persistent_props = StorageObj._parse_comments(self.__doc__)
        self._persistent_attrs = self._persistent_props.keys()
        self._class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)

        super().__init__(name=name, storage_id=storage_id, *args, **kwargs)
        self._table = self.__class__.__name__.lower()
        args        = self.args(self._get_name(), self._tokens, self.storage_id, self._class_name, self._built_remotely)

        self._build_args = args

        if name or storage_id:  # therefore... are we doing an Instantiation or a Creation? (built_remotely may be used to instantiate a mockup)
            try:
                data    = get_istorage_attrs(self.storage_id)[0]
                # Instantiation
            except Exception:
                self._persist_data(name)
                pass # Creation

        log.debug("CREATED StorageObj(%s)", self._get_name())

    def __eq__(self, other):
        return self.__class__ == other.__class__ and self.getID() == other.getID()

    def _persist_attributes(self):
        """
        Persist in-memory attributes to the data store
        """
        for attribute in self._persistent_props.keys():
            try:
                val = super().__getattribute__(attribute)
                setattr(self, attribute, val)
            except AttributeError:
                pass

    def _build_is_attribute(self, attribute, persistence_name, storage_id):
        # Build the IStorage obj
        info = {"tokens": self._tokens, "storage_id": storage_id}
        info.update(self._persistent_props[attribute])
        info["built_remotely"] = self._built_remotely
        info['name'] = persistence_name
        return build_remotely(info)

    def _create_tables(self):
        """
            Setups the python structures used to communicate with the backend.
            Creates the necessary tables on the backend to store the object data.
        """

        log.info("CREATING KEYSPACE AND TABLE %s %s",self._ksp, self._table)

        query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (self._ksp, config.replication)
        config.executelocked(query_keyspace)

        query_simple = 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + \
                       '( storage_id uuid PRIMARY KEY, '
        for key, entry in self._persistent_props.items():
            query_simple += str(key) + ' '
            if entry['type'] != 'dict' and entry['type'] in valid_types:
                if entry['type'] == 'list' or entry['type'] == 'tuple':
                    query_simple += entry['type'] + '<' + entry['columns'] + '>, '
                else:
                    query_simple += entry['type'] + ', '
            else:
                query_simple += 'uuid, '
        try:
            config.executelocked(query_simple[:-2] + ' )')
        except Exception as ir:
            log.error("Unable to execute %s", query_simple)
            raise ir

    def sync(self):
        super().sync()

        for attr_name in self._persistent_attrs:
            attr = getattr(super(), attr_name, None)
            if isinstance(attr, IStorage):
                attr.sync()

    def _persist_data(self, name):
        self._table = self.__class__.__name__.lower()

        # Arguments used to build objects remotely
        self._build_args = self.args(self._get_name(),
                                     self._tokens,
                                     self.storage_id,
                                     self._class_name,
                                     self._built_remotely)

        # If never existed, must create the tables and register
        if not self._built_remotely:
            self._create_tables()

        # Iterate over the objects the user has requested to be persistent
        # retrieve them from memory and make them persistent
        self._persist_attributes()

        StorageObj._store_meta(self._build_args)

    def make_persistent(self, name):
        """
            Once a StorageObj has been created, it can be made persistent. This function retrieves the information about
            the Object class schema, and creates a Cassandra table with those parameters, where information will be
            saved from now on, until execution finishes or StorageObj is no longer persistent.
            It also inserts into the new table all information that was in memory assigned to the StorageObj prior to
            this call.
            Args:
                name (string): name with which the table in the DB will be created
        """
        # Update name
        super().make_persistent(name)

        self._persist_data(name)

    def stop_persistent(self):
        """
            The StorageObj stops being persistent, but keeps the information already stored in Cassandra
        """
        log.debug("STOP PERSISTENT")

        for obj_name in self._persistent_attrs:
            try:
                attr = object.__getattribute__(self, obj_name)
            except AttributeError:
                attr = None

            if isinstance(attr, IStorage):
                attr.stop_persistent()

        super().stop_persistent()
        self.storage_id = None

    def delete_persistent(self):
        """
            Deletes the Cassandra table where the persistent StorageObj stores data
        """
        log.debug("DELETE PERSISTENT: %s", self._table)

        for obj_name in self._persistent_attrs:
            attr = getattr(self, obj_name, None)
            if isinstance(attr, IStorage):
                attr.delete_persistent()

        # TODO Drop table _ksp._table if it just contains a single element (non-perfomant :(
        query = "DELETE FROM {}.{} where storage_id={}".format(self._ksp, self._table, self.storage_id)
        config.session.execute(query)

        query = "DELETE FROM hecuba.istorage where storage_id={}".format(self.storage_id)
        config.session.execute(query)

        super().delete_persistent()
        self.storage_id = None

    def __getattr__(self, attribute):
        """
            Given an attribute, this function returns the value, obtaining it from either:
            a) memory
            b) the Database
            Args:
                attribute: name of the value that we want to obtain
            Returns:
                value: obtained value
        """
        if attribute.startswith('_') or attribute not in self._persistent_attrs:
            return super().__getattribute__(attribute)

        is_istorage_attr = self._persistent_props[attribute]["type"] not in basic_types

        if not self.storage_id:
            if is_istorage_attr:
                value = self._build_is_attribute(attribute, persistence_name=None, storage_id=None)
                super().__setattr__(attribute, value)
            return super().__getattribute__(attribute)

        '''
        StorageObj is persistent.
        If the attribute is not a built-in object, we might have it in memory.
        Since python works using references any modification from another reference will affect this attribute,
        which is the expected behaviour. Therefore, is safe to store in-memory the Hecuba objects.
        '''
        try:
            return super().__getattribute__(attribute)
        except AttributeError as ex:
            # Not present in memory, we will need to rebuild it
            pass

        query = "SELECT %s FROM %s.%s WHERE storage_id = %s;" % (attribute, self._ksp, self._table, self.storage_id)
        log.debug("GETATTR: %s", query)
        try:
            result = config.session.execute(query)
        except Exception as ex:
            log.warn("GETATTR ex %s", ex)
            raise ex

        try:
            value = result[0][0]
            # if exists but is set to None, the current behaviour is raising AttributeError
            if not is_istorage_attr and value is None:
                raise AttributeError('value not found')
        except IndexError as ex:
            if not is_istorage_attr:
                raise AttributeError('value not found')
            value = None
        except TypeError as ex:
            log.warn("ERROR ON QUERY RESULT {}".format(str(result)))
            raise ex

        if is_istorage_attr:
            # Value is uuid or None, because it was not found

            attr_name = None
            if value is None:
                # Value not found, persist it BY NAME using a random name so we can retrieve it later
                attr_name = attribute.lower()
                my_name   = self._get_name()
                trailing_name = my_name[my_name.rfind('.') + 1:]
                number    = uuid.uuid4() # Random value
                attr_name = self._ksp + "." + ("O" + str(number).replace('-','_') + trailing_name + attr_name)[:40]
                value = self._build_is_attribute(attribute, persistence_name=attr_name, storage_id=None)
                # Following lines emulate "self.__setattr__(attribute, value)" without the checks
                values = [self.storage_id, value.storage_id]
                query = "INSERT INTO %s.%s (storage_id,%s)" % (self._ksp, self._table, attribute)
                query += " VALUES (%s,%s)"
                log.debug("SETATTR: " + query)
                config.session.execute(query, values)

            else :
                value = self._build_is_attribute(attribute, persistence_name=attr_name, storage_id=value)

        super().__setattr__(attribute, value)
        return value

    def __setattr__(self, attribute, value):
        """
            Given a key and its value, this function saves it (depending on if it's persistent or not):
                a) In memory
                b) In the DB
            Args:
                attribute: name of the value that we want to set
                value: value that we want to save
        """
        if attribute[0] == '_' or attribute not in self._persistent_attrs:
            super().__setattr__(attribute, value)
            return

        # Transform numpy.ndarrays and python dicts to StorageNumpy and StorageDicts
        if not isinstance(value, IStorage):
            if isinstance(value, np.ndarray):
                value = StorageNumpy(value)
            elif isinstance(value, dict):
                per_dict = self._persistent_props[attribute]
                info = {"name": '', "tokens": self._build_args.tokens, "storage_id": None,
                        "built_remotely": self._built_remotely}
                info.update(per_dict)
                new_value = build_remotely(info)
                new_value.update(value)
                value = new_value

        if self.storage_id:
            # Write attribute to the storage
            if isinstance(value, IStorage):
                if not value.storage_id:
                    # Value is volatile, persist it BY NAME using a random name so we can retrieve it later
                    attr_name = attribute.lower()
                    my_name = self._get_name()
                    trailing_name = my_name[my_name.rfind('.') + 1:]
                    number = uuid.uuid4() # Random value
                    name   = self._ksp + "." + ("O" + str(number).replace('-','_') + trailing_name + attr_name)[:40]
                    value.make_persistent(name)   # Persist BY NAME
                # We store the storage_id when the object belongs to an Hecuba class
                values = [self.storage_id, value.storage_id]
                # We store the IStorage object in memory, to avoid rebuilding when it is not necessary
            else:
                values = [self.storage_id, value]

            query = "INSERT INTO %s.%s (storage_id,%s)" % (self._ksp, self._table, attribute)
            query += " VALUES (%s,%s)"

            log.debug("SETATTR: " + query)
            config.session.execute(query, values)

        # We store all the attributes in memory
        super().__setattr__(attribute, value)

    def __delattr__(self, name):
        """
        Method that deletes a given attribute from a StorageObj
        Args:
            item: the name of the attribute to be deleted
        """
        super().__delattr__(name)

        if self.storage_id and name in self._persistent_attrs:
            query = "UPDATE %s.%s SET %s = null WHERE storage_id = %s" % (
                self._ksp, self._table, name, self.storage_id)
            config.session.execute(query)

    def sync(self):
        """
        Wait until all pending stores to Cassandra have been finished.
        """
        if not self.storage_id:
            return

        # Persistent Object
        for attribute in self._persistent_props.keys():
            try:
                val = super().__getattribute__(attribute)
                if isinstance(val, IStorage):
                    log.debug("StorageObj sync: %s.%s of type %s", self.storage_id, attribute,type(val))
                    val.sync()
            except AttributeError as ex:
                # Not present in memory
                pass

    def split(self):
        raise NotImplementedError("Split is not supported on StorageObjects");

<commit_msg>Add StorageObj instantiation without schema<commit_after>from collections import namedtuple

import numpy as np
import uuid
from . import config, log, Parser

from .hnumpy import StorageNumpy
from .IStorage import IStorage

from .tools import get_istorage_attrs, build_remotely, storage_id_from_name, basic_types, \
    valid_types, extract_ks_tab

class StorageObj(IStorage):
    args_names = ["name", "columns", "tokens", "storage_id", "class_name", "built_remotely"]
    args = namedtuple('StorageObjArgs', args_names)
    _prepared_store_meta = config.session.prepare('INSERT INTO hecuba.istorage'
                                                  '(storage_id, class_name, name, tokens, '
                                                  'columns)'
                                                  ' VALUES (?,?,?,?,?)')

    """
    This class is where information will be stored in Hecuba.
    The information can be in memory, stored in a python dictionary or local variables, or saved in a
    DB(Cassandra), depending on if it's persistent or not.
    """

    @staticmethod
    def _store_meta(storage_args):
        """
            Saves the information of the object in the istorage table.
            Args:
                storage_args (object): contains all data needed to restore the object from the workers
        """
        log.debug("StorageObj: storing media %s", storage_args)
        try:

            config.session.execute(StorageObj._prepared_store_meta,
                                   [storage_args.storage_id,
                                    storage_args.class_name,
                                    storage_args.name,
                                    storage_args.tokens,
                                    storage_args.columns])
        except Exception as ex:
            log.warn("Error creating the StorageDict metadata: %s, %s", str(storage_args), ex)
            raise ex

    @classmethod
    def _parse_comments(cls, comments):
        parser = Parser("ClassField")
        return parser._parse_comments(comments)

    def __init__(self, name=None, storage_id=None, *args, **kwargs):
        """
            Creates a new storageobj.
            Args:
                name (string): the name of the Cassandra Keyspace + table where information can be found
                tokens (list of tuples): token ranges assigned to the new StorageObj
                storage_id (string):  an unique storageobj identifier
                kwargs: more optional parameters
        """

        super().__init__(name=name, storage_id=storage_id, *args, **kwargs)

        self._columns = [] # Empty object
        if getattr(self, "__doc__", None) is not None:
            # Assign private attributes
            self._persistent_props = StorageObj._parse_comments(self.__doc__)
            self._persistent_attrs = self._persistent_props.keys()
            self._columns = [ (k,v['type']) for k,v in self._persistent_props.items()]
        else:
            if not (name or storage_id):
                raise RuntimeError("Volatile StoragObj WITHOUT specification not allowed")

        self._class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)

        self._table = self.__class__.__name__.lower()
        args        = self.args(self._get_name(), self._columns, self._tokens, self.storage_id, self._class_name, self._built_remotely)

        self._build_args = args

        if name or storage_id:  # therefore... are we doing an Instantiation or a Creation? (built_remotely may be used to instantiate a mockup)
            # Field '_istorage_metas' will be set if it exists in HECUBA.istorage
            if getattr(self, "_istorage_metas", None) is None: #Creation
                self._persist_data(name)
            else: #Instantiation
                if getattr(self, "__doc__", None) is not None:
                    # check that the class metadata stored in HECUBA matches the __doc__
                    pass
                else: # No documentation passed, used metadata from hecuba.istorage
                    self._columns = self._istorage_metas.columns
                    ksp, table = extract_ks_tab(self._istorage_metas.class_name)
                    self._table = table  #Update table name to match the ClassName from metadata
                    self._persistent_props = {i[0]:{"type":i[1]} for i in self._columns}
                    self._persistent_attrs = self._persistent_props.keys()
                    # Rebuild '_build_args' with modified args
                    self._build_args = self._build_args._replace(columns=self._columns,
                                                                class_name=self._class_name)

        log.debug("CREATED StorageObj(%s)", self._get_name())

    def __eq__(self, other):
        return self.__class__ == other.__class__ and self.getID() == other.getID()

    def _persist_attributes(self):
        """
        Persist in-memory attributes to the data store
        """
        for attribute in self._persistent_props.keys():
            try:
                val = super().__getattribute__(attribute)
                setattr(self, attribute, val)
            except AttributeError:
                pass

    def _build_is_attribute(self, attribute, persistence_name, storage_id):
        # Build the IStorage obj
        info = {"tokens": self._tokens, "storage_id": storage_id}
        info.update(self._persistent_props[attribute])
        info["built_remotely"] = self._built_remotely
        info['name'] = persistence_name
        return build_remotely(info)

    def _create_tables(self):
        """
            Setups the python structures used to communicate with the backend.
            Creates the necessary tables on the backend to store the object data.
        """

        log.info("CREATING KEYSPACE AND TABLE %s %s",self._ksp, self._table)

        query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (self._ksp, config.replication)
        config.executelocked(query_keyspace)

        query_simple = 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + \
                       '( storage_id uuid PRIMARY KEY, '
        for key, entry in self._persistent_props.items():
            query_simple += str(key) + ' '
            if entry['type'] != 'dict' and entry['type'] in valid_types:
                if entry['type'] == 'list' or entry['type'] == 'tuple':
                    query_simple += entry['type'] + '<' + entry['columns'] + '>, '
                else:
                    query_simple += entry['type'] + ', '
            else:
                query_simple += 'uuid, '
        try:
            config.executelocked(query_simple[:-2] + ' )')
        except Exception as ir:
            log.error("Unable to execute %s", query_simple)
            raise ir

    def sync(self):
        super().sync()

        for attr_name in self._persistent_attrs:
            attr = getattr(super(), attr_name, None)
            if isinstance(attr, IStorage):
                attr.sync()

    def _persist_data(self, name):
        self._table = self.__class__.__name__.lower()

        # Arguments used to build objects remotely
        self._build_args = self.args(self._get_name(),
                                     self._columns,
                                     self._tokens,
                                     self.storage_id,
                                     self._class_name,
                                     self._built_remotely)

        # If never existed, must create the tables and register
        if not self._built_remotely:
            self._create_tables()

        # Iterate over the objects the user has requested to be persistent
        # retrieve them from memory and make them persistent
        self._persist_attributes()

        StorageObj._store_meta(self._build_args)

    def make_persistent(self, name):
        """
            Once a StorageObj has been created, it can be made persistent. This function retrieves the information about
            the Object class schema, and creates a Cassandra table with those parameters, where information will be
            saved from now on, until execution finishes or StorageObj is no longer persistent.
            It also inserts into the new table all information that was in memory assigned to the StorageObj prior to
            this call.
            Args:
                name (string): name with which the table in the DB will be created
        """
        # Update name
        super().make_persistent(name)

        self._persist_data(name)

    def stop_persistent(self):
        """
            The StorageObj stops being persistent, but keeps the information already stored in Cassandra
        """
        log.debug("STOP PERSISTENT")

        for obj_name in self._persistent_attrs:
            try:
                attr = object.__getattribute__(self, obj_name)
            except AttributeError:
                attr = None

            if isinstance(attr, IStorage):
                attr.stop_persistent()

        super().stop_persistent()
        self.storage_id = None

    def delete_persistent(self):
        """
            Deletes the Cassandra table where the persistent StorageObj stores data
        """
        log.debug("DELETE PERSISTENT: %s", self._table)

        for obj_name in self._persistent_attrs:
            attr = getattr(self, obj_name, None)
            if isinstance(attr, IStorage):
                attr.delete_persistent()

        # TODO Drop table _ksp._table if it just contains a single element (non-perfomant :(
        query = "DELETE FROM {}.{} where storage_id={}".format(self._ksp, self._table, self.storage_id)
        config.session.execute(query)

        query = "DELETE FROM hecuba.istorage where storage_id={}".format(self.storage_id)
        config.session.execute(query)

        super().delete_persistent()
        self.storage_id = None

    def __getattr__(self, attribute):
        """
            Given an attribute, this function returns the value, obtaining it from either:
            a) memory
            b) the Database
            Args:
                attribute: name of the value that we want to obtain
            Returns:
                value: obtained value
        """
        if attribute.startswith('_') or attribute not in self._persistent_attrs:
            return super().__getattribute__(attribute)

        is_istorage_attr = self._persistent_props[attribute]["type"] not in basic_types

        if not self.storage_id:
            if is_istorage_attr:
                value = self._build_is_attribute(attribute, persistence_name=None, storage_id=None)
                super().__setattr__(attribute, value)
            return super().__getattribute__(attribute)

        '''
        StorageObj is persistent.
        If the attribute is not a built-in object, we might have it in memory.
        Since python works using references any modification from another reference will affect this attribute,
        which is the expected behaviour. Therefore, is safe to store in-memory the Hecuba objects.
        '''
        try:
            return super().__getattribute__(attribute)
        except AttributeError as ex:
            # Not present in memory, we will need to rebuild it
            pass

        query = "SELECT %s FROM %s.%s WHERE storage_id = %s;" % (attribute, self._ksp, self._table, self.storage_id)
        log.debug("GETATTR: %s", query)
        try:
            result = config.session.execute(query)
        except Exception as ex:
            log.warn("GETATTR ex %s", ex)
            raise ex

        try:
            value = result[0][0]
            # if exists but is set to None, the current behaviour is raising AttributeError
            if not is_istorage_attr and value is None:
                raise AttributeError('value not found')
        except IndexError as ex:
            if not is_istorage_attr:
                raise AttributeError('value not found')
            value = None
        except TypeError as ex:
            log.warn("ERROR ON QUERY RESULT {}".format(str(result)))
            raise ex

        if is_istorage_attr:
            # Value is uuid or None, because it was not found

            attr_name = None
            if value is None:
                # Value not found, persist it BY NAME using a random name so we can retrieve it later
                attr_name = attribute.lower()
                my_name   = self._get_name()
                trailing_name = my_name[my_name.rfind('.') + 1:]
                number    = uuid.uuid4() # Random value
                attr_name = self._ksp + "." + ("O" + str(number).replace('-','_') + trailing_name + attr_name)[:40]
                value = self._build_is_attribute(attribute, persistence_name=attr_name, storage_id=None)
                # Following lines emulate "self.__setattr__(attribute, value)" without the checks
                values = [self.storage_id, value.storage_id]
                query = "INSERT INTO %s.%s (storage_id,%s)" % (self._ksp, self._table, attribute)
                query += " VALUES (%s,%s)"
                log.debug("SETATTR: " + query)
                config.session.execute(query, values)

            else :
                value = self._build_is_attribute(attribute, persistence_name=attr_name, storage_id=value)

        super().__setattr__(attribute, value)
        return value

    def __setattr__(self, attribute, value):
        """
            Given a key and its value, this function saves it (depending on if it's persistent or not):
                a) In memory
                b) In the DB
            Args:
                attribute: name of the value that we want to set
                value: value that we want to save
        """
        if attribute[0] == '_' or attribute not in getattr(self, "_persistent_attrs", []):
            super().__setattr__(attribute, value)
            return

        # Transform numpy.ndarrays and python dicts to StorageNumpy and StorageDicts
        if not isinstance(value, IStorage):
            if isinstance(value, np.ndarray):
                value = StorageNumpy(value)
            elif isinstance(value, dict):
                per_dict = self._persistent_props[attribute]
                info = {"name": '', "tokens": self._build_args.tokens, "storage_id": None,
                        "built_remotely": self._built_remotely}
                info.update(per_dict)
                new_value = build_remotely(info)
                new_value.update(value)
                value = new_value

        if self.storage_id:
            # Write attribute to the storage
            if isinstance(value, IStorage):
                if not value.storage_id:
                    # Value is volatile, persist it BY NAME using a random name so we can retrieve it later
                    attr_name = attribute.lower()
                    my_name = self._get_name()
                    trailing_name = my_name[my_name.rfind('.') + 1:]
                    number = uuid.uuid4() # Random value
                    name   = self._ksp + "." + ("O" + str(number).replace('-','_') + trailing_name + attr_name)[:40]
                    value.make_persistent(name)   # Persist BY NAME
                # We store the storage_id when the object belongs to an Hecuba class
                values = [self.storage_id, value.storage_id]
                # We store the IStorage object in memory, to avoid rebuilding when it is not necessary
            else:
                values = [self.storage_id, value]

            query = "INSERT INTO %s.%s (storage_id,%s)" % (self._ksp, self._table, attribute)
            query += " VALUES (%s,%s)"

            log.debug("SETATTR: " + query)
            config.session.execute(query, values)

        # We store all the attributes in memory
        super().__setattr__(attribute, value)

    def __delattr__(self, name):
        """
        Method that deletes a given attribute from a StorageObj
        Args:
            item: the name of the attribute to be deleted
        """
        super().__delattr__(name)

        if self.storage_id and name in self._persistent_attrs:
            query = "UPDATE %s.%s SET %s = null WHERE storage_id = %s" % (
                self._ksp, self._table, name, self.storage_id)
            config.session.execute(query)

    def sync(self):
        """
        Wait until all pending stores to Cassandra have been finished.
        """
        if not self.storage_id:
            return

        # Persistent Object
        for attribute in self._persistent_props.keys():
            try:
                val = super().__getattribute__(attribute)
                if isinstance(val, IStorage):
                    log.debug("StorageObj sync: %s.%s of type %s", self.storage_id, attribute,type(val))
                    val.sync()
            except AttributeError as ex:
                # Not present in memory
                pass

    def split(self):
        raise NotImplementedError("Split is not supported on StorageObjects");

<|endoftext|>


<commit_before>try:
    from PIL import Image
except ImportError:
    import Image
<commit_msg>fixed compat.py<commit_after>try:
    from PIL import Image
except ImportError:
    import Image

<|endoftext|>


<commit_before>import unittest
import datetime as dt
import sjcx-payments.scripts.exchange_rates as exchange_rate
from datetime import timedelta

class ExchangeRates(unittest.TestCase):

    def test_btc_sjcx(self):
        date = dt.datetime.now() - timedelta(seconds = 86400)
        btc_sjcx = exchange_rate.btc_sjcx_rate(date)
        self.assertTrue(isinstance(btc_sjcx, float))

    def test_btc_usd(self):
	date = dt.datetime.now() - timedelta(seconds = 86400)
        prev_date = dt.datetime(date.year, date.month, date.day, 0, 0, 0)
        btc_usd = exchange_rate.btc_usd_rate(prev_date)
        self.assertTrue(isinstance(btc_usd, float))
<commit_msg>import fix<commit_after>import unittest
import datetime as dt
import scripts.exchange_rates as exchange_rate
from datetime import timedelta

class ExchangeRates(unittest.TestCase):

    def test_btc_sjcx(self):
        date = dt.datetime.now() - timedelta(seconds = 86400)
        btc_sjcx = exchange_rate.btc_sjcx_rate(date)
        self.assertTrue(isinstance(btc_sjcx, float))

    def test_btc_usd(self):
	date = dt.datetime.now() - timedelta(seconds = 86400)
        prev_date = dt.datetime(date.year, date.month, date.day, 0, 0, 0)
        btc_usd = exchange_rate.btc_usd_rate(prev_date)
        self.assertTrue(isinstance(btc_usd, float))
<|endoftext|>


<commit_before>from collections import defaultdict
import os
import time

import pytest
import requests

import ray
from ray.test_utils import SignalActor
from ray import serve
from ray.serve.utils import get_random_letters

@pytest.mark.parametrize("use_handle", [True, False])
def test_deploy(serve_instance, use_handle):
    name = "test"

    @serve.deployment(name, version="1")
    def d(*args):
        return f"1|{os.getpid()}"

    def call():
        if use_handle:
            ret = ray.get(d.get_handle().remote())
        else:
            ret = requests.get(f"http://localhost:8000/{name}").text

        return ret.split("|")[0], ret.split("|")[1]

    d.deploy()
    val1, pid1 = call()
    assert val1 == "1"

    # Redeploying with the same version and code should do nothing.
    d.deploy()
    val2, pid2 = call()
    assert val2 == "1"
    assert pid2 == pid1

    # Redeploying with a new version should start a new actor.
    d.options(version="2").deploy()
    val3, pid3 = call()
    assert val3 == "1"
    assert pid3 != pid2

    @serve.deployment(name, version="2")
    def d(*args):
        return f"2|{os.getpid()}"

    # Redeploying with the same version and new code should do nothing.
    d.deploy()
    val4, pid4 = call()
    assert val4 == "1"
    assert pid4 == pid3

    # Redeploying with new code and a new version should start a new actor
    # running the new code.
    d.options(version="3").deploy()
    val5, pid5 = call()
    assert val5 == "2"
    assert pid5 != pid4

@pytest.mark.parametrize("use_handle", [True, False])
def test_config_change(serve_instance, use_handle):
    name = "test"

    @serve.deployment(name, version="1")
    class D:
        def __init__(self):
            self.ret = "1"

        def reconfigure(self, d):
            self.ret = d["ret"]

        def __call__(self, *args):
            return f"{self.ret}|{os.getpid()}"

    def call():
        if use_handle:
            ret = ray.get(D.get_handle().remote())
        else:
            ret = requests.get(f"http://localhost:8000/{name}").text

        return ret.split("|")[0], ret.split("|")[1]

    # First deploy with no user config set.
    D.deploy()
    val1, pid1 = call()
    assert val1 == "1"

    # Now update the user config without changing versions. Actor should stay
    # alive but return value should change.
    D.options(config={"user_config": {"ret": "2"}}).deploy()
    val2, pid2 = call()
    assert pid2 == pid1
    assert val2 == "2"

    # Update the user config without changing the version again.
    D.options(config={"user_config": {"ret": "3"}}).deploy()
    val3, pid3 = call()
    assert pid3 == pid2
    assert val3 == "3"

    # Update the version without changing the user config.
    D.options(version="2", config={"user_config": {"ret": "3"}}).deploy()
    val4, pid4 = call()
    assert pid4 != pid3
    assert val4 == "3"

    # Update the version and the user config.
    D.options(version="3", config={"user_config": {"ret": "4"}}).deploy()
    val5, pid5 = call()
    assert pid5 != pid4
    assert val5 == "4"

@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_single_replica(serve_instance, use_handle):
    # Tests that redeploying a deployment with a single replica waits for the
    # replica to completely shut down before starting a new one.
    client = serve_instance

    name = "test"

    @ray.remote
    def call(block=False):
        if use_handle:
            ret = ray.get(serve.get_handle(name).remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    signal_name = f"signal-{get_random_letters()}"
    signal = SignalActor.options(name=signal_name).remote()

    async def v1(request):
        if request.query_params["block"] == "True":
            signal = ray.get_actor(signal_name)
            await signal.wait.remote()
        return f"1|{os.getpid()}"

    def v2(*args):
        return f"2|{os.getpid()}"

    client.deploy(name, v1, version="1")
    ref1 = call.remote(block=False)
    val1, pid1 = ray.get(ref1)
    assert val1 == "1"

    # ref2 will block until the signal is sent.
    ref2 = call.remote(block=True)
    assert len(ray.wait([ref2], timeout=0.1)[0]) == 0

    # Redeploy new version. This should not go through until the old version
    # replica completely stops.
    goal_ref = client.deploy(name, v2, version="2", _blocking=False)
    assert not client._wait_for_goal(goal_ref, timeout=0.1)

    # It may take some time for the handle change to propagate and requests
    # to get sent to the new version. Repeatedly send requests until they
    # start blocking
    start = time.time()
    new_version_ref = None
    while time.time() - start < 30:
        ready, not_ready = ray.wait([call.remote(block=False)], timeout=0.5)
        if len(ready) == 1:
            # If the request doesn't block, it must have been the old version.
            val, pid = ray.get(ready[0])
            assert val == "1"
            assert pid == pid1
        elif len(not_ready) == 1:
            # If the request blocks, it must have been the new version.
            new_version_ref = not_ready[0]
            break
    else:
        assert False, "Timed out waiting for new version to be called."

    # Signal the original call to exit.
    ray.get(signal.send.remote())
    val2, pid2 = ray.get(ref2)
    assert val2 == "1"
    assert pid2 == pid1

    # Now the goal and request to the new version should complete.
    assert client._wait_for_goal(goal_ref)
    new_version_val, new_version_pid = ray.get(new_version_ref)
    assert new_version_val == "2"
    assert new_version_pid != pid2

@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_multiple_replicas(serve_instance, use_handle):
    # Tests that redeploying a deployment with multiple replicas performs
    # a rolling update.
    client = serve_instance

    name = "test"

    @ray.remote(num_cpus=0)
    def call(block=False):
        if use_handle:
            handle = serve.get_handle(name, missing_ok=True)
            ret = ray.get(handle.remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    signal_name = f"signal-{get_random_letters()}"
    signal = SignalActor.options(name=signal_name).remote()

    async def v1(request):
        if request.query_params["block"] == "True":
            signal = ray.get_actor(signal_name)
            await signal.wait.remote()
        return f"1|{os.getpid()}"

    def v2(*args):
        return f"2|{os.getpid()}"

    def make_nonblocking_calls(expected, expect_blocking=False):
        # Returns dict[val, set(pid)].
        blocking = []
        responses = defaultdict(set)
        start = time.time()
        while time.time() - start < 30:
            refs = [call.remote(block=False) for _ in range(10)]
            ready, not_ready = ray.wait(refs, timeout=0.5)
            for ref in ready:
                val, pid = ray.get(ref)
                responses[val].add(pid)
            for ref in not_ready:
                blocking.extend(not_ready)

            if (all(
                    len(responses[val]) == num
                    for val, num in expected.items())
                    and (expect_blocking is False or len(blocking) > 0)):
                break
        else:
            assert False, f"Timed out, responses: {responses}."

        return responses, blocking

    client.deploy(name, v1, version="1", config={"num_replicas": 2})
    responses1, _ = make_nonblocking_calls({"1": 2})
    pids1 = responses1["1"]

    # ref2 will block a single replica until the signal is sent. Check that
    # some requests are now blocking.
    ref2 = call.remote(block=True)
    responses2, blocking2 = make_nonblocking_calls(
        {
            "1": 1
        }, expect_blocking=True)
    assert list(responses2["1"])[0] in pids1

    # Redeploy new version. Since there is one replica blocking, only one new
    # replica should be started up.
    goal_ref = client.deploy(
        name,
        v2,
        version="2",
        config={"num_replicas": 2},
        _blocking=False,
    )
    assert not client._wait_for_goal(goal_ref, timeout=0.1)
    responses3, blocking3 = make_nonblocking_calls(
        {
            "1": 1
        }, expect_blocking=True)

    # Signal the original call to exit.
    ray.get(signal.send.remote())
    val, pid = ray.get(ref2)
    assert val == "1"
    assert pid in responses1["1"]

    # Now the goal and requests to the new version should complete.
    # We should have two running replicas of the new version.
    assert client._wait_for_goal(goal_ref)
    make_nonblocking_calls({"2": 2})

@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_scale_down(serve_instance, use_handle):
    # Tests redeploying with a new version and lower num_replicas.
    name = "test"

    @serve.deployment(name, version="1", config={"num_replicas": 4})
    def v1(request):
        return f"1|{os.getpid()}"

    @ray.remote(num_cpus=0)
    def call(block=False):
        if use_handle:
            handle = v1.get_handle()
            ret = ray.get(handle.remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    def make_calls(expected):
        # Returns dict[val, set(pid)].
        responses = defaultdict(set)
        start = time.time()
        while time.time() - start < 30:
            refs = [call.remote(block=False) for _ in range(10)]
            ready, not_ready = ray.wait(refs, timeout=0.5)
            for ref in ready:
                val, pid = ray.get(ref)
                responses[val].add(pid)

            if all(
                    len(responses[val]) == num
                    for val, num in expected.items()):
                break
        else:
            assert False, f"Timed out, responses: {responses}."

        return responses

    v1.deploy()
    responses1 = make_calls({"1": 4})
    pids1 = responses1["1"]

    @serve.deployment(name, version="2", config={"num_replicas": 2})
    def v2(*args):
        return f"2|{os.getpid()}"

    v2.deploy()
    responses2 = make_calls({"2": 2})
    assert all(pid not in pids1 for pid in responses2["2"])

@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_scale_up(serve_instance, use_handle):
    # Tests redeploying with a new version and higher num_replicas.
    name = "test"

    @serve.deployment(name, version="1", config={"num_replicas": 2})
    def v1(request):
        return f"1|{os.getpid()}"

    @ray.remote(num_cpus=0)
    def call(block=False):
        if use_handle:
            handle = v1.get_handle()
            ret = ray.get(handle.remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    def make_calls(expected):
        # Returns dict[val, set(pid)].
        responses = defaultdict(set)
        start = time.time()
        while time.time() - start < 30:
            refs = [call.remote(block=False) for _ in range(10)]
            ready, not_ready = ray.wait(refs, timeout=0.5)
            for ref in ready:
                val, pid = ray.get(ref)
                responses[val].add(pid)

            if all(
                    len(responses[val]) == num
                    for val, num in expected.items()):
                break
        else:
            assert False, f"Timed out, responses: {responses}."

        return responses

    v1.deploy()
    responses1 = make_calls({"1": 2})
    pids1 = responses1["1"]

    @serve.deployment(name, version="2", config={"num_replicas": 4})
    def v2(*args):
        return f"2|{os.getpid()}"

    v2.deploy()
    responses2 = make_calls({"2": 4})
    assert all(pid not in pids1 for pid in responses2["2"])

def test_deploy_handle_validation(serve_instance):
    class A:
        def b(self, *args):
            return "hello"

    serve_instance.deploy("f", A)
    handle = serve.get_handle("f")

    # Legacy code path
    assert ray.get(handle.options(method_name="b").remote()) == "hello"
    # New code path
    assert ray.get(handle.b.remote()) == "hello"
    with pytest.raises(AttributeError):
        handle.c.remote()

    # Test missing_ok case
    missing_handle = serve.get_handle("g", missing_ok=True)
    with pytest.raises(AttributeError):
        missing_handle.b.remote()
    serve_instance.deploy("g", A)
    # Old code path still work
    assert ray.get(missing_handle.options(method_name="b").remote()) == "hello"
    # Because the missing_ok flag, handle.b.remote won't work.
    with pytest.raises(AttributeError):
        missing_handle.b.remote()

if __name__ == "__main__":
    import sys
    sys.exit(pytest.main(["-v", "-s", __file__]))
<commit_msg>[serve] Skip failing test_deploy tests on windows (#14957)<commit_after>from collections import defaultdict
import os
import sys
import time

import pytest
import requests

import ray
from ray.test_utils import SignalActor
from ray import serve
from ray.serve.utils import get_random_letters

@pytest.mark.parametrize("use_handle", [True, False])
def test_deploy(serve_instance, use_handle):
    name = "test"

    @serve.deployment(name, version="1")
    def d(*args):
        return f"1|{os.getpid()}"

    def call():
        if use_handle:
            ret = ray.get(d.get_handle().remote())
        else:
            ret = requests.get(f"http://localhost:8000/{name}").text

        return ret.split("|")[0], ret.split("|")[1]

    d.deploy()
    val1, pid1 = call()
    assert val1 == "1"

    # Redeploying with the same version and code should do nothing.
    d.deploy()
    val2, pid2 = call()
    assert val2 == "1"
    assert pid2 == pid1

    # Redeploying with a new version should start a new actor.
    d.options(version="2").deploy()
    val3, pid3 = call()
    assert val3 == "1"
    assert pid3 != pid2

    @serve.deployment(name, version="2")
    def d(*args):
        return f"2|{os.getpid()}"

    # Redeploying with the same version and new code should do nothing.
    d.deploy()
    val4, pid4 = call()
    assert val4 == "1"
    assert pid4 == pid3

    # Redeploying with new code and a new version should start a new actor
    # running the new code.
    d.options(version="3").deploy()
    val5, pid5 = call()
    assert val5 == "2"
    assert pid5 != pid4

@pytest.mark.parametrize("use_handle", [True, False])
def test_config_change(serve_instance, use_handle):
    name = "test"

    @serve.deployment(name, version="1")
    class D:
        def __init__(self):
            self.ret = "1"

        def reconfigure(self, d):
            self.ret = d["ret"]

        def __call__(self, *args):
            return f"{self.ret}|{os.getpid()}"

    def call():
        if use_handle:
            ret = ray.get(D.get_handle().remote())
        else:
            ret = requests.get(f"http://localhost:8000/{name}").text

        return ret.split("|")[0], ret.split("|")[1]

    # First deploy with no user config set.
    D.deploy()
    val1, pid1 = call()
    assert val1 == "1"

    # Now update the user config without changing versions. Actor should stay
    # alive but return value should change.
    D.options(config={"user_config": {"ret": "2"}}).deploy()
    val2, pid2 = call()
    assert pid2 == pid1
    assert val2 == "2"

    # Update the user config without changing the version again.
    D.options(config={"user_config": {"ret": "3"}}).deploy()
    val3, pid3 = call()
    assert pid3 == pid2
    assert val3 == "3"

    # Update the version without changing the user config.
    D.options(version="2", config={"user_config": {"ret": "3"}}).deploy()
    val4, pid4 = call()
    assert pid4 != pid3
    assert val4 == "3"

    # Update the version and the user config.
    D.options(version="3", config={"user_config": {"ret": "4"}}).deploy()
    val5, pid5 = call()
    assert pid5 != pid4
    assert val5 == "4"

@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_single_replica(serve_instance, use_handle):
    # Tests that redeploying a deployment with a single replica waits for the
    # replica to completely shut down before starting a new one.
    client = serve_instance

    name = "test"

    @ray.remote
    def call(block=False):
        if use_handle:
            ret = ray.get(serve.get_handle(name).remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    signal_name = f"signal-{get_random_letters()}"
    signal = SignalActor.options(name=signal_name).remote()

    async def v1(request):
        if request.query_params["block"] == "True":
            signal = ray.get_actor(signal_name)
            await signal.wait.remote()
        return f"1|{os.getpid()}"

    def v2(*args):
        return f"2|{os.getpid()}"

    client.deploy(name, v1, version="1")
    ref1 = call.remote(block=False)
    val1, pid1 = ray.get(ref1)
    assert val1 == "1"

    # ref2 will block until the signal is sent.
    ref2 = call.remote(block=True)
    assert len(ray.wait([ref2], timeout=0.1)[0]) == 0

    # Redeploy new version. This should not go through until the old version
    # replica completely stops.
    goal_ref = client.deploy(name, v2, version="2", _blocking=False)
    assert not client._wait_for_goal(goal_ref, timeout=0.1)

    # It may take some time for the handle change to propagate and requests
    # to get sent to the new version. Repeatedly send requests until they
    # start blocking
    start = time.time()
    new_version_ref = None
    while time.time() - start < 30:
        ready, not_ready = ray.wait([call.remote(block=False)], timeout=0.5)
        if len(ready) == 1:
            # If the request doesn't block, it must have been the old version.
            val, pid = ray.get(ready[0])
            assert val == "1"
            assert pid == pid1
        elif len(not_ready) == 1:
            # If the request blocks, it must have been the new version.
            new_version_ref = not_ready[0]
            break
    else:
        assert False, "Timed out waiting for new version to be called."

    # Signal the original call to exit.
    ray.get(signal.send.remote())
    val2, pid2 = ray.get(ref2)
    assert val2 == "1"
    assert pid2 == pid1

    # Now the goal and request to the new version should complete.
    assert client._wait_for_goal(goal_ref)
    new_version_val, new_version_pid = ray.get(new_version_ref)
    assert new_version_val == "2"
    assert new_version_pid != pid2

@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_multiple_replicas(serve_instance, use_handle):
    # Tests that redeploying a deployment with multiple replicas performs
    # a rolling update.
    client = serve_instance

    name = "test"

    @ray.remote(num_cpus=0)
    def call(block=False):
        if use_handle:
            handle = serve.get_handle(name, missing_ok=True)
            ret = ray.get(handle.remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    signal_name = f"signal-{get_random_letters()}"
    signal = SignalActor.options(name=signal_name).remote()

    async def v1(request):
        if request.query_params["block"] == "True":
            signal = ray.get_actor(signal_name)
            await signal.wait.remote()
        return f"1|{os.getpid()}"

    def v2(*args):
        return f"2|{os.getpid()}"

    def make_nonblocking_calls(expected, expect_blocking=False):
        # Returns dict[val, set(pid)].
        blocking = []
        responses = defaultdict(set)
        start = time.time()
        while time.time() - start < 30:
            refs = [call.remote(block=False) for _ in range(10)]
            ready, not_ready = ray.wait(refs, timeout=0.5)
            for ref in ready:
                val, pid = ray.get(ref)
                responses[val].add(pid)
            for ref in not_ready:
                blocking.extend(not_ready)

            if (all(
                    len(responses[val]) == num
                    for val, num in expected.items())
                    and (expect_blocking is False or len(blocking) > 0)):
                break
        else:
            assert False, f"Timed out, responses: {responses}."

        return responses, blocking

    client.deploy(name, v1, version="1", config={"num_replicas": 2})
    responses1, _ = make_nonblocking_calls({"1": 2})
    pids1 = responses1["1"]

    # ref2 will block a single replica until the signal is sent. Check that
    # some requests are now blocking.
    ref2 = call.remote(block=True)
    responses2, blocking2 = make_nonblocking_calls(
        {
            "1": 1
        }, expect_blocking=True)
    assert list(responses2["1"])[0] in pids1

    # Redeploy new version. Since there is one replica blocking, only one new
    # replica should be started up.
    goal_ref = client.deploy(
        name,
        v2,
        version="2",
        config={"num_replicas": 2},
        _blocking=False,
    )
    assert not client._wait_for_goal(goal_ref, timeout=0.1)
    responses3, blocking3 = make_nonblocking_calls(
        {
            "1": 1
        }, expect_blocking=True)

    # Signal the original call to exit.
    ray.get(signal.send.remote())
    val, pid = ray.get(ref2)
    assert val == "1"
    assert pid in responses1["1"]

    # Now the goal and requests to the new version should complete.
    # We should have two running replicas of the new version.
    assert client._wait_for_goal(goal_ref)
    make_nonblocking_calls({"2": 2})

@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_scale_down(serve_instance, use_handle):
    # Tests redeploying with a new version and lower num_replicas.
    name = "test"

    @serve.deployment(name, version="1", config={"num_replicas": 4})
    def v1(request):
        return f"1|{os.getpid()}"

    @ray.remote(num_cpus=0)
    def call(block=False):
        if use_handle:
            handle = v1.get_handle()
            ret = ray.get(handle.remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    def make_calls(expected):
        # Returns dict[val, set(pid)].
        responses = defaultdict(set)
        start = time.time()
        while time.time() - start < 30:
            refs = [call.remote(block=False) for _ in range(10)]
            ready, not_ready = ray.wait(refs, timeout=0.5)
            for ref in ready:
                val, pid = ray.get(ref)
                responses[val].add(pid)

            if all(
                    len(responses[val]) == num
                    for val, num in expected.items()):
                break
        else:
            assert False, f"Timed out, responses: {responses}."

        return responses

    v1.deploy()
    responses1 = make_calls({"1": 4})
    pids1 = responses1["1"]

    @serve.deployment(name, version="2", config={"num_replicas": 2})
    def v2(*args):
        return f"2|{os.getpid()}"

    v2.deploy()
    responses2 = make_calls({"2": 2})
    assert all(pid not in pids1 for pid in responses2["2"])

@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
@pytest.mark.parametrize("use_handle", [True, False])
def test_redeploy_scale_up(serve_instance, use_handle):
    # Tests redeploying with a new version and higher num_replicas.
    name = "test"

    @serve.deployment(name, version="1", config={"num_replicas": 2})
    def v1(request):
        return f"1|{os.getpid()}"

    @ray.remote(num_cpus=0)
    def call(block=False):
        if use_handle:
            handle = v1.get_handle()
            ret = ray.get(handle.remote(block=str(block)))
        else:
            ret = requests.get(
                f"http://localhost:8000/{name}", params={
                    "block": block
                }).text

        return ret.split("|")[0], ret.split("|")[1]

    def make_calls(expected):
        # Returns dict[val, set(pid)].
        responses = defaultdict(set)
        start = time.time()
        while time.time() - start < 30:
            refs = [call.remote(block=False) for _ in range(10)]
            ready, not_ready = ray.wait(refs, timeout=0.5)
            for ref in ready:
                val, pid = ray.get(ref)
                responses[val].add(pid)

            if all(
                    len(responses[val]) == num
                    for val, num in expected.items()):
                break
        else:
            assert False, f"Timed out, responses: {responses}."

        return responses

    v1.deploy()
    responses1 = make_calls({"1": 2})
    pids1 = responses1["1"]

    @serve.deployment(name, version="2", config={"num_replicas": 4})
    def v2(*args):
        return f"2|{os.getpid()}"

    v2.deploy()
    responses2 = make_calls({"2": 4})
    assert all(pid not in pids1 for pid in responses2["2"])

@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
def test_deploy_handle_validation(serve_instance):
    class A:
        def b(self, *args):
            return "hello"

    serve_instance.deploy("f", A)
    handle = serve.get_handle("f")

    # Legacy code path
    assert ray.get(handle.options(method_name="b").remote()) == "hello"
    # New code path
    assert ray.get(handle.b.remote()) == "hello"
    with pytest.raises(AttributeError):
        handle.c.remote()

    # Test missing_ok case
    missing_handle = serve.get_handle("g", missing_ok=True)
    with pytest.raises(AttributeError):
        missing_handle.b.remote()
    serve_instance.deploy("g", A)
    # Old code path still work
    assert ray.get(missing_handle.options(method_name="b").remote()) == "hello"
    # Because the missing_ok flag, handle.b.remote won't work.
    with pytest.raises(AttributeError):
        missing_handle.b.remote()

if __name__ == "__main__":
    import sys
    sys.exit(pytest.main(["-v", "-s", __file__]))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_generate_files
-------------------

Test formerly known from a unittest residing in test_generate.py named
TestGenerateFiles.test_generate_files_nontemplated_exception
"""

from __future__ import unicode_literals
import os
import pytest

from cookiecutter import generate
from cookiecutter import exceptions
from cookiecutter import utils

@pytest.fixture(scope="function")
def clean_system_remove_additional_folders(request, clean_system):
    """
    Use the global clean_system fixture and run additional teardown code to
    remove some special folders.

    For a better understanding - order of fixture calls:
    clean_system setup code
    clean_system_remove_additional_folders setup code
    clean_system_remove_additional_folders teardown code
    clean_system teardown code
    """
    def remove_additional_folders():
        if os.path.exists('inputpizzä'):
            utils.rmtree('inputpizzä')
        if os.path.exists('inputgreen'):
            utils.rmtree('inputgreen')
        if os.path.exists('inputbinary_files'):
            utils.rmtree('inputbinary_files')
        if os.path.exists('tests/custom_output_dir'):
            utils.rmtree('tests/custom_output_dir')
        if os.path.exists('inputpermissions'):
            utils.rmtree('inputpermissions')
    request.addfinalizer(remove_additional_folders)

@pytest.mark.usefixtures("clean_system_remove_additional_folders")
def test_generate_files_nontemplated_exception():
    with pytest.raises(exceptions.NonTemplatedInputDirException):
        generate.generate_files(
            context={'cookiecutter': {'food': 'pizza'}},
            repo_dir='tests/test-generate-files-nontemplated'
        )
<commit_msg>Copy original test_generate_files and adapt doc str<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_generate_files
-------------------

Test formerly known from a unittest residing in test_generate.py named
TestGenerateFiles.test_generate_files_nontemplated_exception
TestGenerateFiles.test_generate_files
"""

from __future__ import unicode_literals
import os
import io
import pytest

from cookiecutter import generate
from cookiecutter import exceptions
from cookiecutter import utils

@pytest.fixture(scope="function")
def clean_system_remove_additional_folders(request, clean_system):
    """
    Use the global clean_system fixture and run additional teardown code to
    remove some special folders.

    For a better understanding - order of fixture calls:
    clean_system setup code
    clean_system_remove_additional_folders setup code
    clean_system_remove_additional_folders teardown code
    clean_system teardown code
    """
    def remove_additional_folders():
        if os.path.exists('inputpizzä'):
            utils.rmtree('inputpizzä')
        if os.path.exists('inputgreen'):
            utils.rmtree('inputgreen')
        if os.path.exists('inputbinary_files'):
            utils.rmtree('inputbinary_files')
        if os.path.exists('tests/custom_output_dir'):
            utils.rmtree('tests/custom_output_dir')
        if os.path.exists('inputpermissions'):
            utils.rmtree('inputpermissions')
    request.addfinalizer(remove_additional_folders)

@pytest.mark.usefixtures("clean_system_remove_additional_folders")
def test_generate_files_nontemplated_exception():
    with pytest.raises(exceptions.NonTemplatedInputDirException):
        generate.generate_files(
            context={'cookiecutter': {'food': 'pizza'}},
            repo_dir='tests/test-generate-files-nontemplated'
        )

def test_generate_files(self):
    generate.generate_files(
        context={
            'cookiecutter': {'food': 'pizzä'}
        },
        repo_dir='tests/test-generate-files'
    )
    self.assertTrue(os.path.isfile('inputpizzä/simple.txt'))
    simple_text = io.open('inputpizzä/simple.txt', 'rt', encoding='utf-8').read()
    self.assertEqual(simple_text, u'I eat pizzä')
<|endoftext|>