<commit_before>'''Author: Tyler Reddy

The purpose of this Python module is to provide utility code for handling spherical Voronoi Diagrams.'''

import scipy
import scipy.spatial
import numpy
import numpy.linalg

class Voronoi_Sphere_Surface:
    '''Voronoi diagrams on the surface of a sphere.

    Parameters
    ----------
    points: array, shape (npoints, 3)
        Coordinates of points to construct a Voronoi diagram on the surface of a sphere

    References
    ----------

    .. [Caroli] Caroli et al. (2009) INRIA 7004

    '''

    def __init__(self,points):
        self.original_point_array = points
        self.sphere_centroid = numpy.average(self.original_point_array,axis=0)
        self.estimated_sphere_radius = numpy.average(scipy.spatial.distance.cdist(self.original_point_array,self.sphere_centroid[numpy.newaxis,:]))

    def Delaunay_triangulation_spherical_surface(self):
        '''Delaunay tessellation of the points on the surface of the sphere. This is simply the 3D convex hull of the points. Returns a shape (N,3,3) array of points representing the vertices of the Delaunay triangulation on the sphere (i.e., N three-dimensional triangle vertex arrays).'''
        hull_instance = scipy.spatial.ConvexHull(self.original_point_array)
        list_points_vertices_Delaunay_triangulation = []
        for simplex in hull_instance.simplices: #for each simplex (face; presumably a triangle) of the convex hull
            convex_hull_triangular_facet_vertex_coordinates = self.original_point_array[simplex]
            assert convex_hull_triangular_facet_vertex_coordinates.shape == (3,3), "Triangular facet of convex hull should be a triangle in 3D space specified by coordinates in a shape (3,3) numpy array."
            list_points_vertices_Delaunay_triangulation.append(convex_hull_triangular_facet_vertex_coordinates)
        array_points_vertices_Delaunay_triangulation = numpy.array(list_points_vertices_Delaunay_triangulation)
        return array_points_vertices_Delaunay_triangulation

    def Voronoi_vertices_spherical_surface(self):
        '''Determine the Voronoi vertices on the surface of the sphere given the vertices of the Delaunay triangulation on the surface of the sphere. Inspired by response here: http://stackoverflow.com/a/22234783'''
        #follow spherical Delaunay calculation as above (can't call the class method from within the class itself, so reusing the code below; will probaly want to factor this out eventually)
        hull_instance = scipy.spatial.ConvexHull(self.original_point_array)
        list_points_vertices_Delaunay_triangulation = []
        for simplex in hull_instance.simplices: #for each simplex (face; presumably a triangle) of the convex hull
            convex_hull_triangular_facet_vertex_coordinates = self.original_point_array[simplex]
            assert convex_hull_triangular_facet_vertex_coordinates.shape == (3,3), "Triangular facet of convex hull should be a triangle in 3D space specified by coordinates in a shape (3,3) numpy array."
            list_points_vertices_Delaunay_triangulation.append(convex_hull_triangular_facet_vertex_coordinates)
        facet_coordinate_array_Delaunay_triangulation = numpy.array(list_points_vertices_Delaunay_triangulation) #produce (N,3,3) array of facet (triangle) Delaunay vertices on sphere

        #calculate the surface normal of each triangle as the vector cross product of two edges
        list_triangle_facet_normals = []
        for triangle_coord_array in facet_coordinate_array_Delaunay_triangulation:
            facet_normal = numpy.cross(triangle_coord_array[1] - triangle_coord_array[0],triangle_coord_array[2] - triangle_coord_array[0])
            facet_normal_magnitude = numpy.linalg.norm(facet_normal) #length of surface normal
            facet_normal_unit_vector = facet_normal / facet_normal_magnitude #vector of length 1 in same direction
            list_triangle_facet_normals.append(facet_normal_unit_vector)

        array_facet_normals = numpy.array(list_triangle_facet_normals) * self.estimated_sphere_radius #adjust for radius of sphere
        array_Voronoi_vertices = array_facet_normals

        return array_Voronoi_vertices

<commit_msg>-voronoi_utility.py: 	-started working on new method (Voronoi_polygons_spherical_surface) to identify the polygons in the spherical Voronoi diagram and generate a convenient data structure for handling them 	-refactored code a bit to prevent duplication / facilitate maintenance<commit_after>'''Author: Tyler Reddy

The purpose of this Python module is to provide utility code for handling spherical Voronoi Diagrams.'''

import scipy
import scipy.spatial
import numpy
import numpy.linalg

def produce_triangle_vertex_coordinate_array_Delaunay_sphere(input_point_array):
    '''Return shape (N,3,3) numpy array of the Delaunay triangle vertex coordinates on the surface of the sphere.'''
    hull_instance = scipy.spatial.ConvexHull(input_point_array)
    list_points_vertices_Delaunay_triangulation = []
    for simplex in hull_instance.simplices: #for each simplex (face; presumably a triangle) of the convex hull
        convex_hull_triangular_facet_vertex_coordinates = input_point_array[simplex]
        assert convex_hull_triangular_facet_vertex_coordinates.shape == (3,3), "Triangular facet of convex hull should be a triangle in 3D space specified by coordinates in a shape (3,3) numpy array."
        list_points_vertices_Delaunay_triangulation.append(convex_hull_triangular_facet_vertex_coordinates)
    array_points_vertices_Delaunay_triangulation = numpy.array(list_points_vertices_Delaunay_triangulation)
    return array_points_vertices_Delaunay_triangulation

def produce_array_Voronoi_vertices_on_sphere_surface(facet_coordinate_array_Delaunay_triangulation,sphere_radius):
    '''Return shape (N,3) array of coordinates for the vertices of the Voronoi diagram on the sphere surface given a shape (N,3,3) array of Delaunay triangulation vertices.'''
    assert facet_coordinate_array_Delaunay_triangulation.shape[1:] == (3,3), "facet_coordinate_array_Delaunay_triangulation should have shape (N,3,3)."
    #calculate the surface normal of each triangle as the vector cross product of two edges
    list_triangle_facet_normals = []
    for triangle_coord_array in facet_coordinate_array_Delaunay_triangulation:
        facet_normal = numpy.cross(triangle_coord_array[1] - triangle_coord_array[0],triangle_coord_array[2] - triangle_coord_array[0])
        facet_normal_magnitude = numpy.linalg.norm(facet_normal) #length of surface normal
        facet_normal_unit_vector = facet_normal / facet_normal_magnitude #vector of length 1 in same direction
        list_triangle_facet_normals.append(facet_normal_unit_vector)

    array_facet_normals = numpy.array(list_triangle_facet_normals) * sphere_radius #adjust for radius of sphere
    array_Voronoi_vertices = array_facet_normals
    assert array_Voronoi_vertices.shape[1] == 3, "The array of Voronoi vertices on the sphere should have shape (N,3)."
    return array_Voronoi_vertices

class Voronoi_Sphere_Surface:
    '''Voronoi diagrams on the surface of a sphere.

    Parameters
    ----------
    points: array, shape (npoints, 3)
        Coordinates of points to construct a Voronoi diagram on the surface of a sphere

    References
    ----------

    .. [Caroli] Caroli et al. (2009) INRIA 7004

    '''

    def __init__(self,points):
        self.original_point_array = points
        self.sphere_centroid = numpy.average(self.original_point_array,axis=0)
        self.estimated_sphere_radius = numpy.average(scipy.spatial.distance.cdist(self.original_point_array,self.sphere_centroid[numpy.newaxis,:]))

    def Delaunay_triangulation_spherical_surface(self):
        '''Delaunay tessellation of the points on the surface of the sphere. This is simply the 3D convex hull of the points. Returns a shape (N,3,3) array of points representing the vertices of the Delaunay triangulation on the sphere (i.e., N three-dimensional triangle vertex arrays).'''
        array_points_vertices_Delaunay_triangulation = produce_triangle_vertex_coordinate_array_Delaunay_sphere(self.original_point_array)
        return array_points_vertices_Delaunay_triangulation

    def Voronoi_vertices_spherical_surface(self):
        '''Determine the Voronoi vertices on the surface of the sphere given the vertices of the Delaunay triangulation on the surface of the sphere. Inspired by response here: http://stackoverflow.com/a/22234783'''
        facet_coordinate_array_Delaunay_triangulation = produce_triangle_vertex_coordinate_array_Delaunay_sphere(self.original_point_array)
        array_Voronoi_vertices = produce_array_Voronoi_vertices_on_sphere_surface(facet_coordinate_array_Delaunay_triangulation,self.sphere_radius)
        return array_Voronoi_vertices

    def Voronoi_polygons_spherical_surface(self):
        '''Compute useful dictionary data structure relating to the polygons in the spherical Voronoi diagram and the original data points that they contain.'''
        pass

<|endoftext|>


<commit_before>from urllib.parse import urlencode
from modules.base.command import Command

class LetMeGoogleThatForYou(Command):
    def __init__(self, bot):
        super().__init__(bot)
        self.name = ['lmgtfy', 'google']
        self.help = 'Te ayuda a buscar algo en google'

    async def handle(self, message, cmd):
        if len(cmd.args) < 1:
            await cmd.answer('Formato: !lmgtfy <texto>')
            return

        texto = ' '.join(cmd.args)
        url = 'https://lmgtfy.com/?' + urlencode({'q': texto})

        await cmd.answer(url)
<commit_msg>Versión completa de lmgtfy<commit_after>from urllib.parse import urlencode
from modules.base.command import Command

class LetMeGoogleThatForYou(Command):
    def __init__(self, bot):
        super().__init__(bot)
        self.name = ['lmgtfy', 'google']
        self.help = 'Te ayuda a buscar algo en google'

    async def handle(self, message, cmd):
        if len(cmd.args) < 1:
            await cmd.answer('Formato: !lmgtfy <texto>')
            return

        texto = ' '.join(cmd.args)
        url = 'https://lmgtfy.com/?' + urlencode({'iie': texto})

        await cmd.answer(url)
<|endoftext|>


<commit_before># See http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/resources.html

from pyramid.events import (
    ContextFound,
    subscriber,
)
from pyramid.httpexceptions import (
    HTTPForbidden,
    HTTPInternalServerError,
    HTTPNotFound,
)
from pyramid.location import lineage
from pyramid.security import (
    Allow,
    Everyone,
    has_permission,
)
from pyramid.threadlocal import (
    get_current_request,
    manager,
)
from pyramid.view import view_config
from urllib import unquote
from uuid import UUID
from .objtemplate import ObjectTemplate
from .schema_utils import validate_request
from .storage import (
    DBSession,
    CurrentStatement,
    Resource,
)

def includeme(config):
    config.scan(__name__)

def make_subrequest(request, path):
    """ Make a subrequest

    Copies request environ data for authentication.

    May be better to just pull out the resource through traversal and manually
    perform security checks.
    """
    env = request.environ.copy()
    if path and '?' in path:
        path_info, query_string = path.split('?', 1)
        path_info = unquote(path_info)
    else:
        path_info = unquote(path)
        query_string = ''
    env['PATH_INFO'] = path_info
    env['QUERY_STRING'] = query_string
    subreq = request.__class__(env, method='GET', content_type=None,
                               body=b'')
    subreq.remove_conditional_headers()
    # XXX "This does not remove headers like If-Match"
    return subreq

def embed(request, path, result=None):
    # Should really be more careful about what gets included instead.
    # Cache cut response time from ~800ms to ~420ms.
    embedded = None
    if manager.stack:
        embedded = manager.stack[0].setdefault('encoded_embedded', {})
    if result is not None:
        embedded[path] = result
        return result
    result = embedded.get(path, None)
    if result is not None:
        return result
    subreq = make_subrequest(request, path)
    subreq.override_renderer = 'null_renderer'
    try:
        result = request.invoke_subrequest(subreq)
    except HTTPNotFound:
        raise KeyError(path)
    if embedded is not None:
        embedded[path] = result
    return result

def maybe_include_embedded(request, result):
    if len(manager.stack) != 1:
        return
    embedded = manager.stack[0].get('encoded_embedded', None)
    if embedded:
        result['_embedded'] = {'resources': embedded}

def no_body_needed(request):
    # No need for request data when rendering the single page html
    return request.environ.get('encoded.format') == 'html'

def validate_item_content(context, request):
    data = request.json_body
    if isinstance(context, Item):
        schema = context.__parent__.schema
    else:
        schema = context.schema
    if schema is None:
        request.validated = data
        return
    validate_request(schema, request)

def permission_checker(context, request):
    def checker(permission):
        return has_permission(permission, context, request)
    return checker

class Root(object):
    __name__ = ''
    __parent__ = None

    __acl__ = [
        (Allow, Everyone, 'list'),
        (Allow, 'group:admin', 'add'),
        (Allow, Everyone, 'view'),
        (Allow, 'group:admin', 'edit'),
        (Allow, Everyone, 'traverse'),
    ]

    def __init__(self, **properties):
        self.properties = properties
        self.collections = {}

    def __call__(self, request):
        return self

    def __getitem__(self, name):
        return self.collections[name]

    def __json__(self, request=None):
        return self.properties.copy()

    def location(self, name):
        """ Attach a collection at the location ``name``.

        Use as a decorator on Collection subclasses.
        """
        def decorate(factory):
            self.collections[name] = factory(self, name)
            return factory
        return decorate

class MergedLinksMeta(type):
    """ Merge the links from the subclass with its bases
    """
    def __init__(self, name, bases, attrs):
        super(MergedLinksMeta, self).__init__(name, bases, attrs)
        self.merged_links = {}
        for cls in reversed(self.mro()):
            links = vars(cls).get('links', None)
            if links is not None:
                self.merged_links.update(links)

class Item(object):
    __metaclass__ = MergedLinksMeta
    links = {
        'self': {'href': '{collection_uri}{_uuid}', 'templated': True},
        'collection': {'href': '{collection_uri}', 'templated': True},
        'profile': {'href': '/profiles/{item_type}.json', 'templated': True},
    }

    def __init__(self, collection, model, acl=None):
        self.__name__ = model.rid
        self.__parent__ = collection
        self.model = model
        if acl is not None:
            self.__acl__ = acl

    @property
    def properties(self):
        return self.model.statement.object

    def __json__(self, request):
        properties = self.properties.copy()
        links = self.expand_links(properties, request)
        if links is not None:
            properties['_links'] = links
        return properties

    def expand_links(self, properties, request):
        # Expand templated links
        ns = properties.copy()
        ns['collection_uri'] = request.resource_path(self.__parent__)
        ns['item_type'] = self.model.predicate
        ns['permission'] = permission_checker(self, get_current_request())
        compiled = ObjectTemplate(self.merged_links)
        links = compiled(ns)
        # Embed resources
        embedded = self.__parent__.embedded
        for rel, value in links.items():
            if rel not in embedded:
                continue
            if isinstance(value, list):
                for member in value:
                    embed(request, member['href'])
            else:
                embed(request, value['href'])
        return links

class CustomItemMeta(type):
    """ Give each collection its own Item class to enable
        specific view registration.
    """
    def __init__(self, name, bases, attrs):
        super(CustomItemMeta, self).__init__(name, bases, attrs)
        if 'Item' in attrs:
            return
        item_bases = tuple(base.Item for base in bases
                           if issubclass(base, Collection))
        qualname = getattr(self, '__qualname__', self.__name__)  # PY3 only
        item_attrs = {
            '__module__': self.__module__,
            '__name__': 'Item',
            '__qualname__': qualname + '.Item',
        }
        if 'links' in attrs:
            item_attrs['links'] = attrs['links']
        self.Item = type('Item', item_bases, item_attrs)

class Collection(object):
    __metaclass__ = CustomItemMeta
    Item = Item
    schema = None
    properties = None
    item_type = None
    embedded = {}

    def __init__(self, parent, name):
        self.__name__ = name
        self.__parent__ = parent
        if self.item_type is None:
            self.item_type = type(self).__name__.lower()

    def item_acl(self, model):
        return None

    def __getitem__(self, name):
        try:
            name = UUID(name)
        except ValueError:
            raise KeyError(name)
        try:
            item = self.get(name)
        except KeyError:
            # Just in case we get an unexpected KeyError
            # FIXME: exception logging.
            raise HTTPInternalServerError('Traversal raised KeyError')
        if item is None:
            raise KeyError(name)
        return item

    def get(self, name, default=None):
        key = (name, self.item_type)
        session = DBSession()
        model = session.query(CurrentStatement).get(key)
        if model is not None:
            return self.make_item(model)
        return default

    def make_item(self, model):
        acl = self.item_acl(model)
        return self.Item(self, model, acl)

    def add(self, properties):
        rid = properties.get('_uuid', None)
        session = DBSession()
        resource = Resource({self.item_type: properties}, rid)
        session.add(resource)
        item = self.make_item(resource.data[self.item_type])
        self.after_add(item)
        return item

    def after_add(self, item):
        '''Hook for subclasses'''

@view_config(context=Collection, permission='list', request_method='GET')
def collection_list(context, request):
    nrows = request.params.get('limit', None)
    if no_body_needed(request):
        return {}
    session = DBSession()
    query = session.query(CurrentStatement).filter(CurrentStatement.predicate == context.item_type).limit(nrows)
    items = []
    for model in query.all():
        item = context.make_item(model)
        properties = item.__json__(request)
        item_uri = request.resource_path(context, item.__name__)
        embed(request, item_uri, properties)
        items.append({'href': item_uri})
    collection_uri = request.resource_path(context)
    result = {
        '_embedded': {
            'items': items,
        },
        '_links': {
            'self': {'href': collection_uri},
            'items': items,
            '/rels/actions': [
                {
                    'name': 'add-antibody',
                    'title': 'Add antibody',
                    'method': 'POST',
                    'type': 'application/json',
                    'href': collection_uri,
                }
            ],
        },
    }

    if context.properties is not None:
        result.update(context.properties)

    maybe_include_embedded(request, result)
    return result

@view_config(context=Collection, validators=(validate_item_content,), permission='add', request_method='POST')
def collection_add(context, request):
    properties = request.validated
    item = context.add(properties)
    item_uri = request.resource_path(context, item.__name__)
    request.response.status = 201
    request.response.location = item_uri
    result = {
        'result': 'success',
        '_links': {
            'profile': {'href': '/profiles/result'},
            'items': [
                {'href': item_uri},
            ],
        },
    }
    return result

@subscriber(ContextFound)
def traversal_security(event):
    """ Check traversal was permitted at each step
    """
    request = event.request
    for resource in reversed(list(lineage(request.context))):
        result = has_permission('traverse', resource, request)
        if not result:
            msg = 'Unauthorized: traversal failed permission check'
            raise HTTPForbidden(msg, result=result)

@view_config(context=Item, permission='view', request_method='GET')
def item_view(context, request):
    if no_body_needed(request):
        return {}
    properties = context.__json__(request)
    maybe_include_embedded(request, properties)
    return properties

@view_config(context=Item, validators=[validate_item_content], permission='edit', request_method='POST')
def item_edit(context, request):
    uuid = context.model.rid
    collection = context.__parent__
    properties = request.validated
    properties['_uuid'] = uuid  # XXX Should this really be stored in object?
    context.model.resource[collection.item_type] = properties
    item_uri = request.resource_path(context.__parent__, context.__name__)
    request.response.status = 200
    result = {
        'result': 'success',
        '_links': {
            'profile': {'href': '/profiles/result'},
            'items': [
                {'href': item_uri},
            ],
        },
    }
    return result
<commit_msg>Remove item_acl now using custom Item class instead.<commit_after># See http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/resources.html

from pyramid.events import (
    ContextFound,
    subscriber,
)
from pyramid.httpexceptions import (
    HTTPForbidden,
    HTTPInternalServerError,
    HTTPNotFound,
)
from pyramid.location import lineage
from pyramid.security import (
    Allow,
    Everyone,
    has_permission,
)
from pyramid.threadlocal import (
    get_current_request,
    manager,
)
from pyramid.view import view_config
from urllib import unquote
from uuid import UUID
from .objtemplate import ObjectTemplate
from .schema_utils import validate_request
from .storage import (
    DBSession,
    CurrentStatement,
    Resource,
)

def includeme(config):
    config.scan(__name__)

def make_subrequest(request, path):
    """ Make a subrequest

    Copies request environ data for authentication.

    May be better to just pull out the resource through traversal and manually
    perform security checks.
    """
    env = request.environ.copy()
    if path and '?' in path:
        path_info, query_string = path.split('?', 1)
        path_info = unquote(path_info)
    else:
        path_info = unquote(path)
        query_string = ''
    env['PATH_INFO'] = path_info
    env['QUERY_STRING'] = query_string
    subreq = request.__class__(env, method='GET', content_type=None,
                               body=b'')
    subreq.remove_conditional_headers()
    # XXX "This does not remove headers like If-Match"
    return subreq

def embed(request, path, result=None):
    # Should really be more careful about what gets included instead.
    # Cache cut response time from ~800ms to ~420ms.
    embedded = None
    if manager.stack:
        embedded = manager.stack[0].setdefault('encoded_embedded', {})
    if result is not None:
        embedded[path] = result
        return result
    result = embedded.get(path, None)
    if result is not None:
        return result
    subreq = make_subrequest(request, path)
    subreq.override_renderer = 'null_renderer'
    try:
        result = request.invoke_subrequest(subreq)
    except HTTPNotFound:
        raise KeyError(path)
    if embedded is not None:
        embedded[path] = result
    return result

def maybe_include_embedded(request, result):
    if len(manager.stack) != 1:
        return
    embedded = manager.stack[0].get('encoded_embedded', None)
    if embedded:
        result['_embedded'] = {'resources': embedded}

def no_body_needed(request):
    # No need for request data when rendering the single page html
    return request.environ.get('encoded.format') == 'html'

def validate_item_content(context, request):
    data = request.json_body
    if isinstance(context, Item):
        schema = context.__parent__.schema
    else:
        schema = context.schema
    if schema is None:
        request.validated = data
        return
    validate_request(schema, request)

def permission_checker(context, request):
    def checker(permission):
        return has_permission(permission, context, request)
    return checker

class Root(object):
    __name__ = ''
    __parent__ = None

    __acl__ = [
        (Allow, Everyone, 'list'),
        (Allow, 'group:admin', 'add'),
        (Allow, Everyone, 'view'),
        (Allow, 'group:admin', 'edit'),
        (Allow, Everyone, 'traverse'),
    ]

    def __init__(self, **properties):
        self.properties = properties
        self.collections = {}

    def __call__(self, request):
        return self

    def __getitem__(self, name):
        return self.collections[name]

    def __json__(self, request=None):
        return self.properties.copy()

    def location(self, name):
        """ Attach a collection at the location ``name``.

        Use as a decorator on Collection subclasses.
        """
        def decorate(factory):
            self.collections[name] = factory(self, name)
            return factory
        return decorate

class MergedLinksMeta(type):
    """ Merge the links from the subclass with its bases
    """
    def __init__(self, name, bases, attrs):
        super(MergedLinksMeta, self).__init__(name, bases, attrs)
        self.merged_links = {}
        for cls in reversed(self.mro()):
            links = vars(cls).get('links', None)
            if links is not None:
                self.merged_links.update(links)

class Item(object):
    __metaclass__ = MergedLinksMeta
    links = {
        'self': {'href': '{collection_uri}{_uuid}', 'templated': True},
        'collection': {'href': '{collection_uri}', 'templated': True},
        'profile': {'href': '/profiles/{item_type}.json', 'templated': True},
    }

    def __init__(self, collection, model):
        self.__name__ = model.rid
        self.__parent__ = collection
        self.model = model

    @property
    def properties(self):
        return self.model.statement.object

    def __json__(self, request):
        properties = self.properties.copy()
        links = self.expand_links(properties, request)
        if links is not None:
            properties['_links'] = links
        return properties

    def expand_links(self, properties, request):
        # Expand templated links
        ns = properties.copy()
        ns['collection_uri'] = request.resource_path(self.__parent__)
        ns['item_type'] = self.model.predicate
        ns['permission'] = permission_checker(self, get_current_request())
        compiled = ObjectTemplate(self.merged_links)
        links = compiled(ns)
        # Embed resources
        embedded = self.__parent__.embedded
        for rel, value in links.items():
            if rel not in embedded:
                continue
            if isinstance(value, list):
                for member in value:
                    embed(request, member['href'])
            else:
                embed(request, value['href'])
        return links

class CustomItemMeta(type):
    """ Give each collection its own Item class to enable
        specific view registration.
    """
    def __init__(self, name, bases, attrs):
        super(CustomItemMeta, self).__init__(name, bases, attrs)
        if 'Item' in attrs:
            return
        item_bases = tuple(base.Item for base in bases
                           if issubclass(base, Collection))
        qualname = getattr(self, '__qualname__', self.__name__)  # PY3 only
        item_attrs = {
            '__module__': self.__module__,
            '__name__': 'Item',
            '__qualname__': qualname + '.Item',
        }
        if 'links' in attrs:
            item_attrs['links'] = attrs['links']
        self.Item = type('Item', item_bases, item_attrs)

class Collection(object):
    __metaclass__ = CustomItemMeta
    Item = Item
    schema = None
    properties = None
    item_type = None
    embedded = {}

    def __init__(self, parent, name):
        self.__name__ = name
        self.__parent__ = parent
        if self.item_type is None:
            self.item_type = type(self).__name__.lower()

    def __getitem__(self, name):
        try:
            name = UUID(name)
        except ValueError:
            raise KeyError(name)
        try:
            item = self.get(name)
        except KeyError:
            # Just in case we get an unexpected KeyError
            # FIXME: exception logging.
            raise HTTPInternalServerError('Traversal raised KeyError')
        if item is None:
            raise KeyError(name)
        return item

    def get(self, name, default=None):
        key = (name, self.item_type)
        session = DBSession()
        model = session.query(CurrentStatement).get(key)
        if model is not None:
            return self.Item(self, model)
        return default

    def add(self, properties):
        rid = properties.get('_uuid', None)
        session = DBSession()
        resource = Resource({self.item_type: properties}, rid)
        session.add(resource)
        model = resource.data[self.item_type]
        item = self.Item(self, model)
        self.after_add(item)
        return item

    def after_add(self, item):
        '''Hook for subclasses'''

@view_config(context=Collection, permission='list', request_method='GET')
def collection_list(context, request):
    nrows = request.params.get('limit', None)
    if no_body_needed(request):
        return {}
    session = DBSession()
    query = session.query(CurrentStatement).filter(CurrentStatement.predicate == context.item_type).limit(nrows)
    items = []
    for model in query.all():
        item = context.Item(context, model)
        properties = item.__json__(request)
        item_uri = request.resource_path(context, item.__name__)
        embed(request, item_uri, properties)
        items.append({'href': item_uri})
    collection_uri = request.resource_path(context)
    result = {
        '_embedded': {
            'items': items,
        },
        '_links': {
            'self': {'href': collection_uri},
            'items': items,
            '/rels/actions': [
                {
                    'name': 'add-antibody',
                    'title': 'Add antibody',
                    'method': 'POST',
                    'type': 'application/json',
                    'href': collection_uri,
                }
            ],
        },
    }

    if context.properties is not None:
        result.update(context.properties)

    maybe_include_embedded(request, result)
    return result

@view_config(context=Collection, validators=(validate_item_content,), permission='add', request_method='POST')
def collection_add(context, request):
    properties = request.validated
    item = context.add(properties)
    item_uri = request.resource_path(context, item.__name__)
    request.response.status = 201
    request.response.location = item_uri
    result = {
        'result': 'success',
        '_links': {
            'profile': {'href': '/profiles/result'},
            'items': [
                {'href': item_uri},
            ],
        },
    }
    return result

@subscriber(ContextFound)
def traversal_security(event):
    """ Check traversal was permitted at each step
    """
    request = event.request
    for resource in reversed(list(lineage(request.context))):
        result = has_permission('traverse', resource, request)
        if not result:
            msg = 'Unauthorized: traversal failed permission check'
            raise HTTPForbidden(msg, result=result)

@view_config(context=Item, permission='view', request_method='GET')
def item_view(context, request):
    if no_body_needed(request):
        return {}
    properties = context.__json__(request)
    maybe_include_embedded(request, properties)
    return properties

@view_config(context=Item, validators=[validate_item_content], permission='edit', request_method='POST')
def item_edit(context, request):
    uuid = context.model.rid
    collection = context.__parent__
    properties = request.validated
    properties['_uuid'] = uuid  # XXX Should this really be stored in object?
    context.model.resource[collection.item_type] = properties
    item_uri = request.resource_path(context.__parent__, context.__name__)
    request.response.status = 200
    result = {
        'result': 'success',
        '_links': {
            'profile': {'href': '/profiles/result'},
            'items': [
                {'href': item_uri},
            ],
        },
    }
    return result
<|endoftext|>


<commit_before>#!/bin/python
# -*- coding: utf-8 -*-
import json
import argparse
import requests
import unittest

class GoSpatialTester(unittest.TestCase):

	# @classmethod
	# def setUpClass(cls):
	# 	cls.myCustomer = myCustomer

	# @classmethod
	# def tearDownClass(cls):
	# 	print("delete")

	def test_api(self):

		print("[POST] NEW CUSTOMER")
		req = requests.post("http://localhost:8888/management/customer", params={"apikey":"7q1qcqmsxnvw"})
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		apikey = res['apikey']
		print("[ok]", apikey)

		print("[POST] NEW DATASOURCE")
		req = requests.post("http://localhost:8888/api/v1/layer", params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		ds = res["datasource"]
		print("[ok]", ds)

		print("[GET] DATASOURCE LIST")
		req = requests.get("http://localhost:8888/api/v1/layers", params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		res = req.json()
		self.assertEqual(apikey, res['apikey'])
		self.assertEqual(ds, res['datasources'][0])
		print("[ok]", ds)

		print("[GET] DATASOURCE")
		req = requests.get("http://localhost:8888/api/v1/layer/" + ds, params={"apikey": apikey})
		self.assertEqual(200, req.status_code)

		print("[POST] NEW FEATURE")
		payload = {
			"geometry": {
				"type": "Point",
				"coordinates": [10,-10]
			},
			"properties": {
				"name": "test point 1"
			}
		}
		req = requests.post("http://localhost:8888/api/v1/layer/" + ds + "/feature", params={"apikey": apikey}, data=json.dumps(payload))
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		self.assertEqual(ds, res["datasource"])
		print("[ok]", res["datasource"])

		print("[GET] FEATURE")
		req = requests.get("http://localhost:8888/api/v1/layer/" + ds + "/feature/0", params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		print("[ok]", ds, 0)

		print("[DELETE] Layer")
		req = requests.delete("http://localhost:8888/api/v1/layer/" + ds, params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		self.assertEqual(ds, res["datasource"])
		print("[ok]", res["datasource"])

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description='GoSpatial Unittester')
	parser.add_argument('-c', type=str, help='customer apikey')
	args = parser.parse_args()
	unittest.main(warnings='ignore')

<commit_msg>test fixed for authkey<commit_after>#!/bin/python
# -*- coding: utf-8 -*-
import json
import argparse
import requests
import unittest

class GoSpatialTester(unittest.TestCase):

	# @classmethod
	# def setUpClass(cls):
	# 	cls.myCustomer = myCustomer

	# @classmethod
	# def tearDownClass(cls):
	# 	print("delete")

	def test_api(self):

		print("[POST] NEW CUSTOMER")
		req = requests.post("http://localhost:8888/management/customer", params={"auth":"7q1qcqmsxnvw"})
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		apikey = res['apikey']
		print("[ok]", apikey)

		print("[POST] NEW DATASOURCE")
		req = requests.post("http://localhost:8888/api/v1/layer", params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		ds = res["datasource"]
		print("[ok]", ds)

		print("[GET] DATASOURCE LIST")
		req = requests.get("http://localhost:8888/api/v1/layers", params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		res = req.json()
		self.assertEqual(apikey, res['apikey'])
		self.assertEqual(ds, res['datasources'][0])
		print("[ok]", ds)

		print("[GET] DATASOURCE")
		req = requests.get("http://localhost:8888/api/v1/layer/" + ds, params={"apikey": apikey})
		self.assertEqual(200, req.status_code)

		print("[POST] NEW FEATURE")
		payload = {
			"geometry": {
				"type": "Point",
				"coordinates": [10,-10]
			},
			"properties": {
				"name": "test point 1"
			}
		}
		req = requests.post("http://localhost:8888/api/v1/layer/" + ds + "/feature", params={"apikey": apikey}, data=json.dumps(payload))
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		self.assertEqual(ds, res["datasource"])
		print("[ok]", res["datasource"])

		print("[GET] FEATURE")
		req = requests.get("http://localhost:8888/api/v1/layer/" + ds + "/feature/0", params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		print("[ok]", ds, 0)

		print("[DELETE] Layer")
		req = requests.delete("http://localhost:8888/api/v1/layer/" + ds, params={"apikey": apikey})
		self.assertEqual(200, req.status_code)
		res = json.loads(req.json())
		self.assertEqual(ds, res["datasource"])
		print("[ok]", res["datasource"])

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description='GoSpatial Unittester')
	parser.add_argument('-c', type=str, help='customer apikey')
	args = parser.parse_args()
	unittest.main(warnings='ignore')

<|endoftext|>


<commit_before>import os
import json
import unittest
from pyunio import pyunio

pyunio = pyunio.PyUnio(os.getcwd())

pyunio.use('httpbin')

params = {
	'name': 'James Bond'
}

def test_get():

	response = json.loads(pyunio.get('get', params).text)

	assert(response['args']['name'] == 'James Bond')

def test_post():

	response = json.loads(pyunio.post('post', params).text)

	assert(response['args']['name'] == 'James Bond')

def test_put():

	response = json.loads(pyunio.put('put', params).text)

	assert(response['args']['name'] == 'James Bond')

def test_delete():

	response = json.loads(pyunio.delete('delete', params).text)

	assert(response['args']['name'] == 'James Bond')<commit_msg>Updated for v0.1.4.<commit_after>import json
import unittest
from pyunio import pyunio

pyunio.use('httpbin')

params = {
            'body': {
                        'name': 'James Bond'
                    }
         }

def test_get():

    response = json.loads(pyunio.get('get', params).text)

    assert(response['args']['name'] == 'James Bond')

def test_post():

    response = json.loads(pyunio.post('post', params).text)

    assert(response['args']['name'] == 'James Bond')

def test_put():

    response = json.loads(pyunio.put('put', params).text)

    assert(response['args']['name'] == 'James Bond')

def test_delete():

    response = json.loads(pyunio.delete('delete', params).text)

    assert(response['args']['name'] == 'James Bond')
<|endoftext|>


<commit_before>'''Clashogram tests.'''
import unittest
from unittest.mock import MagicMock
import json
from clashogram import CoCAPI, ClanInfo, WarInfo, WarStats, MessageFactory, WarMonitor, TelegramNotifier

class ClanInfoTestCase(unittest.TestCase):
    def setUp(self):
        self.claninfo = ClanInfo({'location': {'name': 'Iran',
                                               'isCountry': 'true',
                                               'countryCode': 'IR'},
                                  'warWinStreak': 0})

    def test_location(self):
        assert self.claninfo.get_location() == 'Iran'

    def test_country_imoji(self):
        assert self.claninfo.get_country_flag_imoji() ==  '🇮🇷'

    def test_winstreak(self):
        assert self.claninfo.get_winstreak() == 0

class WarInfoTestCase(unittest.TestCase):
    def setUp(self):
        self.warinfo = WarInfo(json.loads(open('data/inWar_40.json', 'r').read()))
        self.op_member = {
            "tag": "#2GCR2YLP8",
            "name": "captain spock",
            "townhallLevel": 9,
            "mapPosition": 18,
            "opponentAttacks": 2,
            "bestOpponentAttack": {
                "attackerTag": "#G0QPL0LQ",
                "defenderTag": "#2GCR2YLP8",
                "stars": 3,
                "destructionPercentage": 100,
                "order": 78
            }
        }
        self.clan_member = {
            "tag": "#9QVR8R29C",
            "name": "VAHID",
            "townhallLevel": 7,
            "mapPosition": 35,
            "opponentAttacks": 2,
            "bestOpponentAttack": {
                "attackerTag": "#P0C92YP99",
                "defenderTag": "#9QVR8R29C",
                "stars": 3,
                "destructionPercentage": 100,
                "order": 3
            }
        }

    def test_start_time(self):
        assert self.warinfo.start_time == '20170603T191148.000Z'

    def test_team_size(self):
        assert self.warinfo.team_size == 40

    def test_get_ordered_attacks(self):
        ordered_attacks = self.warinfo.get_ordered_attacks()

        assert len(ordered_attacks) == 126

    def test_player_count(self):
        assert len(self.warinfo.players) == 80

    def test_get_player_attacks(self):
        player = self.warinfo.players['#2GCR2YLP8']

        assert self.warinfo.get_player_attacks(player) == []

    def test_get_player_info(self):
        self.assertRaises(Exception, '#2GCZZZZP8')

    def test_is_not_in_war(self):
        assert not self.warinfo.is_not_in_war()

    def test_is_in_preparation(self):
        assert not self.warinfo.is_in_preparation()

    def test_is_in_war(self):
        assert self.warinfo.is_in_war()

    def test_is_war_over(self):
        assert not self.warinfo.is_war_over()

    def test_is_clan_member(self):
        self.assertFalse(self.warinfo.is_clan_member(self.op_member))
        self.assertTrue(self.warinfo.is_clan_member(self.clan_member))

    def test_is_win(self):
        self.assertTrue(self.warinfo.is_win())

    def test_is_draw(self):
        self.assertFalse(self.warinfo.is_draw())

    def test_create_war_id(self):
        self.assertEqual(self.warinfo.create_war_id(), "#YVL0C8UY#JC0L922Y20170602T201148.000Z")

class WarStatsTestCase(unittest.TestCase):
    def setUp(self):
        warinfo = WarInfo(json.loads(open('data/warEnded_50.json', 'r').read()))
        self.stats = WarStats(warinfo)

    def test_first_attack_stats(self):
        stats = self.stats.calculate_war_stats_sofar(1)

        self.assertEqual(stats['clan_destruction'], 0)
        self.assertEqual(stats['op_destruction'], 1.76)
        self.assertEqual(stats['clan_stars'], 0)
        self.assertEqual(stats['op_stars'], 2)
        self.assertEqual(stats['clan_used_attacks'], 0)
        self.assertEqual(stats['op_used_attacks'], 1)

    def test_42th_attack_stats(self):
        stats = self.stats.calculate_war_stats_sofar(42)

        self.assertEqual(stats['clan_destruction'], 44.16)
        self.assertEqual(stats['op_destruction'], 26.56)
        self.assertEqual(stats['clan_stars'], 61)
        self.assertEqual(stats['op_stars'], 37)
        self.assertEqual(stats['clan_used_attacks'], 27)
        self.assertEqual(stats['op_used_attacks'], 15)

    def test_last_attack_stats(self):
        stats = self.stats.calculate_war_stats_sofar(162)

        self.assertEqual(stats['clan_destruction'], 96.72)
        self.assertEqual(stats['op_destruction'], 98.90)
        self.assertEqual(stats['clan_stars'], 142)
        self.assertEqual(stats['op_stars'], 147)
        self.assertEqual(stats['clan_used_attacks'], 87)
        self.assertEqual(stats['op_used_attacks'], 75)

class MessageFactoryTestCase(unittest.TestCase):
    pass

class WarMonitorTestCase(unittest.TestCase):
    def setUp(self):
        coc_api = CoCAPI(None)
        self.warinfo = WarInfo(json.loads(open('data/inWar_40.json', 'r').read()))
        our_claninfo = ClanInfo({'location': {'name': 'Iran',
                                              'isCountry': 'true',
                                              'countryCode': 'IR'},
                                 'warWinStreak': 0})
        coc_api.get_currentwar = MagicMock(return_value=self.warinfo)
        coc_api.get_claninfo = MagicMock(return_value=our_claninfo)
        notifier = TelegramNotifier(None, None)
        notifier.send = MagicMock()
        self.monitor = WarMonitor({}, coc_api, notifier)
        self.monitor.update(self.warinfo)

        self.clan_attack = {
            "attackerTag": "#98VVJ8LV8",
            "defenderTag": "#8CCLRP2JC",
            "stars": 3,
            "destructionPercentage": 100,
            "order": 10
        }

    def test_send_preparation_msg(self):
        self.monitor.send_preparation_msg()

        self.assertTrue(self.monitor.is_preparation_msg_sent())

    def test_send_war_msg(self):
        self.monitor.send_war_msg()

        self.assertTrue(self.monitor.is_war_msg_sent())

    def test_is_attack_msg_sent(self):
        self.assertTrue(self.monitor.is_attack_msg_sent(self.clan_attack))

    def test_get_attack_id(self):
        self.assertEqual(self.monitor.get_attack_id(self.clan_attack), 'attack98VVJ8LV88CCLRP2JC')

    def test_is_war_over_msg_sent(self):
        self.assertFalse(self.monitor.is_war_over_msg_sent(self.warinfo))

if __name__ == '__main__':
    unittest.main()<commit_msg>Add stars and destruction tets.<commit_after>'''Clashogram tests.'''
import unittest
from unittest.mock import MagicMock
import json
from clashogram import CoCAPI, ClanInfo, WarInfo, WarStats, MessageFactory, WarMonitor, TelegramNotifier

class ClanInfoTestCase(unittest.TestCase):
    def setUp(self):
        self.claninfo = ClanInfo({'location': {'name': 'Iran',
                                               'isCountry': 'true',
                                               'countryCode': 'IR'},
                                  'warWinStreak': 0})

    def test_location(self):
        assert self.claninfo.get_location() == 'Iran'

    def test_country_imoji(self):
        assert self.claninfo.get_country_flag_imoji() ==  '🇮🇷'

    def test_winstreak(self):
        assert self.claninfo.get_winstreak() == 0

class WarInfoTestCase(unittest.TestCase):
    def setUp(self):
        self.warinfo = WarInfo(json.loads(open('data/inWar_40.json', 'r').read()))
        self.op_member = {
            "tag": "#2GCR2YLP8",
            "name": "captain spock",
            "townhallLevel": 9,
            "mapPosition": 18,
            "opponentAttacks": 2,
            "bestOpponentAttack": {
                "attackerTag": "#G0QPL0LQ",
                "defenderTag": "#2GCR2YLP8",
                "stars": 3,
                "destructionPercentage": 100,
                "order": 78
            }
        }
        self.clan_member = {
            "tag": "#9QVR8R29C",
            "name": "VAHID",
            "townhallLevel": 7,
            "mapPosition": 35,
            "opponentAttacks": 2,
            "bestOpponentAttack": {
                "attackerTag": "#P0C92YP99",
                "defenderTag": "#9QVR8R29C",
                "stars": 3,
                "destructionPercentage": 100,
                "order": 3
            }
        }

    def test_start_time(self):
        assert self.warinfo.start_time == '20170603T191148.000Z'

    def test_team_size(self):
        assert self.warinfo.team_size == 40

    def test_get_ordered_attacks(self):
        ordered_attacks = self.warinfo.get_ordered_attacks()

        assert len(ordered_attacks) == 126

    def test_player_count(self):
        assert len(self.warinfo.players) == 80

    def test_get_player_attacks(self):
        player = self.warinfo.players['#2GCR2YLP8']

        assert self.warinfo.get_player_attacks(player) == []

    def test_get_player_info(self):
        self.assertRaises(Exception, '#2GCZZZZP8')

    def test_is_not_in_war(self):
        assert not self.warinfo.is_not_in_war()

    def test_is_in_preparation(self):
        assert not self.warinfo.is_in_preparation()

    def test_is_in_war(self):
        assert self.warinfo.is_in_war()

    def test_is_war_over(self):
        assert not self.warinfo.is_war_over()

    def test_is_clan_member(self):
        self.assertFalse(self.warinfo.is_clan_member(self.op_member))
        self.assertTrue(self.warinfo.is_clan_member(self.clan_member))

    def test_is_win(self):
        self.assertTrue(self.warinfo.is_win())

    def test_is_draw(self):
        self.assertFalse(self.warinfo.is_draw())

    def test_create_war_id(self):
        self.assertEqual(self.warinfo.create_war_id(), "#YVL0C8UY#JC0L922Y20170602T201148.000Z")

class WarStatsTestCase(unittest.TestCase):
    def setUp(self):
        warinfo = WarInfo(json.loads(open('data/warEnded_50.json', 'r').read()))
        self.stats = WarStats(warinfo)
        self.attack161 = {
            "destructionPercentage": 53,
            "attackerTag": "#9YUVL0CU",
            "order": 161,
            "stars": 2,
            "defenderTag": "#228U8G88L"
        }
        self.attack150 = {
            "destructionPercentage": 100,
            "attackerTag": "#2Q02GYCYV",
            "order": 150,
            "stars": 3,
            "defenderTag": "#2Y0C8YPYU"
        }

    def test_first_attack_stats(self):
        stats = self.stats.calculate_war_stats_sofar(1)

        self.assertEqual(stats['clan_destruction'], 0)
        self.assertEqual(stats['op_destruction'], 1.76)
        self.assertEqual(stats['clan_stars'], 0)
        self.assertEqual(stats['op_stars'], 2)
        self.assertEqual(stats['clan_used_attacks'], 0)
        self.assertEqual(stats['op_used_attacks'], 1)

    def test_42th_attack_stats(self):
        stats = self.stats.calculate_war_stats_sofar(42)

        self.assertEqual(stats['clan_destruction'], 44.16)
        self.assertEqual(stats['op_destruction'], 26.56)
        self.assertEqual(stats['clan_stars'], 61)
        self.assertEqual(stats['op_stars'], 37)
        self.assertEqual(stats['clan_used_attacks'], 27)
        self.assertEqual(stats['op_used_attacks'], 15)

    def test_last_attack_stats(self):
        stats = self.stats.calculate_war_stats_sofar(162)

        self.assertEqual(stats['clan_destruction'], 96.72)
        self.assertEqual(stats['op_destruction'], 98.90)
        self.assertEqual(stats['clan_stars'], 142)
        self.assertEqual(stats['op_stars'], 147)
        self.assertEqual(stats['clan_used_attacks'], 87)
        self.assertEqual(stats['op_used_attacks'], 75)

    def test_attack_destruction(self):
        self.assertEqual(self.stats.get_attack_new_destruction(self.attack161), 0)
        self.assertEqual(self.stats.get_attack_new_destruction(self.attack150), 3)

    def test_attack_new_stars(self):
        self.assertEqual(self.stats.get_attack_new_stars(self.attack161), 0)
        self.assertEqual(self.stats.get_attack_new_stars(self.attack150), 1)

class MessageFactoryTestCase(unittest.TestCase):
    pass

class WarMonitorTestCase(unittest.TestCase):
    def setUp(self):
        coc_api = CoCAPI(None)
        self.warinfo = WarInfo(json.loads(open('data/inWar_40.json', 'r').read()))
        our_claninfo = ClanInfo({'location': {'name': 'Iran',
                                              'isCountry': 'true',
                                              'countryCode': 'IR'},
                                 'warWinStreak': 0})
        coc_api.get_currentwar = MagicMock(return_value=self.warinfo)
        coc_api.get_claninfo = MagicMock(return_value=our_claninfo)
        notifier = TelegramNotifier(None, None)
        notifier.send = MagicMock()
        self.monitor = WarMonitor({}, coc_api, notifier)
        self.monitor.update(self.warinfo)

        self.clan_attack = {
            "attackerTag": "#98VVJ8LV8",
            "defenderTag": "#8CCLRP2JC",
            "stars": 3,
            "destructionPercentage": 100,
            "order": 10
        }

    def test_send_preparation_msg(self):
        self.monitor.send_preparation_msg()

        self.assertTrue(self.monitor.is_preparation_msg_sent())

    def test_send_war_msg(self):
        self.monitor.send_war_msg()

        self.assertTrue(self.monitor.is_war_msg_sent())

    def test_is_attack_msg_sent(self):
        self.assertTrue(self.monitor.is_attack_msg_sent(self.clan_attack))

    def test_get_attack_id(self):
        self.assertEqual(self.monitor.get_attack_id(self.clan_attack), 'attack98VVJ8LV88CCLRP2JC')

    def test_is_war_over_msg_sent(self):
        self.assertFalse(self.monitor.is_war_over_msg_sent(self.warinfo))

if __name__ == '__main__':
    unittest.main()<|endoftext|>


<commit_before>import json
import mock
import os
import requests
import subprocess
import sys
import time
import unittest

import docker
import on_startup

from test_utils import TestContainerRunner

class ContainerIntegrationTests(unittest.TestCase):

    def setUp(self):
        self.base_url = "http://localhost:{}/".format(
            test_container_runner.container_port
        )
        self.tilesets_url = '{}api/v1/tilesets/'.format(self.base_url)
        for i in range(60):  # probably overkill, but Travis is slow sometimes
            try:
                requests.get(self.tilesets_url)
                break
            except Exception:
                print('Still waiting for server...')
                time.sleep(1)
        else:
            self.fail('Server never came up')

    def assert_run(self, command, res=[r'']):
        output = subprocess.check_output(
            command.format(**os.environ),
            shell=True
        ).strip()
        for re in res:
            self.assertRegexpMatches(str(output), re)

    def test_home_page(self):
        response = requests.get(self.base_url)
        self.assertEqual(response.status_code, 200)
        self.assertIn(b"HiGlass - App", response.content)

    # Test if the data we specify in INPUT_JSON_URL gets ingested properly by
    # higlass server upon container startup
    def test_data_ingested(self):
        response = json.loads(requests.get(self.tilesets_url).content)
        self.assertEqual(response["count"], 5)

class StartupScriptTests(unittest.TestCase):

    def setUp(self):
        self.tilesets = []
        mock.patch.object(on_startup, "DATA_DIRECTORY", "/tmp/").start()

        with open("test-data/input.json") as f:
            input_dict = json.loads(f.read())
            self.cooler_file_url = input_dict["file_relationships"][0]
            self.bigwig_file_url = input_dict["file_relationships"][3]

        self.cooler_tileset = self._create_tileset(self.cooler_file_url)
        self.bigwig_tileset = self._create_tileset(self.bigwig_file_url)

    def tearDown(self):
        mock.patch.stopall()

    def _create_tileset(self, tileset_url):
        tileset = on_startup.Tileset({"file_url": tileset_url})
        self.tilesets.append(tileset)
        return tileset

    def test_tileset_filetype_is_set(self):
        self.assertEqual(self.cooler_tileset.file_type, "cooler")
        self.assertEqual(self.bigwig_tileset.file_type, "bigwig")

    def test_tileset_datatype_is_set(self):
        self.assertEqual(self.cooler_tileset.data_type, "matrix")
        self.assertEqual(self.bigwig_tileset.data_type, "vector")

    def test_tileset_is_bigwig(self):
        self.assertFalse(self.cooler_tileset.is_bigwig())
        self.assertTrue(self.bigwig_tileset.is_bigwig())

    def _tileset_repr_assertions(self, tileset):
        self.assertEqual(
            "Tileset: {} {} {}".format(
                tileset.file_path,
                tileset.file_type,
                tileset.data_type
            ),
            str(tileset)
        )

    def test_tileset_repr(self):
        for tileset in self.tilesets:
            self._tileset_repr_assertions(tileset)

    def test_tileset_file_downloaded(self):
        for tileset in self.tilesets:
            self.assertTrue(os.path.exists("/tmp/" + tileset.file_name))

    def test_tileset_ingest(self):
        with mock.patch("on_startup.call_command") as call_command_mock:
            self.cooler_tileset.ingest()
            self.bigwig_tileset.ingest()
            self.assertEqual(call_command_mock.call_count, 2)

    @mock.patch("django.setup")
    @mock.patch("on_startup.call_command")
    @mock.patch("on_startup._start_nginx")
    def test_module_invocation(
        self, start_nginx_mock, call_command_mock, django_setup_mock
    ):
        os.environ["INPUT_JSON_URL"] = "http://{}:{}/test-data/input.json".format(
            test_container_runner.test_fixture_server.ip,
            test_container_runner.test_fixture_server.port
        )
        on_startup.main()
        self.assertTrue(django_setup_mock.called)
        self.assertTrue(start_nginx_mock.called)
        self.assertEqual(call_command_mock.call_count, 5)

if __name__ == '__main__':
    test_container_runner = TestContainerRunner()
    with test_container_runner:
        unittest.main()
<commit_msg>str() on bytes doesn't do what you would think...; str(b"Silly monkeys") -> "b'Silly monkeys'"<commit_after>import json
import mock
import os
import requests
import subprocess
import sys
import time
import unittest

import docker
import on_startup

from test_utils import TestContainerRunner

class ContainerIntegrationTests(unittest.TestCase):

    def setUp(self):
        self.base_url = "http://localhost:{}/".format(
            test_container_runner.container_port
        )
        self.tilesets_url = '{}api/v1/tilesets/'.format(self.base_url)
        for i in range(60):  # probably overkill, but Travis is slow sometimes
            try:
                requests.get(self.tilesets_url)
                break
            except Exception:
                print('Still waiting for server...')
                time.sleep(1)
        else:
            self.fail('Server never came up')

    def assert_run(self, command, res=[r'']):
        output = subprocess.check_output(
            command.format(**os.environ),
            shell=True
        ).strip()
        for re in res:
            self.assertRegexpMatches(output.decode('utf-8'), re)

    def test_home_page(self):
        response = requests.get(self.base_url)
        self.assertEqual(response.status_code, 200)
        self.assertIn(b"HiGlass - App", response.content)

    # Test if the data we specify in INPUT_JSON_URL gets ingested properly by
    # higlass server upon container startup
    def test_data_ingested(self):
        response = json.loads(requests.get(self.tilesets_url).content)
        self.assertEqual(response["count"], 5)

class StartupScriptTests(unittest.TestCase):

    def setUp(self):
        self.tilesets = []
        mock.patch.object(on_startup, "DATA_DIRECTORY", "/tmp/").start()

        with open("test-data/input.json") as f:
            input_dict = json.loads(f.read())
            self.cooler_file_url = input_dict["file_relationships"][0]
            self.bigwig_file_url = input_dict["file_relationships"][3]

        self.cooler_tileset = self._create_tileset(self.cooler_file_url)
        self.bigwig_tileset = self._create_tileset(self.bigwig_file_url)

    def tearDown(self):
        mock.patch.stopall()

    def _create_tileset(self, tileset_url):
        tileset = on_startup.Tileset({"file_url": tileset_url})
        self.tilesets.append(tileset)
        return tileset

    def test_tileset_filetype_is_set(self):
        self.assertEqual(self.cooler_tileset.file_type, "cooler")
        self.assertEqual(self.bigwig_tileset.file_type, "bigwig")

    def test_tileset_datatype_is_set(self):
        self.assertEqual(self.cooler_tileset.data_type, "matrix")
        self.assertEqual(self.bigwig_tileset.data_type, "vector")

    def test_tileset_is_bigwig(self):
        self.assertFalse(self.cooler_tileset.is_bigwig())
        self.assertTrue(self.bigwig_tileset.is_bigwig())

    def _tileset_repr_assertions(self, tileset):
        self.assertEqual(
            "Tileset: {} {} {}".format(
                tileset.file_path,
                tileset.file_type,
                tileset.data_type
            ),
            str(tileset)
        )

    def test_tileset_repr(self):
        for tileset in self.tilesets:
            self._tileset_repr_assertions(tileset)

    def test_tileset_file_downloaded(self):
        for tileset in self.tilesets:
            self.assertTrue(os.path.exists("/tmp/" + tileset.file_name))

    def test_tileset_ingest(self):
        with mock.patch("on_startup.call_command") as call_command_mock:
            self.cooler_tileset.ingest()
            self.bigwig_tileset.ingest()
            self.assertEqual(call_command_mock.call_count, 2)

    @mock.patch("django.setup")
    @mock.patch("on_startup.call_command")
    @mock.patch("on_startup._start_nginx")
    def test_module_invocation(
        self, start_nginx_mock, call_command_mock, django_setup_mock
    ):
        os.environ["INPUT_JSON_URL"] = "http://{}:{}/test-data/input.json".format(
            test_container_runner.test_fixture_server.ip,
            test_container_runner.test_fixture_server.port
        )
        on_startup.main()
        self.assertTrue(django_setup_mock.called)
        self.assertTrue(start_nginx_mock.called)
        self.assertEqual(call_command_mock.call_count, 5)

if __name__ == '__main__':
    test_container_runner = TestContainerRunner()
    with test_container_runner:
        unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

from copy import deepcopy

from django.conf import settings
from django.db.models import F

# For each instance, cache "adjunct" objects -- frequently-accessed objects
# which change rarely -- by storing them in local memory. Track cache validity
# via a database timestamp on each instance (instance.adjunct_timestamp).
# Check the timestamp before returning any object from the cache; if it's stale
# invalidate all adjunct objects for the instance.
#
# When an adjunct object is modified (saved to the db or deleted), invalidate
# the appropriate instance's cache and update its timestamp. The timestamp
# update will cause the change to propagate to any other servers.

_adjuncts = {}

# ------------------------------------------------------------------------
# Interface functions

def permissions(user, instance, model_name=None):
    if settings.USE_OBJECT_CACHES:
        return _get_adjuncts(instance).permissions(user, model_name)
    else:
        return _permissions_from_db(user, instance, model_name)

def role_permissions(role, instance=None, model_name=None):
    if settings.USE_OBJECT_CACHES:
        if not instance:
            instance = role.instance
        return _get_adjuncts(instance).role_permissions(role.id, model_name)
    else:
        return _role_permissions_from_db(role, model_name)

def udf_defs(instance, model_name=None):
    if settings.USE_OBJECT_CACHES:
        return _get_adjuncts(instance).udf_defs(model_name)
    else:
        return _udf_defs_from_db(instance, model_name)

def clear_caches():
    global _adjuncts
    _adjuncts = {}

def invalidate_adjuncts(*args, **kwargs):
    # Called by 'save' and 'delete' signal handlers for adjunct objects
    if settings.USE_OBJECT_CACHES:
        adjunct_object = kwargs['instance']  # 'instance' is a Django term here
        instance = adjunct_object.instance
        if instance.id in _adjuncts:
            del _adjuncts[instance.id]
        increment_adjuncts_timestamp(instance)

def increment_adjuncts_timestamp(instance):
    # Increment the timestamp carefully.
    # Don't call save(), to avoid storing possibly-stale data in "instance".
    # Use a SQL increment, to prevent race conditions between servers.
    from treemap.models import Instance
    Instance.objects.filter(pk=instance.id)\
                    .update(adjuncts_timestamp=F('adjuncts_timestamp') + 1)
    instance.refresh_from_db()

# ------------------------------------------------------------------------
# Fetch info from database when not using cache

def _permissions_from_db(user, instance, model_name):
    if user is None or user.is_anonymous():
        role = instance.default_role
    else:
        role = user.get_role(instance)
    return _role_permissions_from_db(role, model_name)

def _role_permissions_from_db(role, model_name):
    if model_name:
        perms = role.fieldpermission_set.filter(model_name=model_name)
    else:
        perms = role.fieldpermission_set.all()
    return list(perms)

def _udf_defs_from_db(instance, model_name):
    from treemap.udf import UserDefinedFieldDefinition
    defs = UserDefinedFieldDefinition.objects.filter(instance=instance)
    if model_name:
        defs = defs.filter(model_type=model_name)
    return list(defs)

# ------------------------------------------------------------------------
# Fetch info from cache

def _get_adjuncts(instance):
    adjuncts = _adjuncts.get(instance.id)
    if not adjuncts or adjuncts.timestamp < instance.adjuncts_timestamp:
        adjuncts = _InstanceAdjuncts(instance)
        _adjuncts[instance.id] = adjuncts
    return adjuncts

class _InstanceAdjuncts:
    def __init__(self, instance):
        self._instance = instance
        self._user_role_ids = {}
        self._permissions = {}
        self._udf_defs = {}
        self.timestamp = instance.adjuncts_timestamp

    def permissions(self, user, model_name):
        if not self._user_role_ids:
            self._load_roles()
        if user and user.id in self._user_role_ids:
            role_id = self._user_role_ids[user.id]
        else:
            role_id = self._user_role_ids[None]
        return self.role_permissions(role_id, model_name)

    def role_permissions(self, role_id, model_name):
        if not self._permissions:
            self._load_permissions()
        perms = self._permissions.get((role_id, model_name))

        # We must always deepcopy the cached items before returning them,
        # to prevent inadvertent modifcations to the cached items
        return deepcopy(perms) if perms else []

    def udf_defs(self, model_name):
        if not self._udf_defs:
            self._load_udf_defs()
        defs = self._udf_defs.get(model_name)

        # We must always deepcopy the cached items before returning them,
        # to prevent inadvertent modifcations to the cached items
        return deepcopy(defs) if defs else []

    def _load_roles(self):
        from treemap.models import InstanceUser

        for iu in InstanceUser.objects.filter(instance=self._instance):
            self._user_role_ids[iu.user_id] = iu.role_id

        self._user_role_ids[None] = self._instance.default_role_id

    def _load_permissions(self):
        from treemap.audit import FieldPermission
        for fp in FieldPermission.objects.filter(instance=self._instance):
            dict = self._permissions
            self._append_value(dict, (fp.role_id, fp.model_name), fp)
            self._append_value(dict, (fp.role_id, None), fp)

    def _append_value(self, dict, key, value):
        if key not in dict:
            dict[key] = []
        dict[key].append(value)

    def _load_udf_defs(self):
        from treemap.udf import UserDefinedFieldDefinition
        qs = UserDefinedFieldDefinition.objects.filter(instance=self._instance)
        for udfd in qs:
            self._append_value(self._udf_defs, udfd.model_type, udfd)
            # Add to the "None" key for looking up UDF defs without model name
            self._append_value(self._udf_defs, None, udfd)
<commit_msg>Improve timestamp refreshing in object_caches.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

from copy import deepcopy

from django.conf import settings
from django.db.models import F

# For each instance, cache "adjunct" objects -- frequently-accessed objects
# which change rarely -- by storing them in local memory. Track cache validity
# via a database timestamp on each instance (instance.adjunct_timestamp).
# Check the timestamp before returning any object from the cache; if it's stale
# invalidate all adjunct objects for the instance.
#
# When an adjunct object is modified (saved to the db or deleted), invalidate
# the appropriate instance's cache and update its timestamp. The timestamp
# update will cause the change to propagate to any other servers.

_adjuncts = {}

# ------------------------------------------------------------------------
# Interface functions

def permissions(user, instance, model_name=None):
    if settings.USE_OBJECT_CACHES:
        return _get_adjuncts(instance).permissions(user, model_name)
    else:
        return _permissions_from_db(user, instance, model_name)

def role_permissions(role, instance=None, model_name=None):
    if settings.USE_OBJECT_CACHES:
        if not instance:
            instance = role.instance
        return _get_adjuncts(instance).role_permissions(role.id, model_name)
    else:
        return _role_permissions_from_db(role, model_name)

def udf_defs(instance, model_name=None):
    if settings.USE_OBJECT_CACHES:
        return _get_adjuncts(instance).udf_defs(model_name)
    else:
        return _udf_defs_from_db(instance, model_name)

def clear_caches():
    global _adjuncts
    _adjuncts = {}

def invalidate_adjuncts(*args, **kwargs):
    # Called by 'save' and 'delete' signal handlers for adjunct objects
    if settings.USE_OBJECT_CACHES:
        adjunct_object = kwargs['instance']  # 'instance' is a Django term here
        instance = adjunct_object.instance
        if instance.id in _adjuncts:
            del _adjuncts[instance.id]
        increment_adjuncts_timestamp(instance)

def increment_adjuncts_timestamp(instance):
    # Increment the timestamp carefully.
    # Don't call save(), to avoid storing possibly-stale data in "instance".
    # Use a SQL increment, to prevent race conditions between servers.
    from treemap.models import Instance
    qs = Instance.objects.filter(pk=instance.id)
    qs.update(adjuncts_timestamp=F('adjuncts_timestamp') + 1)

    # Update timestamp from DB to prevent saving stale timestamps
    instance.adjuncts_timestamp = qs[0].adjuncts_timestamp

# ------------------------------------------------------------------------
# Fetch info from database when not using cache

def _permissions_from_db(user, instance, model_name):
    if user is None or user.is_anonymous():
        role = instance.default_role
    else:
        role = user.get_role(instance)
    return _role_permissions_from_db(role, model_name)

def _role_permissions_from_db(role, model_name):
    if model_name:
        perms = role.fieldpermission_set.filter(model_name=model_name)
    else:
        perms = role.fieldpermission_set.all()
    return list(perms)

def _udf_defs_from_db(instance, model_name):
    from treemap.udf import UserDefinedFieldDefinition
    defs = UserDefinedFieldDefinition.objects.filter(instance=instance)
    if model_name:
        defs = defs.filter(model_type=model_name)
    return list(defs)

# ------------------------------------------------------------------------
# Fetch info from cache

def _get_adjuncts(instance):
    adjuncts = _adjuncts.get(instance.id)
    if not adjuncts or adjuncts.timestamp < instance.adjuncts_timestamp:
        adjuncts = _InstanceAdjuncts(instance)
        _adjuncts[instance.id] = adjuncts
    return adjuncts

class _InstanceAdjuncts:
    def __init__(self, instance):
        self._instance = instance
        self._user_role_ids = {}
        self._permissions = {}
        self._udf_defs = {}
        self.timestamp = instance.adjuncts_timestamp

    def permissions(self, user, model_name):
        if not self._user_role_ids:
            self._load_roles()
        if user and user.id in self._user_role_ids:
            role_id = self._user_role_ids[user.id]
        else:
            role_id = self._user_role_ids[None]
        return self.role_permissions(role_id, model_name)

    def role_permissions(self, role_id, model_name):
        if not self._permissions:
            self._load_permissions()
        perms = self._permissions.get((role_id, model_name))

        # We must always deepcopy the cached items before returning them,
        # to prevent inadvertent modifcations to the cached items
        return deepcopy(perms) if perms else []

    def udf_defs(self, model_name):
        if not self._udf_defs:
            self._load_udf_defs()
        defs = self._udf_defs.get(model_name)

        # We must always deepcopy the cached items before returning them,
        # to prevent inadvertent modifcations to the cached items
        return deepcopy(defs) if defs else []

    def _load_roles(self):
        from treemap.models import InstanceUser

        for iu in InstanceUser.objects.filter(instance=self._instance):
            self._user_role_ids[iu.user_id] = iu.role_id

        self._user_role_ids[None] = self._instance.default_role_id

    def _load_permissions(self):
        from treemap.audit import FieldPermission
        for fp in FieldPermission.objects.filter(instance=self._instance):
            dict = self._permissions
            self._append_value(dict, (fp.role_id, fp.model_name), fp)
            self._append_value(dict, (fp.role_id, None), fp)

    def _append_value(self, dict, key, value):
        if key not in dict:
            dict[key] = []
        dict[key].append(value)

    def _load_udf_defs(self):
        from treemap.udf import UserDefinedFieldDefinition
        qs = UserDefinedFieldDefinition.objects.filter(instance=self._instance)
        for udfd in qs:
            self._append_value(self._udf_defs, udfd.model_type, udfd)
            # Add to the "None" key for looking up UDF defs without model name
            self._append_value(self._udf_defs, None, udfd)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import sys
if not (sys.version_info.major == 3 and sys.version_info.minor > 5):
    print("Python version %s.%s not supported version 3.6 or above required - exiting" % (sys.version_info.major,sys.version_info.minor))
    sys.exit(1)

import os
import time
import shutil
for path in [os.getcwd(),"SchemaTerms","SchemaExamples"]:
  sys.path.insert( 1, path ) #Pickup libs from local  directories

if os.path.basename(os.getcwd()) != "schemaorg":
    print("\nScript should be run from within the 'schemaorg' directory! - Exiting\n")
    sys.exit(1)

for dir in ["util","docs","gcloud","data"]:
    if not os.path.isdir(dir):
        print("\nRequired directory '%s' not found - Exiting\n" % dir)
        sys.exit(1)

import glob
import re
import argparse
import rdflib
import jinja2

from sdotermsource import SdoTermSource
from sdoterm import *
from schemaexamples import SchemaExamples
from localmarkdown import Markdown
from schemaversion import *

SITENAME="Schema.org"
TEMPLATESDIR = "templates"

parser = argparse.ArgumentParser()
parser.add_argument("-a","--autobuild",default=False, action='store_true', help="clear output directory and build all components - overrides all other settings (except -examplesnum)")
parser.add_argument("-c","--clearfirst",default=False, action='store_true', help="clear output directory before creating contents")
parser.add_argument("-d","--docspages",default= [],action='append',nargs='*',  help="create docs page(repeatable) - ALL = all pages")
parser.add_argument("-e","--examplesnum",default=False, action='store_true',  help="Add missing example ids")
parser.add_argument("-f","--files",default= [],action='append',nargs='*',  help="create files(repeatable) - ALL = all files")
parser.add_argument("-o","--output", help="output site directory (default: ./site)")
parser.add_argument("-r","--runtests",default=False, action='store_true', help="run test scripts before creating contents")
parser.add_argument("-s","--static",default=False, action='store_true',  help="Refresh static docs in site image")
parser.add_argument("-t","--terms",default= [],action='append',nargs='*',  help="create page for term (repeatable) - ALL = all terms")
parser.add_argument("--release",default=False, action='store_true',  help="create page for term (repeatable) - ALL = all terms")
args = parser.parse_args()

TERMS = []
for ter in args.terms:
    TERMS.extend(ter)
PAGES = []
for pgs in args.docspages:
    PAGES.extend(pgs)
FILES = []
for fls in args.files:
    FILES.extend(fls)

if args.output:
    OUTPUTDIR = args.output
else:
    OUTPUTDIR = "site"
DOCSOUTPUTDIR = OUTPUTDIR + "/docs"

if args.autobuild or args.release:
    TERMS = ["ALL"]
    PAGES = ["ALL"]
    FILES = ["ALL"]

def clear():
    if args.clearfirst or args.autobuild:
        print("Clearing %s directory" % OUTPUTDIR)
        if os.path.isdir(OUTPUTDIR):
            for root, dirs, files in os.walk(OUTPUTDIR):
                for f in files:
                    os.unlink(os.path.join(root, f))
                for d in dirs:
                    shutil.rmtree(os.path.join(root, d))

###################################################
#RUN TESTS
###################################################
def runtests():
    import runtests
    if args.runtests or args.autobuild:
        print("Running test scripts before proceeding...\n")
        errorcount = runtests.main('./tests/')
        if errorcount:
            print("Errors returned: %d" % errorcount)
            sys.exit(errorcount)
        else:
            print("Tests succesful!\n")

DOCSDOCSDIR = "/docs"
TERMDOCSDIR = "/docs"
DOCSHREFSUFFIX=""
DOCSHREFPREFIX="/"
TERMHREFSUFFIX=""
TERMHREFPREFIX="/"

###################################################
#INITIALISE Directory
###################################################
def createMissingDir(dir):
    if not os.path.exists(dir):
        os.makedirs(dir)

def initdir():
    print("Building site in '%s' directory" % OUTPUTDIR)
    createMissingDir(OUTPUTDIR)
    clear()
    createMissingDir(OUTPUTDIR + "/docs")
    createMissingDir(OUTPUTDIR + "/releases/%s" % getVersion())

    gdir = OUTPUTDIR + "/gcloud"
    createMissingDir(gdir)

    print("\nCopying docs static files")
    cmd = "./util/copystaticdocs+insert.py"
    os.system(cmd)
    print("Done")

    print("\nPreparing GCloud files")
    cmd = "cp gcloud/*.yaml %s" % gdir
    os.system(cmd)
    print("Files copied")

    cmd = 'sed "s/{{ver}}/%s/g" %s/handlers-template.yaml > %s/handlers.yaml' % (getVersion(),gdir,gdir)
    print("Created handlers.yaml for version: %s" % getVersion())
    os.system(cmd)
    print("Done\n")

from shutil import *
def mycopytree(src, dst, symlinks=False, ignore=None):
    #copes with already existing directories
    names = os.listdir(src)
    if ignore is not None:
        ignored_names = ignore(src, names)
    else:
        ignored_names = set()

    if not os.path.isdir(dst):
        os.makedirs(dst)
    errors = []
    for name in names:
        if name in ignored_names:
            continue
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                mycopytree(srcname, dstname, symlinks, ignore)
            else:
                # Will raise a SpecialFileError for unsupported file types
                copy2(srcname, dstname)
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except Error as err:
            errors.extend(err.args[0])
        except EnvironmentError as why:
            errors.append((srcname, dstname, str(why)))
    try:
        copystat(src, dst)
    except OSError as why:
        if WindowsError is not None and isinstance(why, WindowsError):
            # Copying file access times may fail on Windows
            pass
        else:
            errors.extend((src, dst, str(why)))
    if errors:
        raise Error (errors)

###################################################
#MARKDOWN INITIALISE
###################################################
Markdown.setWikilinkCssClass("localLink")
Markdown.setWikilinkPrePath("/")
Markdown.setWikilinkPostPath("")

###################################################
#TERMS SOURCE LOAD
###################################################
LOADEDTERMS = False
def loadTerms():
    global LOADEDTERMS
    if not LOADEDTERMS:
        LOADEDTERMS = True
        print("Loading triples files")
        SdoTermSource.loadSourceGraph("default")
        print ("loaded %s triples - %s terms" % (len(SdoTermSource.sourceGraph()),len(SdoTermSource.getAllTerms())) )

###################################################
#EXAMPLES SOURCE LOAD
###################################################
LOADEDEXAMPLES = False
def loadExamples():

    global LOADEDEXAMPLES
    if not LOADEDEXAMPLES:
        SchemaExamples.loadExamplesFiles("default")
        print("Loaded %d examples " % (SchemaExamples.count()))

###################################################
#JINJA INITIALISATION
###################################################
jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATESDIR),
        extensions=['jinja2.ext.autoescape'], autoescape=True, cache_size=0)

def jinjaDebug(text):
    print("Jinja: %s" % text)
    return ''

jenv.filters['debug']=jinjaDebug

local_vars = {}
def set_local_var(local_vars, name, value):
  local_vars[name] = value
  return ''
jenv.globals['set_local_var'] = set_local_var

### Template rendering

def templateRender(template,extra_vars=None):
    #Basic varibles configuring UI
    tvars = {
        'local_vars': local_vars,
        'version': getVersion(),
        'versiondate': getCurrentVersionDate(),
        'sitename': SITENAME,
        'TERMHREFPREFIX': TERMHREFPREFIX,
        'TERMHREFSUFFIX': TERMHREFSUFFIX,
        'DOCSHREFPREFIX': DOCSHREFPREFIX,
        'DOCSHREFSUFFIX': DOCSHREFSUFFIX,
        'home_page': "False"
    }
    if extra_vars:
        tvars.update(extra_vars)

    template = jenv.get_template(template)
    return template.render(tvars)

###################################################
#JINJA INITIALISATION - End
###################################################
###################################################
#Comment Handling
###################################################

def StripHtmlTags(source):
    if source and len(source) > 0:
        return re.sub('<[^<]+?>', '', source)
    return ""

def ShortenOnSentence(source,lengthHint=250):
    if source and len(source) > lengthHint:
        source = source.strip()
        sentEnd = re.compile('[.!?]')
        sentList = sentEnd.split(source)
        com=""
        count = 0
        while count < len(sentList):
            if(count > 0 ):
                if len(com) < len(source):
                    com += source[len(com)]
            com += sentList[count]
            count += 1
            if count == len(sentList):
                if len(com) < len(source):
                    com += source[len(source) - 1]
            if len(com) > lengthHint:
                if len(com) < len(source):
                    com += source[len(com)]
                break

        if len(source) > len(com) + 1:
            com += ".."
        source = com
    return source

#Check / create file paths
CHECKEDPATHS =[]
def checkFilePath(path):
    if not path in CHECKEDPATHS:
        CHECKEDPATHS.append(path)
        if not path.startswith('/'):
            path = os.getcwd() + "/" + path
        try:
            os.makedirs(path)
        except OSError as e:
            if not os.path.isdir(path):
                raise e

###################################################
#BUILD INDIVIDUAL TERM PAGES
###################################################
def processTerms():
    import buildtermpages
    global TERMS
    if len(TERMS):
        print("Building term definition pages\n")
        loadTerms()
        loadExamples()
    buildtermpages.buildTerms(TERMS)

###################################################
#BUILD DYNAMIC DOCS PAGES
###################################################
def processDocs():
    global PAGES
    import buildocspages
    if len(PAGES):
        print("Building dynamic documentation pages\n")
        loadTerms()
        buildocspages.buildDocs(PAGES)

###################################################
#BUILD FILES
###################################################
def processFiles():
    global FILES
    import buildfiles
    if len(FILES):
        print("Building supporting files\n")
        loadTerms()
        loadExamples()
        buildfiles.buildFiles(FILES)
###################################################
#COPY CREATED RELEASE FILES into Data area
###################################################
def copyReleaseFiles():
    print("Copying release files for version %s to data/releases" % getVersion() )
    SRCDIR = './site/releases/%s' % getVersion()
    DESTDIR = './data/releases/%s' % getVersion()
    mycopytree(SRCDIR,DESTDIR)
    cmd= "git add %s" % DESTDIR
    os.system(cmd)

if __name__ == '__main__':
    print("Version: %s  Released: %s" % (getVersion(),getCurrentVersionDate()))
    copyReleaseFiles()
    sys.exit()
    if args.release:
        args.autobuild = True
        print("BUILDING RELEASE VERSION")
        time.sleep(2)
        print()
    if args.examplesnum or args.release:
        print("Checking Examples for assigned identifiers")
        time.sleep(2)
        print()
        cmd ="./SchemaExamples/utils/assign-example-ids.py"
        os.system(cmd)
        print()
    initdir()
    runtests()
    processTerms()
    processDocs()
    processFiles()
    if args.release:
        copyReleaseFiles()

<commit_msg>Removed debug<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import sys
if not (sys.version_info.major == 3 and sys.version_info.minor > 5):
    print("Python version %s.%s not supported version 3.6 or above required - exiting" % (sys.version_info.major,sys.version_info.minor))
    sys.exit(1)

import os
import time
import shutil
for path in [os.getcwd(),"SchemaTerms","SchemaExamples"]:
  sys.path.insert( 1, path ) #Pickup libs from local  directories

if os.path.basename(os.getcwd()) != "schemaorg":
    print("\nScript should be run from within the 'schemaorg' directory! - Exiting\n")
    sys.exit(1)

for dir in ["util","docs","gcloud","data"]:
    if not os.path.isdir(dir):
        print("\nRequired directory '%s' not found - Exiting\n" % dir)
        sys.exit(1)

import glob
import re
import argparse
import rdflib
import jinja2

from sdotermsource import SdoTermSource
from sdoterm import *
from schemaexamples import SchemaExamples
from localmarkdown import Markdown
from schemaversion import *

SITENAME="Schema.org"
TEMPLATESDIR = "templates"

parser = argparse.ArgumentParser()
parser.add_argument("-a","--autobuild",default=False, action='store_true', help="clear output directory and build all components - overrides all other settings (except -examplesnum)")
parser.add_argument("-c","--clearfirst",default=False, action='store_true', help="clear output directory before creating contents")
parser.add_argument("-d","--docspages",default= [],action='append',nargs='*',  help="create docs page(repeatable) - ALL = all pages")
parser.add_argument("-e","--examplesnum",default=False, action='store_true',  help="Add missing example ids")
parser.add_argument("-f","--files",default= [],action='append',nargs='*',  help="create files(repeatable) - ALL = all files")
parser.add_argument("-o","--output", help="output site directory (default: ./site)")
parser.add_argument("-r","--runtests",default=False, action='store_true', help="run test scripts before creating contents")
parser.add_argument("-s","--static",default=False, action='store_true',  help="Refresh static docs in site image")
parser.add_argument("-t","--terms",default= [],action='append',nargs='*',  help="create page for term (repeatable) - ALL = all terms")
parser.add_argument("--release",default=False, action='store_true',  help="create page for term (repeatable) - ALL = all terms")
args = parser.parse_args()

TERMS = []
for ter in args.terms:
    TERMS.extend(ter)
PAGES = []
for pgs in args.docspages:
    PAGES.extend(pgs)
FILES = []
for fls in args.files:
    FILES.extend(fls)

if args.output:
    OUTPUTDIR = args.output
else:
    OUTPUTDIR = "site"
DOCSOUTPUTDIR = OUTPUTDIR + "/docs"

if args.autobuild or args.release:
    TERMS = ["ALL"]
    PAGES = ["ALL"]
    FILES = ["ALL"]

def clear():
    if args.clearfirst or args.autobuild:
        print("Clearing %s directory" % OUTPUTDIR)
        if os.path.isdir(OUTPUTDIR):
            for root, dirs, files in os.walk(OUTPUTDIR):
                for f in files:
                    os.unlink(os.path.join(root, f))
                for d in dirs:
                    shutil.rmtree(os.path.join(root, d))

###################################################
#RUN TESTS
###################################################
def runtests():
    import runtests
    if args.runtests or args.autobuild:
        print("Running test scripts before proceeding...\n")
        errorcount = runtests.main('./tests/')
        if errorcount:
            print("Errors returned: %d" % errorcount)
            sys.exit(errorcount)
        else:
            print("Tests succesful!\n")

DOCSDOCSDIR = "/docs"
TERMDOCSDIR = "/docs"
DOCSHREFSUFFIX=""
DOCSHREFPREFIX="/"
TERMHREFSUFFIX=""
TERMHREFPREFIX="/"

###################################################
#INITIALISE Directory
###################################################
def createMissingDir(dir):
    if not os.path.exists(dir):
        os.makedirs(dir)

def initdir():
    print("Building site in '%s' directory" % OUTPUTDIR)
    createMissingDir(OUTPUTDIR)
    clear()
    createMissingDir(OUTPUTDIR + "/docs")
    createMissingDir(OUTPUTDIR + "/releases/%s" % getVersion())

    gdir = OUTPUTDIR + "/gcloud"
    createMissingDir(gdir)

    print("\nCopying docs static files")
    cmd = "./util/copystaticdocs+insert.py"
    os.system(cmd)
    print("Done")

    print("\nPreparing GCloud files")
    cmd = "cp gcloud/*.yaml %s" % gdir
    os.system(cmd)
    print("Files copied")

    cmd = 'sed "s/{{ver}}/%s/g" %s/handlers-template.yaml > %s/handlers.yaml' % (getVersion(),gdir,gdir)
    print("Created handlers.yaml for version: %s" % getVersion())
    os.system(cmd)
    print("Done\n")

from shutil import *
def mycopytree(src, dst, symlinks=False, ignore=None):
    #copes with already existing directories
    names = os.listdir(src)
    if ignore is not None:
        ignored_names = ignore(src, names)
    else:
        ignored_names = set()

    if not os.path.isdir(dst):
        os.makedirs(dst)
    errors = []
    for name in names:
        if name in ignored_names:
            continue
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                mycopytree(srcname, dstname, symlinks, ignore)
            else:
                # Will raise a SpecialFileError for unsupported file types
                copy2(srcname, dstname)
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except Error as err:
            errors.extend(err.args[0])
        except EnvironmentError as why:
            errors.append((srcname, dstname, str(why)))
    try:
        copystat(src, dst)
    except OSError as why:
        if WindowsError is not None and isinstance(why, WindowsError):
            # Copying file access times may fail on Windows
            pass
        else:
            errors.extend((src, dst, str(why)))
    if errors:
        raise Error (errors)

###################################################
#MARKDOWN INITIALISE
###################################################
Markdown.setWikilinkCssClass("localLink")
Markdown.setWikilinkPrePath("/")
Markdown.setWikilinkPostPath("")

###################################################
#TERMS SOURCE LOAD
###################################################
LOADEDTERMS = False
def loadTerms():
    global LOADEDTERMS
    if not LOADEDTERMS:
        LOADEDTERMS = True
        print("Loading triples files")
        SdoTermSource.loadSourceGraph("default")
        print ("loaded %s triples - %s terms" % (len(SdoTermSource.sourceGraph()),len(SdoTermSource.getAllTerms())) )

###################################################
#EXAMPLES SOURCE LOAD
###################################################
LOADEDEXAMPLES = False
def loadExamples():

    global LOADEDEXAMPLES
    if not LOADEDEXAMPLES:
        SchemaExamples.loadExamplesFiles("default")
        print("Loaded %d examples " % (SchemaExamples.count()))

###################################################
#JINJA INITIALISATION
###################################################
jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATESDIR),
        extensions=['jinja2.ext.autoescape'], autoescape=True, cache_size=0)

def jinjaDebug(text):
    print("Jinja: %s" % text)
    return ''

jenv.filters['debug']=jinjaDebug

local_vars = {}
def set_local_var(local_vars, name, value):
  local_vars[name] = value
  return ''
jenv.globals['set_local_var'] = set_local_var

### Template rendering

def templateRender(template,extra_vars=None):
    #Basic varibles configuring UI
    tvars = {
        'local_vars': local_vars,
        'version': getVersion(),
        'versiondate': getCurrentVersionDate(),
        'sitename': SITENAME,
        'TERMHREFPREFIX': TERMHREFPREFIX,
        'TERMHREFSUFFIX': TERMHREFSUFFIX,
        'DOCSHREFPREFIX': DOCSHREFPREFIX,
        'DOCSHREFSUFFIX': DOCSHREFSUFFIX,
        'home_page': "False"
    }
    if extra_vars:
        tvars.update(extra_vars)

    template = jenv.get_template(template)
    return template.render(tvars)

###################################################
#JINJA INITIALISATION - End
###################################################
###################################################
#Comment Handling
###################################################

def StripHtmlTags(source):
    if source and len(source) > 0:
        return re.sub('<[^<]+?>', '', source)
    return ""

def ShortenOnSentence(source,lengthHint=250):
    if source and len(source) > lengthHint:
        source = source.strip()
        sentEnd = re.compile('[.!?]')
        sentList = sentEnd.split(source)
        com=""
        count = 0
        while count < len(sentList):
            if(count > 0 ):
                if len(com) < len(source):
                    com += source[len(com)]
            com += sentList[count]
            count += 1
            if count == len(sentList):
                if len(com) < len(source):
                    com += source[len(source) - 1]
            if len(com) > lengthHint:
                if len(com) < len(source):
                    com += source[len(com)]
                break

        if len(source) > len(com) + 1:
            com += ".."
        source = com
    return source

#Check / create file paths
CHECKEDPATHS =[]
def checkFilePath(path):
    if not path in CHECKEDPATHS:
        CHECKEDPATHS.append(path)
        if not path.startswith('/'):
            path = os.getcwd() + "/" + path
        try:
            os.makedirs(path)
        except OSError as e:
            if not os.path.isdir(path):
                raise e

###################################################
#BUILD INDIVIDUAL TERM PAGES
###################################################
def processTerms():
    import buildtermpages
    global TERMS
    if len(TERMS):
        print("Building term definition pages\n")
        loadTerms()
        loadExamples()
    buildtermpages.buildTerms(TERMS)

###################################################
#BUILD DYNAMIC DOCS PAGES
###################################################
def processDocs():
    global PAGES
    import buildocspages
    if len(PAGES):
        print("Building dynamic documentation pages\n")
        loadTerms()
        buildocspages.buildDocs(PAGES)

###################################################
#BUILD FILES
###################################################
def processFiles():
    global FILES
    import buildfiles
    if len(FILES):
        print("Building supporting files\n")
        loadTerms()
        loadExamples()
        buildfiles.buildFiles(FILES)
###################################################
#COPY CREATED RELEASE FILES into Data area
###################################################
def copyReleaseFiles():
    print("Copying release files for version %s to data/releases" % getVersion() )
    SRCDIR = './site/releases/%s' % getVersion()
    DESTDIR = './data/releases/%s' % getVersion()
    mycopytree(SRCDIR,DESTDIR)
    cmd= "git add %s" % DESTDIR
    os.system(cmd)

if __name__ == '__main__':
    print("Version: %s  Released: %s" % (getVersion(),getCurrentVersionDate()))
    if args.release:
        args.autobuild = True
        print("BUILDING RELEASE VERSION")
        time.sleep(2)
        print()
    if args.examplesnum or args.release:
        print("Checking Examples for assigned identifiers")
        time.sleep(2)
        print()
        cmd ="./SchemaExamples/utils/assign-example-ids.py"
        os.system(cmd)
        print()
    initdir()
    runtests()
    processTerms()
    processDocs()
    processFiles()
    if args.release:
        copyReleaseFiles()

<|endoftext|>