<commit_before># Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
import json
import webob

from glance.api import policy
from glance.common import exception
from glance.common import utils
from glance.common import wsgi
import glance.db
import glance.domain
import glance.gateway
import glance.notifier
from glance.openstack.common import timeutils
import glance.schema
import glance.store

class ImageMembersController(object):
    def __init__(self, db_api=None, policy_enforcer=None, notifier=None,
                 store_api=None):
        self.db_api = db_api or glance.db.get_api()
        self.db_api.setup_db_env()
        self.policy = policy_enforcer or policy.Enforcer()
        self.notifier = notifier or glance.notifier.Notifier()
        self.store_api = store_api or glance.store
        self.gateway = glance.gateway.Gateway(self.db_api, self.store_api,
                                              self.notifier, self.policy)

    @utils.mutating
    def create(self, req, image_id, member_id):
        """
        Adds a membership to the image.
        :param req: the Request object coming from the wsgi layer
        :param image_id: the image identifier
        :param member_id: the member identifier
        :retval The response body is a mapping of the following form::

            {'member_id': <MEMBER>,
             'image_id': <IMAGE>,
             'status': <MEMBER_STATUS>
             'created_at': ..,
             'updated_at': ..}

        """
        image_repo = self.gateway.get_repo(req.context)
        image_member_factory = self.gateway\
                                   .get_image_member_factory(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            new_member = image_member_factory.new_image_member(image,
                                                               member_id)
            member = member_repo.add(new_member)

            self._update_store_acls(req, image)
            return member
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))

    @utils.mutating
    def update(self, req, image_id, member_id, status):
        """
        Adds a membership to the image.
        :param req: the Request object coming from the wsgi layer
        :param image_id: the image identifier
        :param member_id: the member identifier
        :retval The response body is a mapping of the following form::

            {'member_id': <MEMBER>,
             'image_id': <IMAGE>,
             'status': <MEMBER_STATUS>
             'created_at': ..,
             'updated_at': ..}

        """
        image_repo = self.gateway.get_repo(req.context)
        image_member_factory = self.gateway\
                                   .get_image_member_factory(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            member = member_repo.get(member_id)
            member.status = status
            member = member_repo.save(member)
            return member
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))
        except ValueError as e:
            raise webob.exc.HTTPBadRequest(explanation=unicode(e))

    def index(self, req, image_id):
        """
        Return a list of dictionaries indicating the members of the
        image, i.e., those tenants the image is shared with.

        :param req: the Request object coming from the wsgi layer
        :param image_id: The image identifier
        :retval The response body is a mapping of the following form::

            {'members': [
                {'member_id': <MEMBER>,
                 'image_id': <IMAGE>,
                 'status': <MEMBER_STATUS>
                 'created_at': ..,
                 'updated_at': ..}, ..
            ]}
        """
        image_repo = self.gateway.get_repo(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            members = []
            for member in member_repo.list():
                members.append(member)
            return dict(members=members)
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))

    @utils.mutating
    def delete(self, req, image_id, member_id):
        """
        Removes a membership from the image.
        """

        image_repo = self.gateway.get_repo(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            member = member_repo.get(member_id)
            member_repo.remove(member)
            self._update_store_acls(req, image)
            return webob.Response(body='', status=200)
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))

    def _update_store_acls(self, req, image):
        public = image.visibility == 'public'
        member_repo = image.get_member_repo()
        if image.locations:
            try:
                read_tenants = []
                write_tenants = []
                members = member_repo.list()
                if members:
                    for member in members:
                        read_tenants.append(member.member_id)
                for location in image.locations:
                    glance.store.set_acls(req.context, location, public=public,
                                          read_tenants=read_tenants,
                                          write_tenants=write_tenants)
            except exception.UnknownScheme:
                msg = _("Store for image not found: %s") % image_id
                raise webob.exc.HTTPBadRequest(explanation=msg,
                                               request=req,
                                               content_type='text/plain')

class RequestDeserializer(wsgi.JSONRequestDeserializer):

    def __init__(self):
        super(RequestDeserializer, self).__init__()

    def _get_request_body(self, request):
        output = super(RequestDeserializer, self).default(request)
        if 'body' not in output:
            msg = _('Body expected in request.')
            raise webob.exc.HTTPBadRequest(explanation=msg)
        return output['body']

    def create(self, request):
        body = self._get_request_body(request)
        try:
            member_id = body['member']
        except KeyError:
            msg = _("Member to be added not specified")
            raise webob.exc.HTTPBadRequest(explanation=msg)
        return dict(member_id=member_id)

    def update(self, request):
        body = self._get_request_body(request)
        try:
            status = body['status']
        except KeyError:
            msg = _("Status not specified")
            raise webob.exc.HTTPBadRequest(explanation=msg)
        return dict(status=status)

class ResponseSerializer(wsgi.JSONResponseSerializer):
    def __init__(self, schema=None):
        super(ResponseSerializer, self).__init__()
        self.schema = schema or get_schema()

    def _format_image_member(self, member):
        member_view = {}
        attributes = ['member_id', 'image_id', 'status']
        for key in attributes:
            member_view[key] = getattr(member, key)
        member_view['created_at'] = timeutils.isotime(member.created_at)
        member_view['updated_at'] = timeutils.isotime(member.updated_at)
        member_view['schema'] = '/v2/schemas/member'
        member_view = self.schema.filter(member_view)
        return member_view

    def create(self, response, image_member):
        image_member_view = self._format_image_member(image_member)
        body = json.dumps(image_member_view, ensure_ascii=False)
        response.unicode_body = unicode(body)
        response.content_type = 'application/json'

    def update(self, response, image_member):
        image_member_view = self._format_image_member(image_member)
        body = json.dumps(image_member_view, ensure_ascii=False)
        response.unicode_body = unicode(body)
        response.content_type = 'application/json'

    def index(self, response, image_members):
        image_members = image_members['members']
        image_members_view = []
        for image_member in image_members:
            image_member_view = self._format_image_member(image_member)
            image_members_view.append(image_member_view)
        totalview = dict(members=image_members_view)
        totalview['schema'] = '/v2/schemas/members'
        body = json.dumps(totalview, ensure_ascii=False)
        response.unicode_body = unicode(body)
        response.content_type = 'application/json'

_MEMBER_SCHEMA = {
    'member_id': {
        'type': 'string',
        'description': _('An identifier for the image member (tenantId)')
    },
    'image_id': {
        'type': 'string',
        'description': _('An identifier for the image'),
        'pattern': ('^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}'
                    '-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$'),
    },
    'created_at': {
        'type': 'string',
        'description': _('Date and time of image member creation'),
        #TODO(brian-rosmaita): our jsonschema library doesn't seem to like the
        # format attribute, figure out why (and also fix in images.py)
        #'format': 'date-time',
    },
    'updated_at': {
        'type': 'string',
        'description': _('Date and time of last modification of image member'),
        #'format': 'date-time',
    },
    'status': {
        'type': 'string',
        'description': _('The status of this image member'),
        'enum': [
            'pending',
            'accepted',
            'rejected'
        ]
    },
    'schema': {'type': 'string'}
}

def get_schema():
    properties = copy.deepcopy(_MEMBER_SCHEMA)
    schema = glance.schema.Schema('member', properties)
    return schema

def get_collection_schema():
    member_schema = get_schema()
    return glance.schema.CollectionSchema('members', member_schema)

def create_resource():
    """Image Members resource factory method"""
    deserializer = RequestDeserializer()
    serializer = ResponseSerializer()
    controller = ImageMembersController()
    return wsgi.Resource(controller, deserializer, serializer)
<commit_msg>Remove unused variable 'image_member_factory'<commit_after># Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
import json
import webob

from glance.api import policy
from glance.common import exception
from glance.common import utils
from glance.common import wsgi
import glance.db
import glance.domain
import glance.gateway
import glance.notifier
from glance.openstack.common import timeutils
import glance.schema
import glance.store

class ImageMembersController(object):
    def __init__(self, db_api=None, policy_enforcer=None, notifier=None,
                 store_api=None):
        self.db_api = db_api or glance.db.get_api()
        self.db_api.setup_db_env()
        self.policy = policy_enforcer or policy.Enforcer()
        self.notifier = notifier or glance.notifier.Notifier()
        self.store_api = store_api or glance.store
        self.gateway = glance.gateway.Gateway(self.db_api, self.store_api,
                                              self.notifier, self.policy)

    @utils.mutating
    def create(self, req, image_id, member_id):
        """
        Adds a membership to the image.
        :param req: the Request object coming from the wsgi layer
        :param image_id: the image identifier
        :param member_id: the member identifier
        :retval The response body is a mapping of the following form::

            {'member_id': <MEMBER>,
             'image_id': <IMAGE>,
             'status': <MEMBER_STATUS>
             'created_at': ..,
             'updated_at': ..}

        """
        image_repo = self.gateway.get_repo(req.context)
        image_member_factory = self.gateway\
                                   .get_image_member_factory(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            new_member = image_member_factory.new_image_member(image,
                                                               member_id)
            member = member_repo.add(new_member)

            self._update_store_acls(req, image)
            return member
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))

    @utils.mutating
    def update(self, req, image_id, member_id, status):
        """
        Adds a membership to the image.
        :param req: the Request object coming from the wsgi layer
        :param image_id: the image identifier
        :param member_id: the member identifier
        :retval The response body is a mapping of the following form::

            {'member_id': <MEMBER>,
             'image_id': <IMAGE>,
             'status': <MEMBER_STATUS>
             'created_at': ..,
             'updated_at': ..}

        """
        image_repo = self.gateway.get_repo(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            member = member_repo.get(member_id)
            member.status = status
            member = member_repo.save(member)
            return member
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))
        except ValueError as e:
            raise webob.exc.HTTPBadRequest(explanation=unicode(e))

    def index(self, req, image_id):
        """
        Return a list of dictionaries indicating the members of the
        image, i.e., those tenants the image is shared with.

        :param req: the Request object coming from the wsgi layer
        :param image_id: The image identifier
        :retval The response body is a mapping of the following form::

            {'members': [
                {'member_id': <MEMBER>,
                 'image_id': <IMAGE>,
                 'status': <MEMBER_STATUS>
                 'created_at': ..,
                 'updated_at': ..}, ..
            ]}
        """
        image_repo = self.gateway.get_repo(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            members = []
            for member in member_repo.list():
                members.append(member)
            return dict(members=members)
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))

    @utils.mutating
    def delete(self, req, image_id, member_id):
        """
        Removes a membership from the image.
        """

        image_repo = self.gateway.get_repo(req.context)
        try:
            image = image_repo.get(image_id)
            member_repo = image.get_member_repo()
            member = member_repo.get(member_id)
            member_repo.remove(member)
            self._update_store_acls(req, image)
            return webob.Response(body='', status=200)
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=unicode(e))
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=unicode(e))

    def _update_store_acls(self, req, image):
        public = image.visibility == 'public'
        member_repo = image.get_member_repo()
        if image.locations:
            try:
                read_tenants = []
                write_tenants = []
                members = member_repo.list()
                if members:
                    for member in members:
                        read_tenants.append(member.member_id)
                for location in image.locations:
                    glance.store.set_acls(req.context, location, public=public,
                                          read_tenants=read_tenants,
                                          write_tenants=write_tenants)
            except exception.UnknownScheme:
                msg = _("Store for image not found: %s") % image_id
                raise webob.exc.HTTPBadRequest(explanation=msg,
                                               request=req,
                                               content_type='text/plain')

class RequestDeserializer(wsgi.JSONRequestDeserializer):

    def __init__(self):
        super(RequestDeserializer, self).__init__()

    def _get_request_body(self, request):
        output = super(RequestDeserializer, self).default(request)
        if 'body' not in output:
            msg = _('Body expected in request.')
            raise webob.exc.HTTPBadRequest(explanation=msg)
        return output['body']

    def create(self, request):
        body = self._get_request_body(request)
        try:
            member_id = body['member']
        except KeyError:
            msg = _("Member to be added not specified")
            raise webob.exc.HTTPBadRequest(explanation=msg)
        return dict(member_id=member_id)

    def update(self, request):
        body = self._get_request_body(request)
        try:
            status = body['status']
        except KeyError:
            msg = _("Status not specified")
            raise webob.exc.HTTPBadRequest(explanation=msg)
        return dict(status=status)

class ResponseSerializer(wsgi.JSONResponseSerializer):
    def __init__(self, schema=None):
        super(ResponseSerializer, self).__init__()
        self.schema = schema or get_schema()

    def _format_image_member(self, member):
        member_view = {}
        attributes = ['member_id', 'image_id', 'status']
        for key in attributes:
            member_view[key] = getattr(member, key)
        member_view['created_at'] = timeutils.isotime(member.created_at)
        member_view['updated_at'] = timeutils.isotime(member.updated_at)
        member_view['schema'] = '/v2/schemas/member'
        member_view = self.schema.filter(member_view)
        return member_view

    def create(self, response, image_member):
        image_member_view = self._format_image_member(image_member)
        body = json.dumps(image_member_view, ensure_ascii=False)
        response.unicode_body = unicode(body)
        response.content_type = 'application/json'

    def update(self, response, image_member):
        image_member_view = self._format_image_member(image_member)
        body = json.dumps(image_member_view, ensure_ascii=False)
        response.unicode_body = unicode(body)
        response.content_type = 'application/json'

    def index(self, response, image_members):
        image_members = image_members['members']
        image_members_view = []
        for image_member in image_members:
            image_member_view = self._format_image_member(image_member)
            image_members_view.append(image_member_view)
        totalview = dict(members=image_members_view)
        totalview['schema'] = '/v2/schemas/members'
        body = json.dumps(totalview, ensure_ascii=False)
        response.unicode_body = unicode(body)
        response.content_type = 'application/json'

_MEMBER_SCHEMA = {
    'member_id': {
        'type': 'string',
        'description': _('An identifier for the image member (tenantId)')
    },
    'image_id': {
        'type': 'string',
        'description': _('An identifier for the image'),
        'pattern': ('^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}'
                    '-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$'),
    },
    'created_at': {
        'type': 'string',
        'description': _('Date and time of image member creation'),
        #TODO(brian-rosmaita): our jsonschema library doesn't seem to like the
        # format attribute, figure out why (and also fix in images.py)
        #'format': 'date-time',
    },
    'updated_at': {
        'type': 'string',
        'description': _('Date and time of last modification of image member'),
        #'format': 'date-time',
    },
    'status': {
        'type': 'string',
        'description': _('The status of this image member'),
        'enum': [
            'pending',
            'accepted',
            'rejected'
        ]
    },
    'schema': {'type': 'string'}
}

def get_schema():
    properties = copy.deepcopy(_MEMBER_SCHEMA)
    schema = glance.schema.Schema('member', properties)
    return schema

def get_collection_schema():
    member_schema = get_schema()
    return glance.schema.CollectionSchema('members', member_schema)

def create_resource():
    """Image Members resource factory method"""
    deserializer = RequestDeserializer()
    serializer = ResponseSerializer()
    controller = ImageMembersController()
    return wsgi.Resource(controller, deserializer, serializer)
<|endoftext|>


<commit_before>
from werkzeug.contrib.securecookie import SecureCookie
from glashammer.utils import local, get_app, get_request

def get_session():
    return local.session

def setup_session(request):
    app = get_app()
    req = get_request()
    cookie_name = app.conf['sessions/cookie_name']
    session = SecureCookie.load_cookie(req, cookie_name,
                                           app.conf['sessions/secret'])
    local.session = session

def cleanup_sessions(response):
    # save the session
    app = get_app()
    session = get_session()
    if session.should_save:
        cookie_name = app.conf['sessions/cookie_name']
        if session.get('pmt'):
            max_age = 60 * 60 * 24 * 31
            expires = time() + max_age
        else:
            max_age = expires = None
        session.save_cookie(response, cookie_name, max_age=max_age,
                            expires=expires, session_expires=expires)

def setup_sessions(app):
    app.add_config_var('sessions/cookie_name', 'glashammer_session', str)
    app.add_config_var('sessions/secret', 'glashammer_secret', str)

    app.add_request_processor(setup_session)
    app.add_response_processor(cleanup_sessions)

setup_app = setup_sessions

<commit_msg>[project @ aafshar@gmail.com-20080727123021-68xe5l7rkwfxmstr] Minor. Don't get the request twice.<commit_after>
from werkzeug.contrib.securecookie import SecureCookie
from glashammer.utils import local, get_app, get_request

def get_session():
    return local.session

def setup_session(req):
    app = get_app()
    cookie_name = app.conf['sessions/cookie_name']
    session = SecureCookie.load_cookie(req, cookie_name,
                                           app.conf['sessions/secret'])
    local.session = session

def cleanup_sessions(response):
    # save the session
    app = get_app()
    session = get_session()
    if session.should_save:
        cookie_name = app.conf['sessions/cookie_name']
        if session.get('pmt'):
            max_age = 60 * 60 * 24 * 31
            expires = time() + max_age
        else:
            max_age = expires = None
        session.save_cookie(response, cookie_name, max_age=max_age,
                            expires=expires, session_expires=expires)

def setup_sessions(app):
    app.add_config_var('sessions/cookie_name', str, 'glashammer_session')
    app.add_config_var('sessions/secret', str, 'glashammer_secret')

    app.add_request_processor(setup_session)
    app.add_response_processor(cleanup_sessions)

setup_app = setup_sessions

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################
#
# ChriCar Beteiligungs- und Beratungs- GmbH
# Copyright (C) ChriCar Beteiligungs- und Beratungs- GmbH
# all rights reserved
# created 2008-07-05
#
###############################################
import time
from osv import fields,osv
import pooler

class res_partner_parent_company(osv.osv):
     _name  = "res.partner.parent_company"
     _order = "valid_from"

     def _children_ids(self, cr, uid, ids, name, args, context=None):
        res = {}
        for line in self.browse(cr, uid, ids, context=context):
           if line.partner_id.participation_ids:
             for participation in line.partner_id.participation_ids:
               res += participation.id
        return res

     _columns = \
         { 'name'               : fields.float   ('Share', digits=(16,2), required=True)
         , 'paid_in'            : fields.float   ('Paid in Share', digits=(16,2), required=True)
         , 'agio'               : fields.float   ('Agio', digits=(16,2), required=True)
         , 'percentage'         : fields.float   ('Percentage', digits=(9,6))
         , 'partner_id'         : fields.many2one('res.partner', 'Partner', required=True)
         , 'partner_parent_id'  : fields.many2one('res.partner', 'Parent', required=True)
         , 'valid_from'	        : fields.date    ('Valid From')
         , 'valid_until'	    : fields.date    ('Valid Until')
         , 'valid_fiscal_from'  : fields.date    ('Fiscal Valid From')
         , 'valid_fiscal_until' : fields.date    ('Fiscal Valid Until')
         , 'contract_date'      : fields.date    ('Contract Date')
         , 'contract_number'    : fields.char    ('Contract Number', size=32)
         , 'consolidation'      : fields.boolean ('Consolidation')
         , 'comment'            : fields.text    ('Notes')
         , 'active'             : fields.boolean ('Active')
         , 'state'              : fields.char    ('State', size=16)
         #, 'children_ids'       : fields.function(_children_ids, method=True, string='Participations'),
         #, 'children_ids'       :fields.one2many('res.partner.parent_company', 'parent_id','Children Items'),
         }
     _defaults = \
         { 'active'     : lambda *a: True
         , 'name'       : lambda *a: 0.0
         , 'paid_in'    : lambda *a: 0.0
         , 'agio'       : lambda *a: 0.0
         , 'percentage' : lambda *a: 0.0
         }

     def cmp_func(self):
         def cmp (a,b) :
             if a.partner_id.name == b.partner_id.name :
                 if a.valid_from == b.valid_from :
                     return 0
                 elif a.valid_from < b.valid_from :
                     return -1
                 else :
                     return 1
             elif a.partner_id.name < b.partner_id.name :
                 return -1
             else :
                 return 1

         return cmp
    # end def cmp_func
res_partner_parent_company()

class res_partner(osv.osv) :

    class one2many_parent_current(fields.one2many):
        def __init__(self, obj, fields_id, string='unknown', limit=None, **args) :
            if 'order' in args :
                self._order = list(reversed([col.replace(' ', '') for col in args['order'].split(',')]))
            else :
                self._order = []
            if 'search' in args :
                self._search = args['search']
            else :
                self._search = []
            (fields.one2many).__init__(self, obj, fields_id, string=string, limit=limit, **args)
        # end def __init__

        def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}) :
            res = {}
            _obj = obj.pool.get(self._obj)
            for id in ids : res[id] = []
            ids2 = _obj.search \
                ( cr, user
                , [(self._fields_id, 'in', ids)] + self._search
                , limit = self._limit
                )
            undecorated = _obj.read(cr, user, ids2, ['id'] + self._order, context)
            for key in self._order:
                decorated = [(d[key], d) for d in undecorated]
                decorated.sort()
                undecorated = [d for (k, d) in decorated]
            for r in _obj.browse(cr, user, [d['id'] for d in undecorated], context=context) :
                res [getattr(r, self._fields_id).id].append(r.id)
            return res
        # end def get

#        def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}):
#            res = {}
#            _obj = obj.pool.get (self._obj)
#            for id in ids : res[id] = []
#            ids2 = _obj.search \
#                ( cr
#                , user
#                , [(self._fields_id, 'in', ids), ('valid_until', '=', False)]
#                , limit=self._limit
#                )
#            for r in sorted(_obj.browse(cr, user, ids2, context=context), cmp=_obj.cmp_func()) :
#                res[r.partner_id.id].append(r.id)
#            return res
#        # end def get
   # end class one2many_parent_current

    class one2many_participation_current(fields.one2many):
        def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}):
            res = {}
            _obj = obj.pool.get (self._obj)
            for id in ids : res[id] = []
            ids2 = _obj.search \
                ( cr
                , user
                , [(self._fields_id, 'in', ids), ('valid_until', '=', False)]
                , limit=self._limit
                )
            for r in sorted(_obj.browse(cr, user, ids2, context=context), cmp=_obj.cmp_func()) :
                res[r.partner_parent_id.id].append(r.id)
            return res
        # end def get
   # end class one2many_parent_current

    _inherit = "res.partner"
    _columns = \
        { 'partner_ids'               : fields.one2many
            ('res.partner.parent_company','partner_id','Parent Companies')
        , 'partner_current_ids'       : one2many_parent_current
            ( 'res.partner.parent_company'
            , 'partner_id'
            , 'Parent Companies Current'
            , search = [('valid_until', '=', False)]
            , order  = 'partner_id,valid_from'
            )
        , 'participation_ids'         : fields.one2many
            ('res.partner.parent_company','partner_parent_id','Participations')
        , 'participation_current_ids' : one2many_participation_current
            ('res.partner.parent_company','partner_parent_id','Participations Current')
        }
res_partner()
<commit_msg>chricar_partner_parent_companies: sorting a la SQL<commit_after># -*- coding: utf-8 -*-
##############################################
#
# ChriCar Beteiligungs- und Beratungs- GmbH
# Copyright (C) ChriCar Beteiligungs- und Beratungs- GmbH
# all rights reserved
# created 2008-07-05
#
###############################################
import time
from osv import fields,osv
import pooler

class res_partner_parent_company(osv.osv):
     _name  = "res.partner.parent_company"
     _order = "valid_from"

     def _children_ids(self, cr, uid, ids, name, args, context=None):
        res = {}
        for line in self.browse(cr, uid, ids, context=context):
           if line.partner_id.participation_ids:
             for participation in line.partner_id.participation_ids:
               res += participation.id
        return res

     _columns = \
         { 'name'               : fields.float   ('Share', digits=(16,2), required=True)
         , 'paid_in'            : fields.float   ('Paid in Share', digits=(16,2), required=True)
         , 'agio'               : fields.float   ('Agio', digits=(16,2), required=True)
         , 'percentage'         : fields.float   ('Percentage', digits=(9,6))
         , 'partner_id'         : fields.many2one('res.partner', 'Partner', required=True)
         , 'partner_parent_id'  : fields.many2one('res.partner', 'Parent', required=True)
         , 'valid_from'	        : fields.date    ('Valid From')
         , 'valid_until'	    : fields.date    ('Valid Until')
         , 'valid_fiscal_from'  : fields.date    ('Fiscal Valid From')
         , 'valid_fiscal_until' : fields.date    ('Fiscal Valid Until')
         , 'contract_date'      : fields.date    ('Contract Date')
         , 'contract_number'    : fields.char    ('Contract Number', size=32)
         , 'consolidation'      : fields.boolean ('Consolidation')
         , 'comment'            : fields.text    ('Notes')
         , 'active'             : fields.boolean ('Active')
         , 'state'              : fields.char    ('State', size=16)
         #, 'children_ids'       : fields.function(_children_ids, method=True, string='Participations'),
         #, 'children_ids'       :fields.one2many('res.partner.parent_company', 'parent_id','Children Items'),
         }
     _defaults = \
         { 'active'     : lambda *a: True
         , 'name'       : lambda *a: 0.0
         , 'paid_in'    : lambda *a: 0.0
         , 'agio'       : lambda *a: 0.0
         , 'percentage' : lambda *a: 0.0
         }

     def cmp_func(self):
         def cmp (a,b) :
             if a.partner_id.name == b.partner_id.name :
                 if a.valid_from == b.valid_from :
                     return 0
                 elif a.valid_from < b.valid_from :
                     return -1
                 else :
                     return 1
             elif a.partner_id.name < b.partner_id.name :
                 return -1
             else :
                 return 1

         return cmp
    # end def cmp_func
res_partner_parent_company()

class res_partner(osv.osv) :

    class one2many_current(fields.one2many):
        def __init__(self, obj, fields_id, string='unknown', limit=None, **args) :
            order = []
            if 'order' in args :
                for col in args['order'].split(',') :
                    c = col.strip()
                    if ' ASC' in c.upper() :
                        order.append((c[0:c.index(' ')], False))
                    elif ' DESC' in c.upper() :
                        order.append((c[0:c.index(' ')], True))
                    else :
                        order.append((c, False))
                self._order = list(reversed(order))
            else :
                self._order = []
            if 'search' in args :
                self._search = args['search']
            else :
                self._search = []
            if 'set' in args :
                self._set = args['set']
            else :
                self._set = {}
            (fields.one2many).__init__(self, obj, fields_id, string=string, limit=limit, **args)
        # end def __init__

        def get (self, cr, obj, ids, name, user=None, offset=0, context=None, values={}) :
            res = {}
            _obj = obj.pool.get(self._obj)
            for id in ids : res[id] = []
            ids2 = _obj.search \
                ( cr, user
                , [(self._fields_id, 'in', ids)] + self._search
                , limit = self._limit
                )
            undecorated = []
            for r in _obj.browse(cr, user, ids2, context=context) :
                d = {}
                for key in ([('id', False)] + self._order) :
                    o = r
                    for m in key[0].split('.'):
                        o = getattr(o, m)
                    d[key[0]] = o
                undecorated.append(d)
            for key in self._order :
                decorated = [(d[key[0]], d) for d in undecorated]
                decorated.sort(reverse=key[1])
                undecorated = [d for (k, d) in decorated]
            for r in _obj.browse(cr, user, [d['id'] for d in undecorated], context=context) :
                res [getattr(r, self._fields_id).id].append(r.id)
            return res
        # end def get

        def set(self, cr, obj, id, field, values, user=None, context=None):
            for act in values :
                if act[0] == 0 : # "create"
                    for k, v in self._set.iteritems() :
                        act[2][k] = v
            return super(self.__class__,self).set(cr, obj, id, field, values, user, context)
        # end def set
   # end class one2many_current

    _inherit = "res.partner"
    _columns = \
        { 'partner_ids'               : fields.one2many
            ('res.partner.parent_company','partner_id','Parent Companies')
        , 'partner_current_ids'       : one2many_current
            ( 'res.partner.parent_company'
            , 'partner_id'
            , 'Parent Companies Current'
            , search = [('valid_until', '=', False)]
            , order  = 'partner_id.name,valid_from'
            )
        , 'participation_ids'         : fields.one2many
            ('res.partner.parent_company','partner_parent_id','Participations')
        , 'participation_current_ids' : one2many_current
            ( 'res.partner.parent_company'
            , 'partner_parent_id'
            , 'Participations Current'
            , search = [('valid_until', '=', False)]
            , order  = 'partner_id.name,valid_from'
            )
        }
res_partner()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
@package mi.dataset.parser.WFP_E_file_common
@file mi/dataset/parser/WFP_E_file_common
@author Emily Hahn, Mike Nicoletti, Maria Lutz
@brief A common parser for the E file type of the wire following profiler
"""

import re

from mi.core.log import get_logger
from mi.core.exceptions import SampleException, NotImplementedException, DatasetParserException
from mi.core.common import BaseEnum
from mi.dataset.dataset_parser import BufferLoadingParser

__author__ = 'Emily Hahn, Mike Nicoletti, Maria Lutz'
__license__ = 'Apache 2.0'

log = get_logger()

# This regex will be used to match the flags for one of the two bit patterns:
#  0001 0000 0000 0000 0001 0001 0000 0000  (regex: \x00\x01\x00{7}\x01\x00\x01\x00{4})
#  0001 0000 0000 0001 0000 0000 0000 0001 (regex: \x00\x01\x00{5}\x01\x00{7}\x01)
#  followed by 8 bytes of variable timestamp data (regex: [\x00-\xff]{8})
HEADER_REGEX = b'(\x00\x01\x00{5}[\x00-\x01]\x00[\x00-\x01]\x00[\x00-\x01]\x00{3}[\x00-\x01])([\x00-\xff]{8})'
HEADER_MATCHER = re.compile(HEADER_REGEX)

STATUS_START_REGEX = b'\xff\xff\xff[\xfa-\xff]'
STATUS_START_MATCHER = re.compile(STATUS_START_REGEX)

PROFILE_REGEX = b'\xff\xff\xff[\xfa-\xff][\x00-\xff]{12}'
PROFILE_MATCHER = re.compile(PROFILE_REGEX)

PROFILE_WITH_DECIM_FACTOR_REGEX = b'\xff\xff\xff[\xfa-\xff][\x00-\xff]{14}'
PROFILE_WITH_DECIM_FACTOR_MATCHER = re.compile(PROFILE_WITH_DECIM_FACTOR_REGEX)

# This regex will be used to match the flags for the coastal wfp_sio e engineering record:
# 0001 0000 0000 0000 0001 0001 0000 0000  (regex: \x00\x01\x00{7}\x01\x00\x01\x00{4})
# followed by 8 bytes of variable timestamp data (regex: [\x00-\xff]{8})
WFP_E_COASTAL_FLAGS_HEADER_REGEX = b'(\x00\x01\x00{7}\x01\x00\x01\x00{4})([\x00-\xff]{8})'
WFP_E_COASTAL_FLAGS_HEADER_MATCHER = re.compile(WFP_E_COASTAL_FLAGS_HEADER_REGEX)

# This regex will be used to match the flags for the global wfp_sio e engineering record:
# 0001 0000 0000 000P 0000 0000 0000 0001 (regex: \x00\x01\x00{5}[\x01|\x04|\x0c]\x00{7}\x01 - P is x01, x04, or x0c)
# The 2 bytes at position 6 used to be a boolean indicating the port being used, now it is
#   either 4 or 12, indicating the port the that data is coming from.  The regex is backward compatible.
# followed by 8 bytes of variable timestamp data (regex: [\x00-\xff]{8})
WFP_E_GLOBAL_FLAGS_HEADER_REGEX = b'(\x00\x01\x00{5}[\x01|\x04|\x0c]\x00{7}\x01)([\x00-\xff]{8})'
WFP_E_GLOBAL_FLAGS_HEADER_MATCHER = re.compile(WFP_E_GLOBAL_FLAGS_HEADER_REGEX)

# Includes indicator/timestamp and the data consists of variable 26 bytes
WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_REGEX = b'([\x00-\xff]{4})([\x00-\xff]{26})'
WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_MATCHER = re.compile(WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_REGEX)

# 4 bytes for the Engineering Data Record time stamp, 26 bytes for the global Engineering Data Record
WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_BYTES = 30

WFP_TIMESTAMP_BYTES = 4

HEADER_BYTES = 24

SAMPLE_BYTES = 26

STATUS_BYTES = 16
STATUS_BYTES_AUGMENTED = 18

class StateKey(BaseEnum):
    POSITION = "position"

class WfpEFileParser(BufferLoadingParser):

    def __init__(self,
                 config,
                 state,
                 stream_handle,
                 state_callback,
                 publish_callback,
                 *args, **kwargs):

        self._timestamp = 0.0
        self._record_buffer = []  # holds tuples of (record, state)
        self._read_state = {StateKey.POSITION: 0}
        super(WfpEFileParser, self).__init__(config,
                                             stream_handle,
                                             state,
                                             self.sieve_function,
                                             state_callback,
                                             publish_callback,
                                             *args, **kwargs)

        if state:
            self.set_state(state)
            if state[StateKey.POSITION] == 0:
                self._parse_header()
        else:
            self._parse_header()

    def sieve_function(self, raw_data):
        """
        Sort through the raw data to identify new blocks of data that need processing.
        This is needed instead of a regex because blocks are identified by position
        in this binary file.

        :param raw_data: Unprocessed data from the instrument to be parsed.
        """
        data_index = 0
        return_list = []
        raw_data_len = len(raw_data)
        remain_bytes = raw_data_len

        while data_index < raw_data_len:

            # check if this is a status or data sample message
            if remain_bytes >= STATUS_BYTES and STATUS_START_MATCHER.match(raw_data[data_index:data_index+4]):
                return_list.append((data_index, data_index + STATUS_BYTES))
                data_index += STATUS_BYTES
            elif remain_bytes >= SAMPLE_BYTES:
                return_list.append((data_index, data_index + SAMPLE_BYTES))
                data_index += SAMPLE_BYTES
            else:
                log.debug("not enough bytes to deal with")
                break

            remain_bytes = raw_data_len - data_index

        log.debug("returning sieve list %s", return_list)
        return return_list

    def set_state(self, state_obj):
        """
        initialize the state

        :param state_obj: The state to set.
        """
        log.trace("Attempting to set state to: %s", state_obj)
        if not isinstance(state_obj, dict):
            raise DatasetParserException("Invalid state structure")
        if not (StateKey.POSITION in state_obj):
            raise DatasetParserException("Invalid state keys")
        self._chunker.clean_all_chunks()
        self._record_buffer = []
        self._state = state_obj
        self._read_state = state_obj
        self._stream_handle.seek(state_obj[StateKey.POSITION])

    def _increment_state(self, increment):
        """
        Increment the parser position by the given increment in bytes.
        This indicates what has been read from the file, not what has
        been published.
        @ param increment number of bytes to increment parser position
        """
        self._read_state[StateKey.POSITION] += increment

    def _parse_header(self):
        """
        parse the flags and sensor / profiler start time from the header
        """
        # read the first bytes from the file
        header = self._stream_handle.read(HEADER_BYTES)
        match = HEADER_MATCHER.match(header)
        if not match:
            raise SampleException("File header does not match the header regex")

        # update the state to show we have read the header
        self._increment_state(HEADER_BYTES)

    def parse_record(self, record):
        """
        determine if this is a engineering or data record and parse
        FLORT and PARAD can copy paste this and insert their own data particle class
        needs extending for WFP_ENG

        :param record: Record to be parsed.
        """
        raise NotImplementedException("parse_record must be implemented")

    def parse_chunks(self):
        """
        Parse out any pending data chunks in the chunker. If
        it is a valid data piece, build a particle, update the position and
        timestamp. Go until the chunker has no more valid data.
        @retval a list of tuples with sample particles encountered in this
            parsing, plus the state. An empty list of nothing was parsed.
        """
        result_particles = []
        (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()

        while chunk is not None:
            result_particle = self.parse_record(chunk)
            if result_particle:
                result_particles.append(result_particle)

            (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()
            self._chunker.get_next_non_data(clean=True)

        return result_particles
<commit_msg>Issue #15058 - E file parser should handle x04 in header (#94)<commit_after>#!/usr/bin/env python

"""
@package mi.dataset.parser.WFP_E_file_common
@file mi/dataset/parser/WFP_E_file_common
@author Emily Hahn, Mike Nicoletti, Maria Lutz
@brief A common parser for the E file type of the wire following profiler
"""

import re

from mi.core.log import get_logger
from mi.core.exceptions import SampleException, NotImplementedException, DatasetParserException
from mi.core.common import BaseEnum
from mi.dataset.dataset_parser import BufferLoadingParser

__author__ = 'Emily Hahn, Mike Nicoletti, Maria Lutz'
__license__ = 'Apache 2.0'

log = get_logger()

# This regex will be used to match the flags for one of the two bit patterns:
#  0001 0000 0000 0000 0001 0001 0000 0000  (regex: \x00\x01\x00{7}\x01\x00\x01\x00{4})
#  0001 0000 0000 0001 0000 0000 0000 0001 (regex: \x00\x01\x00{5}\x01\x00{7}\x01)
#  followed by 8 bytes of variable timestamp data (regex: [\x00-\xff]{8})
HEADER_REGEX = b'(\x00\x01\x00{5}[\x01|\x04|\x0c]\x00[\x00-\x01]\x00[\x00-\x01]\x00{3}[\x00-\x01])([\x00-\xff]{8})'
HEADER_MATCHER = re.compile(HEADER_REGEX)

STATUS_START_REGEX = b'\xff\xff\xff[\xfa-\xff]'
STATUS_START_MATCHER = re.compile(STATUS_START_REGEX)

PROFILE_REGEX = b'\xff\xff\xff[\xfa-\xff][\x00-\xff]{12}'
PROFILE_MATCHER = re.compile(PROFILE_REGEX)

PROFILE_WITH_DECIM_FACTOR_REGEX = b'\xff\xff\xff[\xfa-\xff][\x00-\xff]{14}'
PROFILE_WITH_DECIM_FACTOR_MATCHER = re.compile(PROFILE_WITH_DECIM_FACTOR_REGEX)

# This regex will be used to match the flags for the coastal wfp_sio e engineering record:
# 0001 0000 0000 0000 0001 0001 0000 0000  (regex: \x00\x01\x00{7}\x01\x00\x01\x00{4})
# followed by 8 bytes of variable timestamp data (regex: [\x00-\xff]{8})
WFP_E_COASTAL_FLAGS_HEADER_REGEX = b'(\x00\x01\x00{7}\x01\x00\x01\x00{4})([\x00-\xff]{8})'
WFP_E_COASTAL_FLAGS_HEADER_MATCHER = re.compile(WFP_E_COASTAL_FLAGS_HEADER_REGEX)

# This regex will be used to match the flags for the global wfp_sio e engineering record:
# 0001 0000 0000 000P 0000 0000 0000 0001 (regex: \x00\x01\x00{5}[\x01|\x04|\x0c]\x00{7}\x01 - P is x01, x04, or x0c)
# The 2 bytes at position 6 used to be a boolean indicating the port being used, now it is
#   either 4 or 12, indicating the port the that data is coming from.  The regex is backward compatible.
# followed by 8 bytes of variable timestamp data (regex: [\x00-\xff]{8})
WFP_E_GLOBAL_FLAGS_HEADER_REGEX = b'(\x00\x01\x00{5}[\x01|\x04|\x0c]\x00{7}\x01)([\x00-\xff]{8})'
WFP_E_GLOBAL_FLAGS_HEADER_MATCHER = re.compile(WFP_E_GLOBAL_FLAGS_HEADER_REGEX)

# Includes indicator/timestamp and the data consists of variable 26 bytes
WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_REGEX = b'([\x00-\xff]{4})([\x00-\xff]{26})'
WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_MATCHER = re.compile(WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_REGEX)

# 4 bytes for the Engineering Data Record time stamp, 26 bytes for the global Engineering Data Record
WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_BYTES = 30

WFP_TIMESTAMP_BYTES = 4

HEADER_BYTES = 24

SAMPLE_BYTES = 26

STATUS_BYTES = 16
STATUS_BYTES_AUGMENTED = 18

class StateKey(BaseEnum):
    POSITION = "position"

class WfpEFileParser(BufferLoadingParser):

    def __init__(self,
                 config,
                 state,
                 stream_handle,
                 state_callback,
                 publish_callback,
                 *args, **kwargs):

        self._timestamp = 0.0
        self._record_buffer = []  # holds tuples of (record, state)
        self._read_state = {StateKey.POSITION: 0}
        super(WfpEFileParser, self).__init__(config,
                                             stream_handle,
                                             state,
                                             self.sieve_function,
                                             state_callback,
                                             publish_callback,
                                             *args, **kwargs)

        if state:
            self.set_state(state)
            if state[StateKey.POSITION] == 0:
                self._parse_header()
        else:
            self._parse_header()

    def sieve_function(self, raw_data):
        """
        Sort through the raw data to identify new blocks of data that need processing.
        This is needed instead of a regex because blocks are identified by position
        in this binary file.

        :param raw_data: Unprocessed data from the instrument to be parsed.
        """
        data_index = 0
        return_list = []
        raw_data_len = len(raw_data)
        remain_bytes = raw_data_len

        while data_index < raw_data_len:

            # check if this is a status or data sample message
            if remain_bytes >= STATUS_BYTES and STATUS_START_MATCHER.match(raw_data[data_index:data_index+4]):
                return_list.append((data_index, data_index + STATUS_BYTES))
                data_index += STATUS_BYTES
            elif remain_bytes >= SAMPLE_BYTES:
                return_list.append((data_index, data_index + SAMPLE_BYTES))
                data_index += SAMPLE_BYTES
            else:
                log.debug("not enough bytes to deal with")
                break

            remain_bytes = raw_data_len - data_index

        log.debug("returning sieve list %s", return_list)
        return return_list

    def set_state(self, state_obj):
        """
        initialize the state

        :param state_obj: The state to set.
        """
        log.trace("Attempting to set state to: %s", state_obj)
        if not isinstance(state_obj, dict):
            raise DatasetParserException("Invalid state structure")
        if not (StateKey.POSITION in state_obj):
            raise DatasetParserException("Invalid state keys")
        self._chunker.clean_all_chunks()
        self._record_buffer = []
        self._state = state_obj
        self._read_state = state_obj
        self._stream_handle.seek(state_obj[StateKey.POSITION])

    def _increment_state(self, increment):
        """
        Increment the parser position by the given increment in bytes.
        This indicates what has been read from the file, not what has
        been published.
        @ param increment number of bytes to increment parser position
        """
        self._read_state[StateKey.POSITION] += increment

    def _parse_header(self):
        """
        parse the flags and sensor / profiler start time from the header
        """
        # read the first bytes from the file
        header = self._stream_handle.read(HEADER_BYTES)
        match = HEADER_MATCHER.match(header)
        if not match:
            raise SampleException("File header does not match the header regex")

        # update the state to show we have read the header
        self._increment_state(HEADER_BYTES)

    def parse_record(self, record):
        """
        determine if this is a engineering or data record and parse
        FLORT and PARAD can copy paste this and insert their own data particle class
        needs extending for WFP_ENG

        :param record: Record to be parsed.
        """
        raise NotImplementedException("parse_record must be implemented")

    def parse_chunks(self):
        """
        Parse out any pending data chunks in the chunker. If
        it is a valid data piece, build a particle, update the position and
        timestamp. Go until the chunker has no more valid data.
        @retval a list of tuples with sample particles encountered in this
            parsing, plus the state. An empty list of nothing was parsed.
        """
        result_particles = []
        (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()

        while chunk is not None:
            result_particle = self.parse_record(chunk)
            if result_particle:
                result_particles.append(result_particle)

            (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index()
            self._chunker.get_next_non_data(clean=True)

        return result_particles
<|endoftext|>


<commit_before>"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""
Description: A library of scripts and constants to support rfam_xml_dumper

TO DO: - Set release version and release date automatically
"""

import datetime

# -----------------------------------------------------------------------------

# MAIN XML Fields
DB_NAME = "Rfam"  # DB name
DB_DESC = "A database for non-protein coding RNA families"  # DB description
DB_RELEASE = "13.0"  # release version
# DB_REL_DATE = datetime.date.today()  # datetime.date.today()

# DELIMITERS
RNA_TYPE_DEL = ';'
AUTH_DEL = ','

# ENTRY TYPES
MOTIF = 'M'
CLAN = 'C'
FAMILY = 'F'
GENOME = 'G'
MATCH = 'R'

# 9606 - human
# 10090 - mouse
# 7955 - zebrafish
# 3702 - Arabidopsis thaliana
# 6239 - Caenorhabditis elegans
# 7227 - Drosophila melanogaster
# 559292 - Saccharomyces cerevisiae S288c
# 4896 - Schizosaccharomyces pombe
# 511145 - Escherichia coli str. K-12 substr. MG1655

# MODEL ORGANISMS - popular species
POPULAR_SPECIES = (
    '9606', '10090', '7955', '3702', '6239', '7227', '559292', '4896', '511145')

# RFAM SEARCH QUERIES
REL_FIELDS = "SELECT rfam_release, rfam_release_date FROM version"

# -------------------------------RFAM ACCESSIONS--------------------------

# FETCHING

CLAN_ACC = "SELECT clan_acc FROM clan"

MOTIF_ACC = "SELECT motif_acc FROM motif"

FAM_ACC = "SELECT rfam_acc FROM family"

GENOME_ACC = "SELECT upid FROM genome WHERE num_families > 0"

# ---------------------------------RFAM FIELDS----------------------------

# Select all family related fields joining tables to fetch pubmed,go and so ids
FAM_FIELDS = ("SELECT f.rfam_acc as id, f.rfam_id as name, f.description,"
              "f.author, f.number_of_species as num_species,"
              "f.number_3d_structures as num_3d_structures, f.num_seed,"
              "f.num_full, f.type as rna_type, f.created, f.updated,"
              "group_concat(distinct concat(dl.db_id,\':\',dl.db_link)) as dbxrefs,"
              "group_concat(distinct concat(fl.pmid)) as pmids\n"
              "FROM family f JOIN full_region fr USING (rfam_acc)\n"
              "JOIN database_link dl using (rfam_acc)\n"
              "JOIN family_literature_reference fl USING (rfam_acc)\n"
              "WHERE f.rfam_acc = \'%s\' AND fr.is_significant=1\n"
              "AND (dl.db_id like \'GO\' OR dl.db_id like \'SO\')\n"
              "GROUP BY f.rfam_acc, f.rfam_id, f.description, f.author, f.number_of_species,"
              "f.number_3d_structures, f.num_seed, f.num_full, f.type, f.created, f.updated")

# Fetching clan fields from the db
CLAN_FIELDS = """
              SELECT c.clan_acc as id, c.id as name, c.description, c.created,
              c.updated, c.author, count(*) as num_families
              FROM clan c, clan_membership cm
              WHERE c.clan_acc=cm.clan_acc
              AND c.clan_acc='%s'
              """

# Fetching clan fields from the db
MOTIF_FIELDS = """
               SELECT m.motif_acc as id, m.motif_id as name, m.description,
               m.created, m.updated, m.author
               FROM motif m
               WHERE m.motif_acc='%s'
               """

# Fetching genome fields from the db
GENOME_FIELDS = """
                SELECT g.upid as id, g.scientific_name as name, g.assembly_acc, g.description,
                g.total_length, tx.tax_string, g.ncbi_id, g.num_rfam_regions,g.num_families,
                g.common_name, g.assembly_name, g.assembly_level, g.created, g.updated
                FROM genome g, taxonomy tx
                WHERE g.ncbi_id=tx.ncbi_id
                AND g.upid='%s'
                """

FULL_REGION_FIELDS = """
    SELECT
    fr.rfamseq_acc, fr.seq_start, fr.seq_end, fr.cm_start, fr.cm_end, fr.evalue_score,
    fr.bit_score, fr.type as alignment_type, fr.truncated, fr.rfam_acc,
    f.rfam_id, f.type as rna_type, rs.description as rfamseq_acc_description
    FROM full_region fr, family f, genseq gs, rfamseq rs
    WHERE fr.rfamseq_acc=gs.rfamseq_acc
    AND gs.rfamseq_acc=rs.rfamseq_acc
    AND fr.rfam_acc=f.rfam_acc
    AND fr.is_significant=1
    AND fr.type='full'
    AND gs.upid = '%s'
    AND gs.version = '14.0'
"""

FULL_REGION_SEEDS = """
    SELECT
    fr.rfamseq_acc, fr.seq_start, fr.seq_end, fr.cm_start, fr.cm_end, fr.evalue_score,
    fr.bit_score, fr.type as alignment_type, fr.truncated, fr.rfam_acc,
    f.rfam_id, f.type as rna_type, rs.description as rfamseq_acc_description
    FROM full_region fr, family f, genome g, rfamseq rs
    WHERE fr.rfamseq_acc=rs.rfamseq_acc
    AND g.ncbi_id=rs.ncbi_id
    AND fr.rfam_acc=f.rfam_acc
    AND fr.is_significant=1
    AND fr.type='seed'
    AND g.upid = '%s'
    and gs.version = '14.0'
"""

# -----------------------------CROSS REFERENCES---------------------------

# FAMILIES
# Fetch pdb ids related to a family accession
PDB_IDs_QUERY = """
                SELECT distinct pdb_id
                FROM pdb_full_region
                WHERE rfam_acc='%s'
                AND is_significant=1
                """

# Fetch ncbi ids related to a family accession
NCBI_IDs_QUERY = """
    SELECT tx.ncbi_id, tx.tax_string
    FROM taxonomy tx, full_region fr, rfamseq rs
    WHERE tx.ncbi_id=rs.ncbi_id
    AND fr.rfamseq_acc=rs.rfamseq_acc
    AND fr.is_significant=1
    AND fr.rfam_acc='%s'
    GROUP BY tx.ncbi_id
"""

# Fetch upids related to a family accession
FAMILY_UPIDS = """
               SELECT distinct upid
               FROM genseq gs, full_region fr
               WHERE gs.rfamseq_acc=fr.rfamseq_acc
               AND fr.rfam_acc='%s'
               AND fr.is_significant = 1
               AND gs.version='14.0'
               """

# Fetch clan based on rfam_acc
FAM_CLAN = """
           SELECT clan_acc
           FROM clan_membership
           WHERE rfam_acc='%s'
           """

# CLANS
# Fetch clan members
CLAN_FAMS = """
            SELECT rfam_acc
            from clan_membership
            WHERE clan_acc='%s'
            """

# MOTIFS
MOTIF_FAMS = """
             SELECT distinct rfam_acc
             FROM motif_matches
             WHERE motif_acc='%s'
             """

# GENOMES
GENOME_FAMS = """
              SELECT distinct rfam_acc
              FROM full_region fr, genseq gs
              WHERE fr.rfamseq_acc=gs.rfamseq_acc
              AND fr.is_significant = 1
              AND gs.upid='%s'
              AND gs.version='14.0'
              """

# -------------------------ADDITIONAL FIELDS------------------------------

# count # families associated with clan
NUM_FAMS_CLAN = """
                SELECT count(*) FROM clan_membership
                WHERE clan_acc='%s'
                """

# count # families associated with motif
NUM_FAMS_MOTIF = """
                 SELECT count(*) FROM motif_family_stats
                 WHERE motif_acc='%s'
                 """

COUNT_FULL_REGION = """
                    SELECT count(*)
                    FROM full_region fr, genseq gs
                    WHERE fr.rfamseq_acc = gs.rfamseq_acc
                    AND fr.is_significant = 1
                    AND fr.type='full'
                    AND gs.upid = '%s'
                    AND gs.version='14.0'
                    """

AU_ORCIDS = """
            SELECT orcid
            FROM author au, family_author fa
            WHERE au.author_id=fa.author_id
            AND rfam_acc='%s'
            AND orcid <> ''
            """

# -----------------------------------------------------------------------------

if __name__ == '__main__':
    pass
<commit_msg>Disable genseq version on seed sequences<commit_after>"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""
Description: A library of scripts and constants to support rfam_xml_dumper

TO DO: - Set release version and release date automatically
"""

import datetime

# -----------------------------------------------------------------------------

# MAIN XML Fields
DB_NAME = "Rfam"  # DB name
DB_DESC = "A database for non-protein coding RNA families"  # DB description
DB_RELEASE = "13.0"  # release version
# DB_REL_DATE = datetime.date.today()  # datetime.date.today()

# DELIMITERS
RNA_TYPE_DEL = ';'
AUTH_DEL = ','

# ENTRY TYPES
MOTIF = 'M'
CLAN = 'C'
FAMILY = 'F'
GENOME = 'G'
MATCH = 'R'

# 9606 - human
# 10090 - mouse
# 7955 - zebrafish
# 3702 - Arabidopsis thaliana
# 6239 - Caenorhabditis elegans
# 7227 - Drosophila melanogaster
# 559292 - Saccharomyces cerevisiae S288c
# 4896 - Schizosaccharomyces pombe
# 511145 - Escherichia coli str. K-12 substr. MG1655

# MODEL ORGANISMS - popular species
POPULAR_SPECIES = (
    '9606', '10090', '7955', '3702', '6239', '7227', '559292', '4896', '511145')

# RFAM SEARCH QUERIES
REL_FIELDS = "SELECT rfam_release, rfam_release_date FROM version"

# -------------------------------RFAM ACCESSIONS--------------------------

# FETCHING

CLAN_ACC = "SELECT clan_acc FROM clan"

MOTIF_ACC = "SELECT motif_acc FROM motif"

FAM_ACC = "SELECT rfam_acc FROM family"

GENOME_ACC = "SELECT upid FROM genome WHERE num_families > 0"

# ---------------------------------RFAM FIELDS----------------------------

# Select all family related fields joining tables to fetch pubmed,go and so ids
FAM_FIELDS = ("SELECT f.rfam_acc as id, f.rfam_id as name, f.description,"
              "f.author, f.number_of_species as num_species,"
              "f.number_3d_structures as num_3d_structures, f.num_seed,"
              "f.num_full, f.type as rna_type, f.created, f.updated,"
              "group_concat(distinct concat(dl.db_id,\':\',dl.db_link)) as dbxrefs,"
              "group_concat(distinct concat(fl.pmid)) as pmids\n"
              "FROM family f JOIN full_region fr USING (rfam_acc)\n"
              "JOIN database_link dl using (rfam_acc)\n"
              "JOIN family_literature_reference fl USING (rfam_acc)\n"
              "WHERE f.rfam_acc = \'%s\' AND fr.is_significant=1\n"
              "AND (dl.db_id like \'GO\' OR dl.db_id like \'SO\')\n"
              "GROUP BY f.rfam_acc, f.rfam_id, f.description, f.author, f.number_of_species,"
              "f.number_3d_structures, f.num_seed, f.num_full, f.type, f.created, f.updated")

# Fetching clan fields from the db
CLAN_FIELDS = """
              SELECT c.clan_acc as id, c.id as name, c.description, c.created,
              c.updated, c.author, count(*) as num_families
              FROM clan c, clan_membership cm
              WHERE c.clan_acc=cm.clan_acc
              AND c.clan_acc='%s'
              """

# Fetching clan fields from the db
MOTIF_FIELDS = """
               SELECT m.motif_acc as id, m.motif_id as name, m.description,
               m.created, m.updated, m.author
               FROM motif m
               WHERE m.motif_acc='%s'
               """

# Fetching genome fields from the db
GENOME_FIELDS = """
                SELECT g.upid as id, g.scientific_name as name, g.assembly_acc, g.description,
                g.total_length, tx.tax_string, g.ncbi_id, g.num_rfam_regions,g.num_families,
                g.common_name, g.assembly_name, g.assembly_level, g.created, g.updated
                FROM genome g, taxonomy tx
                WHERE g.ncbi_id=tx.ncbi_id
                AND g.upid='%s'
                """

FULL_REGION_FIELDS = """
    SELECT
    fr.rfamseq_acc, fr.seq_start, fr.seq_end, fr.cm_start, fr.cm_end, fr.evalue_score,
    fr.bit_score, fr.type as alignment_type, fr.truncated, fr.rfam_acc,
    f.rfam_id, f.type as rna_type, rs.description as rfamseq_acc_description
    FROM full_region fr, family f, genseq gs, rfamseq rs
    WHERE fr.rfamseq_acc=gs.rfamseq_acc
    AND gs.rfamseq_acc=rs.rfamseq_acc
    AND fr.rfam_acc=f.rfam_acc
    AND fr.is_significant=1
    AND fr.type='full'
    AND gs.upid = '%s'
    AND gs.version = '14.0'
"""

FULL_REGION_SEEDS = """
    SELECT
    fr.rfamseq_acc, fr.seq_start, fr.seq_end, fr.cm_start, fr.cm_end, fr.evalue_score,
    fr.bit_score, fr.type as alignment_type, fr.truncated, fr.rfam_acc,
    f.rfam_id, f.type as rna_type, rs.description as rfamseq_acc_description
    FROM full_region fr, family f, genome g, rfamseq rs
    WHERE fr.rfamseq_acc=rs.rfamseq_acc
    AND g.ncbi_id=rs.ncbi_id
    AND fr.rfam_acc=f.rfam_acc
    AND fr.is_significant=1
    AND fr.type='seed'
    AND g.upid = '%s'
"""

# -----------------------------CROSS REFERENCES---------------------------

# FAMILIES
# Fetch pdb ids related to a family accession
PDB_IDs_QUERY = """
                SELECT distinct pdb_id
                FROM pdb_full_region
                WHERE rfam_acc='%s'
                AND is_significant=1
                """

# Fetch ncbi ids related to a family accession
NCBI_IDs_QUERY = """
    SELECT tx.ncbi_id, tx.tax_string
    FROM taxonomy tx, full_region fr, rfamseq rs
    WHERE tx.ncbi_id=rs.ncbi_id
    AND fr.rfamseq_acc=rs.rfamseq_acc
    AND fr.is_significant=1
    AND fr.rfam_acc='%s'
    GROUP BY tx.ncbi_id
"""

# Fetch upids related to a family accession
FAMILY_UPIDS = """
               SELECT distinct upid
               FROM genseq gs, full_region fr
               WHERE gs.rfamseq_acc=fr.rfamseq_acc
               AND fr.rfam_acc='%s'
               AND fr.is_significant = 1
               AND gs.version='14.0'
               """

# Fetch clan based on rfam_acc
FAM_CLAN = """
           SELECT clan_acc
           FROM clan_membership
           WHERE rfam_acc='%s'
           """

# CLANS
# Fetch clan members
CLAN_FAMS = """
            SELECT rfam_acc
            from clan_membership
            WHERE clan_acc='%s'
            """

# MOTIFS
MOTIF_FAMS = """
             SELECT distinct rfam_acc
             FROM motif_matches
             WHERE motif_acc='%s'
             """

# GENOMES
GENOME_FAMS = """
              SELECT distinct rfam_acc
              FROM full_region fr, genseq gs
              WHERE fr.rfamseq_acc=gs.rfamseq_acc
              AND fr.is_significant = 1
              AND gs.upid='%s'
              AND gs.version='14.0'
              """

# -------------------------ADDITIONAL FIELDS------------------------------

# count # families associated with clan
NUM_FAMS_CLAN = """
                SELECT count(*) FROM clan_membership
                WHERE clan_acc='%s'
                """

# count # families associated with motif
NUM_FAMS_MOTIF = """
                 SELECT count(*) FROM motif_family_stats
                 WHERE motif_acc='%s'
                 """

COUNT_FULL_REGION = """
                    SELECT count(*)
                    FROM full_region fr, genseq gs
                    WHERE fr.rfamseq_acc = gs.rfamseq_acc
                    AND fr.is_significant = 1
                    AND fr.type='full'
                    AND gs.upid = '%s'
                    AND gs.version=14.0
                    """

AU_ORCIDS = """
            SELECT orcid
            FROM author au, family_author fa
            WHERE au.author_id=fa.author_id
            AND rfam_acc='%s'
            AND orcid <> ''
            """

# -----------------------------------------------------------------------------

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>from redash.models import db

if __name__ == '__main__':
    db.connect_db()

    with db.database.transaction():
        # Make sure all data sources names are unique.
        db.database.execute_sql("""UPDATE data_sources SET name = name || ' ' || id;""")
        # Add unique constraint on data_sources.name.
        db.database.execute_sql("ALTER TABLE data_sources ADD CONSTRAINT unique_name UNIQUE (name);")

    db.close_db(None)
<commit_msg>Rename only data sources with duplicates<commit_after>from redash.models import db

if __name__ == '__main__':
    db.connect_db()

    with db.database.transaction():
        # Make sure all data sources names are unique.
        db.database.execute_sql("""
        UPDATE data_sources
        SET name = new_names.name
        FROM (
            SELECT id, name || ' ' || id as name
            FROM (SELECT id, name, rank() OVER (PARTITION BY name ORDER BY created_at ASC) FROM data_sources) ds WHERE rank > 1
        ) AS new_names
        WHERE data_sources.id = new_names.id;
        """)
        # Add unique constraint on data_sources.name.
        db.database.execute_sql("ALTER TABLE data_sources ADD CONSTRAINT unique_name UNIQUE (name);")

    db.close_db(None)
<|endoftext|>


<commit_before>import json
import logging

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject, MISPAttribute

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url',
                             'AutonomousSystem': 'AS', 'File': 'sha256'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            links_by_id = {link['id']: link['description'] for link in neighboors['links']}
            for n in neighboors['objs']:
                yield n, links_by_id

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        for obs_to_add, links in self.get_neighboors(obs['id']):
            object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add)
            if object_misp_domain_ip:
                self.misp_event.add_object(object_misp_domain_ip)
            object_misp_url = self.__get_object_url(obs_to_add)
            if object_misp_url:
                self.misp_event.add_object(object_misp_url)
            if not object_misp_url and not object_misp_url:
                self.__get_attribute(obs_to_add, links)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def __get_attribute(self, obs_to_add, links):

        try:
            type_attr = self.misp_mapping[obs_to_add['type']]
            value = None
            if obs_to_add['type'] == 'File':
                value = obs_to_add['value'].split(':')[1]
            else:
                value = obs_to_add['value']
            attr = self.misp_event.add_attribute(value=value, type=type_attr)
            print(links)
            print(obs_to_add['id'])
            if obs_to_add['id'] in links:
                attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value'])
        except KeyError:
            logging.error('type not found %s' % obs_to_add['type'])
            return

        for t in obs_to_add['tags']:
            self.misp_event.add_attribute_tag(t['name'], attr['uuid'])

    def __get_object_domain_ip(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
                                           obj_to_add['value'])
            domain_ip_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False),
                                           self.attribute['value'])
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')

            return domain_ip_object

    def __get_object_url(self, obj_to_add):
        if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or (
            obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url'
        ):
            url_object = MISPObject('url')
            obj_relation = self.__get_relation(obj_to_add)
            if obj_relation:
                url_object.add_attribute(obj_relation, obj_to_add['value'])
            obj_relation = self.__get_relation(self.attribute, is_yeti_object=False)
            if obj_relation:
                url_object.add_attribute(obj_relation,
                                         self.attribute['value'])
            url_object.add_reference(self.attribute['uuid'], 'related_to')

            return url_object

    def __get_relation(self, obj, is_yeti_object=True):
        if is_yeti_object:
            type_attribute = self.misp_mapping[obj['type']]
        else:
            type_attribute = obj['type']
        if type_attribute == 'ip-src' or type_attribute == 'ip-dst':
            return 'ip'
        elif 'domain' == type_attribute:
            return 'domain'
        elif 'hostname' == type_attribute:
            return 'domain'
        elif type_attribute == 'url':
            return type_attribute

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return {'results': yeti_client.get_result()}
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<commit_msg>Update yeti.py<commit_after>import json
import logging

try:
    import pyeti
except ImportError:
    print("pyeti module not installed.")

from pymisp import MISPEvent, MISPObject, MISPAttribute

misperrors = {'error': 'Error'}

mispattributes = {'input': ['ip-src', 'ip-dst', 'hostname', 'domain'],
                  'format': 'misp_standard'
                  }
# possible module-types: 'expansion', 'hover' or both
moduleinfo = {'version': '1', 'author': 'Sebastien Larinier @sebdraven',
              'description': 'Query on yeti',
              'module-type': ['expansion', 'hover']}

moduleconfig = ['apikey', 'url']

class Yeti():

    def __init__(self, url, key,attribute):
        self.misp_mapping = {'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url',
                             'AutonomousSystem': 'AS', 'File': 'sha256'}
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            links_by_id = {link['id']: link['description'] for link in neighboors['links']}
            for n in neighboors['objs']:
                yield n, links_by_id[n['id']]

    def get_tags(self, value):
        obs = self.search(value)
        if obs:
            for t in obs['tags']:
                yield t

    def get_entity(self, obs_id):
        companies = self.yeti_client.observable_to_company(obs_id)
        actors = self.yeti_client.observable_to_actor(obs_id)
        campaigns = self.yeti_client.observable_to_campaign(obs_id)
        exploit_kit = self.yeti_client.observable_to_exploitkit(obs_id)
        exploit = self.yeti_client.observable_to_exploit(obs_id)
        ind = self.yeti_client.observable_to_indicator(obs_id)

        res = []
        res.extend(companies)
        res.extend(actors)
        res.extend(campaigns)
        res.extend(exploit)
        res.extend(exploit_kit)
        res.extend(ind)

        for r in res:
            yield r['name']

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])
        values = []
        types = []
        for obs_to_add, link in self.get_neighboors(obs['id']):
            object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add)
            if object_misp_domain_ip:
                self.misp_event.add_object(object_misp_domain_ip)
            object_misp_url = self.__get_object_url(obs_to_add)
            if object_misp_url:
                self.misp_event.add_object(object_misp_url)
            if not object_misp_url and not object_misp_url:
                self.__get_attribute(obs_to_add, link)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return results

    def __get_attribute(self, obs_to_add, links):

        try:
            type_attr = self.misp_mapping[obs_to_add['type']]
            value = None
            if obs_to_add['type'] == 'File':
                value = obs_to_add['value'].split(':')[1]
            else:
                value = obs_to_add['value']
            attr = self.misp_event.add_attribute(value=value, type=type_attr)
            if obs_to_add['id'] in links:
                attr.comment = '%s of %s' % (links[obs_to_add['id']], self.attribute['value'])
        except KeyError:
            logging.error('type not found %s' % obs_to_add['type'])
            return

        for t in obs_to_add['tags']:
            self.misp_event.add_attribute_tag(t['name'], attr['uuid'])

    def __get_object_domain_ip(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname','domain']) or\
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
                                           obj_to_add['value'])
            domain_ip_object.add_attribute(self.__get_relation(self.attribute, is_yeti_object=False),
                                           self.attribute['value'])
            domain_ip_object.add_reference(self.attribute['uuid'], 'related_to')

            return domain_ip_object

    def __get_object_url(self, obj_to_add):
        if (obj_to_add['type'] == 'Url' and self.attribute['type'] in ['hostname', 'domain', 'ip-src', 'ip-dst']) or (
            obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url'
        ):
            url_object = MISPObject('url')
            obj_relation = self.__get_relation(obj_to_add)
            if obj_relation:
                url_object.add_attribute(obj_relation, obj_to_add['value'])
            obj_relation = self.__get_relation(self.attribute, is_yeti_object=False)
            if obj_relation:
                url_object.add_attribute(obj_relation,
                                         self.attribute['value'])
            url_object.add_reference(self.attribute['uuid'], 'related_to')

            return url_object

    def __get_relation(self, obj, is_yeti_object=True):
        if is_yeti_object:
            type_attribute = self.misp_mapping[obj['type']]
        else:
            type_attribute = obj['type']
        if type_attribute == 'ip-src' or type_attribute == 'ip-dst':
            return 'ip'
        elif 'domain' == type_attribute:
            return 'domain'
        elif 'hostname' == type_attribute:
            return 'domain'
        elif type_attribute == 'url':
            return type_attribute

def handler(q=False):
    if q is False:
        return False

    apikey = None
    yeti_url = None
    yeti_client = None

    request = json.loads(q)
    attribute = request['attribute']
    if attribute['type'] not in mispattributes['input']:
        return {'error': 'Unsupported attributes type'}

    if 'config' in request and 'url' in request['config']:
        yeti_url = request['config']['url']
    if 'config' in request and 'apikey' in request['config']:
        apikey = request['config']['apikey']
    if apikey and yeti_url:
        yeti_client = Yeti(yeti_url, apikey, attribute)

    if yeti_client:
        yeti_client.parse_yeti_result()
        return {'results': yeti_client.get_result()}
    else:
        misperrors['error'] = 'Yeti Config Error'
        return misperrors

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo

def introspection():
    return mispattributes<|endoftext|>