<commit_before># Copyright (C) 2014 Linaro Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Test module for the JobHandler handler."""

import json
import mongomock

from concurrent.futures import ThreadPoolExecutor
from mock import (
    MagicMock,
    patch,
)
from tornado import (
    ioloop,
    testing,
    web,
)

from handlers.app import AppHandler
from urls import _JOB_URL

# Default Content-Type header returned by Tornado.
DEFAULT_CONTENT_TYPE = 'application/json; charset=UTF-8'

class TestJobHandler(testing.AsyncHTTPTestCase, testing.LogTrapTestCase):

    def setUp(self):
        self.mongodb_client = mongomock.Connection()

        super(TestJobHandler, self).setUp()

    def get_app(self):
        settings = {
            'client': self.mongodb_client,
            'executor': ThreadPoolExecutor(max_workers=2),
            'default_handler_class': AppHandler,
        }

        return web.Application([_JOB_URL], **settings)

    def get_new_ioloop(self):
        return ioloop.IOLoop.instance()

    @patch('utils.db.find')
    @patch('utils.db.count')
    def test_get(self, mock_count, mock_find):
        mock_count.return_value = 0
        mock_find.return_value = []

        expected_body = '{"count": 0, "code": 200, "limit": 20, "result": "[]"}'

        response = self.fetch('/api/job')

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)
        self.assertEqual(response.body, expected_body)

    @patch('utils.db.find')
    @patch('utils.db.count')
    def test_get_with_wrong_limit(self, mock_count, mock_find):
        mock_count.return_value = 0
        mock_find.return_value = []

        expected_body = (
            '{"count": 0, "code": 200, "limit": 100, "result": "[]"}'
        )

        response = self.fetch('/api/job?limit=1024')

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)
        self.assertEqual(response.body, expected_body)

    @patch('handlers.job.JobHandler.collection')
    def test_get_by_id_not_found(self, collection):
        collection.find_one = MagicMock()
        collection.find_one.return_value = None

        response = self.fetch('/api/job/job-kernel')

        self.assertEqual(response.code, 404)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    @patch('handlers.job.JobHandler.collection')
    def test_get_by_id_found(self, collection):
        collection.find_one = MagicMock()
        collection.find_one.return_value = []

        expected_body = '{"message": "[]", "code": 200}'

        response = self.fetch('/api/job/job-kernel')

        self.assertEqual(response.code, 200)
        self.assertEqual(response.body, expected_body)

    def test_post_without_xsrf(self):

        body = json.dumps(dict(job='job', kernel='kernel'))

        response = self.fetch('/api/job', method='POST', body=body)

        self.assertEqual(response.code, 403)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_post_not_json(self):
        headers = {'X-XSRF-Header': 'foo'}

        response = self.fetch(
            '/api/job', method='POST', body='', headers=headers
        )

        self.assertEqual(response.code, 415)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    @patch('handlers.job.send_emails')
    @patch('handlers.job.import_job')
    def test_post_correct(self, mock_import_job, mock_send_emails):

        mock_import_job.apply_async = MagicMock()

        headers = {
            'X-XSRF-Header': 'foo',
            'Content-Type': 'application/json',
        }

        body = json.dumps(dict(job='job', kernel='kernel'))

        response = self.fetch(
            '/api/job', method='POST', headers=headers, body=body
        )

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_delete_no_xsrf(self):
        response = self.fetch('/api/job/job', method='DELETE')
        self.assertEqual(response.code, 403)

    def test_delete_with_xsfr_no_job(self):
        headers = {'X-XSRF-Header': 'foo'}

        response = self.fetch(
            '/api/job/job', method='DELETE', headers=headers,
        )

        self.assertEqual(response.code, 404)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_delete_with_xsfr_with_job(self):
        db = self.mongodb_client['kernel-ci']
        db['job'].insert(dict(_id='job', job='job', kernel='kernel'))

        headers = {'X-XSRF-Header': 'foo'}

        response = self.fetch(
            '/api/job/job', method='DELETE', headers=headers,
        )

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)
<commit_msg>Better test coverage on POST requests.<commit_after># Copyright (C) 2014 Linaro Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Test module for the JobHandler handler."""

import json
import mongomock

from concurrent.futures import ThreadPoolExecutor
from mock import (
    MagicMock,
    patch,
)
from tornado import (
    ioloop,
    testing,
    web,
)

from handlers.app import AppHandler
from urls import _JOB_URL

# Default Content-Type header returned by Tornado.
DEFAULT_CONTENT_TYPE = 'application/json; charset=UTF-8'

class TestJobHandler(testing.AsyncHTTPTestCase, testing.LogTrapTestCase):

    def setUp(self):
        self.mongodb_client = mongomock.Connection()

        super(TestJobHandler, self).setUp()

    def get_app(self):
        settings = {
            'client': self.mongodb_client,
            'executor': ThreadPoolExecutor(max_workers=2),
            'default_handler_class': AppHandler,
        }

        return web.Application([_JOB_URL], **settings)

    def get_new_ioloop(self):
        return ioloop.IOLoop.instance()

    @patch('utils.db.find')
    @patch('utils.db.count')
    def test_get(self, mock_count, mock_find):
        mock_count.return_value = 0
        mock_find.return_value = []

        expected_body = '{"count": 0, "code": 200, "limit": 20, "result": "[]"}'

        response = self.fetch('/api/job')

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)
        self.assertEqual(response.body, expected_body)

    @patch('utils.db.find')
    @patch('utils.db.count')
    def test_get_with_wrong_limit(self, mock_count, mock_find):
        mock_count.return_value = 0
        mock_find.return_value = []

        expected_body = (
            '{"count": 0, "code": 200, "limit": 100, "result": "[]"}'
        )

        response = self.fetch('/api/job?limit=1024')

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)
        self.assertEqual(response.body, expected_body)

    @patch('handlers.job.JobHandler.collection')
    def test_get_by_id_not_found(self, collection):
        collection.find_one = MagicMock()
        collection.find_one.return_value = None

        response = self.fetch('/api/job/job-kernel')

        self.assertEqual(response.code, 404)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    @patch('handlers.job.JobHandler.collection')
    def test_get_by_id_found(self, collection):
        collection.find_one = MagicMock()
        collection.find_one.return_value = []

        expected_body = '{"message": "[]", "code": 200}'

        response = self.fetch('/api/job/job-kernel')

        self.assertEqual(response.code, 200)
        self.assertEqual(response.body, expected_body)

    def test_post_without_xsrf(self):

        body = json.dumps(dict(job='job', kernel='kernel'))

        response = self.fetch('/api/job', method='POST', body=body)

        self.assertEqual(response.code, 403)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_post_not_json_content(self):
        headers = {'X-XSRF-Header': 'foo', 'Content-Type': 'application/json'}

        response = self.fetch(
            '/api/job', method='POST', body='', headers=headers
        )

        self.assertEqual(response.code, 420)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_post_wrong_content_type(self):
        headers = {'X-XSRF-Header': 'foo'}

        response = self.fetch(
            '/api/job/job', method='POST', body='', headers=headers
        )

        self.assertEqual(response.code, 415)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_post_wrong_json(self):
        headers = {'X-XSRF-Header': 'foo', 'Content-Type': 'application/json'}

        body = json.dumps(dict(foo='foo', bar='bar'))

        response = self.fetch(
            '/api/job/job', method='POST', body=body, headers=headers
        )

        self.assertEqual(response.code, 400)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    @patch('handlers.job.send_emails')
    @patch('handlers.job.import_job')
    def test_post_correct(self, mock_import_job, mock_send_emails):

        mock_import_job.apply_async = MagicMock()

        headers = {
            'X-XSRF-Header': 'foo',
            'Content-Type': 'application/json',
        }

        body = json.dumps(dict(job='job', kernel='kernel'))

        response = self.fetch(
            '/api/job', method='POST', headers=headers, body=body
        )

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_delete_no_xsrf(self):
        response = self.fetch('/api/job/job', method='DELETE')
        self.assertEqual(response.code, 403)

    def test_delete_with_xsfr_no_job(self):
        headers = {'X-XSRF-Header': 'foo'}

        response = self.fetch(
            '/api/job/job', method='DELETE', headers=headers,
        )

        self.assertEqual(response.code, 404)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)

    def test_delete_with_xsfr_with_job(self):
        db = self.mongodb_client['kernel-ci']
        db['job'].insert(dict(_id='job', job='job', kernel='kernel'))

        headers = {'X-XSRF-Header': 'foo'}

        response = self.fetch(
            '/api/job/job', method='DELETE', headers=headers,
        )

        self.assertEqual(response.code, 200)
        self.assertEqual(
            response.headers['Content-Type'], DEFAULT_CONTENT_TYPE)
<|endoftext|>


<commit_before>#
# Creates a mechanism where Python classes can be converted to <--> from primitive data types
# This is used as an intermediate layer to JSON encoding/decoding
#

def get_all_subclasses(cls):
    all_subclasses = []
    for subclass in cls.__subclasses__():
        all_subclasses.append(subclass)
        all_subclasses.extend(get_all_subclasses(subclass))
    return(list(set(all_subclasses)))

#-------------------------------------------------------------------------------
def get_classid_str(cls):
    return("%s.%s" % (cls.__module__, cls.__name__))

#-------------------------------------------------------------------------------
def do_encode(obj, tmpl, parent_key, depth = 1,):

    if(type(tmpl) == list):
        # Expecting a list of items
        if(type(obj) != list):
            raise TypeError("'%s', depth=%d: Expected 'list'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Template list must have exactly one item
        if(len(tmpl) != 1):
            raise ValueError("'%s', depth=%d: Templates for lists must have exactly one item in them"
                % (parent_key, depth))

        result = []
        for item in obj:
            result.append(do_encode(item, tmpl[0], parent_key, depth+1))

    elif(type(tmpl) == tuple):
        # Expecting a tuple of items
        if(type(obj) != tuple):
            raise TypeError("'%s', depth=%d: Expected 'tuple'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Size of tuples must match
        if(len(tmpl) != len(obj)):
            raise ValueError("'%s', depth=%d: Tuple len(%d) does not match template len(%d)"
                % (parent_key, depth, len(obj), len(tmpl)))

        tmplist = []
        for item in obj:
            tmplist.append(do_encode(item, tmpl[0], parent_key, depth+1))

        result = tuple(tmplist)

    elif(type(tmpl) == type):
        # Got to maximum depth.

        if(issubclass(type(obj), encodable_class)):
            # Got an encodable_class

            # make sure it is what the template expects
            if(not issubclass(type(obj), tmpl)):
                raise TypeError("'%s', depth=%d: Expected '%s'. Got '%s'"
                    % (parent_key, depth, tmpl.__name__, type(obj).__name__))

            result = obj.to_dict()

        else:
            # Everything else

            # types should match (unless it is None. Thats OK)
            if((type(obj) != tmpl) and (obj != None)):
                raise TypeError("'%s', depth=%d: Expected '%s'. Got '%s'"
                    % (parent_key, depth, tmpl.__name__, type(obj).__name__))

            # Pass-through
            result = obj

    else:
        raise TypeError("'%s', depth=%d: Unsupported type '%s'"
                    % (parent_key, depth, type(tmpl).__name__))

    return(result)

#-------------------------------------------------------------------------------
def do_decode(obj, tmpl, parent_key, depth = 1,):

    if(type(tmpl) == list):
        # Expecting a list of items
        if(type(obj) != list):
            raise TypeError("'%s', depth=%d: Expected 'list'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Template list must have exactly one item
        if(len(tmpl) != 1):
            raise ValueError("'%s', depth=%d: Templates for lists must have exactly one item in them"
                % (parent_key, depth))

        result = []
        for item in obj:
            result.append(do_decode(item, tmpl[0], parent_key, depth+1))

    elif(type(tmpl) == tuple):
        # Expecting a tuple of items (a list is OK too...)
        if((type(obj) != tuple) and (type(obj) != list)):
            raise TypeError("'%s', depth=%d: Expected 'tuple' or 'list'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Size of tuples must match
        if(len(tmpl) != len(obj)):
            raise ValueError("'%s', depth=%d: Tuple len(%d) does not match template len(%d)"
                % (parent_key, depth, len(obj), len(tmpl)))

        tmplist = []
        for item in obj:
            tmplist.append(do_decode(item, tmpl[0], parent_key, depth+1))

        result = tuple(tmplist)

    elif(type(tmpl) == type):
        # Got to maximum depth.

        if(issubclass(tmpl, encodable_class)):
            # Expecting an encodable_class

            # Check if current obj looks like an encodable_class
            if((type(obj) != dict) or ('__classtype__' not in obj)):
                raise TypeError("'%s', depth=%d: Dictionary incompatible with '%s'"
                    % (parent_key, depth, tmpl.__name__))

            # Figure out what specific subtype of tmpl should be created.
            subclasses = [tmpl]
            subclasses.extend(get_all_subclasses(tmpl))

            m = list(filter(lambda cls: (get_classid_str(cls) == obj['__classtype__']), subclasses))
            if(len(m) == 0):
                raise TypeError("'%s', depth=%d: Type '%s' is incompatible with '%s'"
                    % (parent_key, depth, obj['__classtype__'], get_classid_str(tmpl)))

            cls = m[0]
            result = cls.from_dict(obj)

        else:
            # Everything else

            # types should match (unless it is None. Thats OK)
            if((type(obj) != tmpl) and (obj != None)):
                raise TypeError("'%s', depth=%d: Expected '%s'. Got '%s'"
                    % (parent_key, depth, tmpl.__name__, type(obj).__name__))

            # Pass-through
            result = obj

    else:
        raise TypeError("'%s', depth=%d: Unsupported type '%s'"
                    % (parent_key, depth, type(tmpl).__name__))

    return(result)

#-------------------------------------------------------------------------------
class encodable_class(object):

    """
    Dictionary of class members to encode
    {key : template}

    key: The name of the class member
    template: Minimal representation of the type of the member's contents

    Aggregate data types in template:
        List: Describes a list where each item in the list is of the same type
        Tuple: Describes a tuple of fixed size, containing the matching type items

    Examples:
        A String:
            {"my_string" : str}
        List of integers:
            {"my_list" : [int]}
        List of mixed tuples:
            {"my_complex_list" : [(int, str, my_class)]}
    """
    _encode_schema = {}

    def to_dict(self):
        """
        Encodes the class, and all its child members to a dictionary
        Only encodes strictly according to what is defined in _encode_schema
        """
        # Collapse all schemas into one
        schema = self._merge_schemas()

        D = {}
        D['__classtype__'] = get_classid_str(type(self))

        for key, template in schema.items():
            D[key] = do_encode(getattr(self,key), template, key)

        return(D)

    @classmethod
    def from_dict(cls, D):
        """
        Construct a class from a dictionary.
        Class members are populated based on what is defined in _encode_schema
        """
        if(('__classtype__' not in D) or (D['__classtype__'] != get_classid_str(cls))):
            raise ValueError("Dictionary is incompatible with object '%s'" % cls.__name__)

        del D['__classtype__']

        # Collapse all schemas into one
        schema = cls._merge_schemas()

        self = cls.__new__(cls)

        for key, template in schema.items():
            setattr(self, key, do_decode(D[key], template, key))

        return(self)

    @classmethod
    def _merge_schemas(cls):
        """
        Combines own _encode_schema with all parent classes
        Returns merged result
        """
        schema = {}
        for base_t in cls.__bases__:
            if(issubclass(base_t, encodable_class)):
                schema.update(base_t._merge_schemas())

        schema.update(cls._encode_schema)
        return(schema)
<commit_msg>Renamed some stuff. Added support to detect and preserve multiple instances of the same object<commit_after>#
# Creates a mechanism where Python classes can be converted to <--> from primitive data types
# This is used as an intermediate layer to JSON encoding/decoding
#

def get_all_subclasses(cls):
    all_subclasses = []
    for subclass in cls.__subclasses__():
        all_subclasses.append(subclass)
        all_subclasses.extend(get_all_subclasses(subclass))
    return(list(set(all_subclasses)))

#-------------------------------------------------------------------------------
def get_classid_str(cls):
    return("%s.%s" % (cls.__module__, cls.__name__))

#-------------------------------------------------------------------------------
class Ref:
    """
    Temporary placeholder for unresolved references
    """
    def __init__(self, ref_id):
        self.ref_id = ref_id

#-------------------------------------------------------------------------------
def do_encode(obj, tmpl, parent_key, _encoded_objs, depth = 1):

    if(type(tmpl) == list):
        # Expecting a list of items
        if(type(obj) != list):
            raise TypeError("'%s', depth=%d: Expected 'list'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Template list must have exactly one item
        if(len(tmpl) != 1):
            raise ValueError("'%s', depth=%d: Templates for lists must have exactly one item in them"
                % (parent_key, depth))

        result = []
        for item in obj:
            result.append(do_encode(item, tmpl[0], parent_key, _encoded_objs, depth+1))

    elif(type(tmpl) == tuple):
        # Expecting a tuple of items
        if(type(obj) != tuple):
            raise TypeError("'%s', depth=%d: Expected 'tuple'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Size of tuples must match
        if(len(tmpl) != len(obj)):
            raise ValueError("'%s', depth=%d: Tuple len(%d) does not match template len(%d)"
                % (parent_key, depth, len(obj), len(tmpl)))

        tmplist = []
        for item in obj:
            tmplist.append(do_encode(item, tmpl[0], parent_key, _encoded_objs, depth+1))

        result = tuple(tmplist)

    elif(type(tmpl) == type):
        # Got to maximum depth.

        if(issubclass(type(obj), EncodableClass)):
            # Got an EncodableClass

            # make sure it is what the template expects
            if(not issubclass(type(obj), tmpl)):
                raise TypeError("'%s', depth=%d: Expected '%s'. Got '%s'"
                    % (parent_key, depth, tmpl.__name__, type(obj).__name__))

            result = obj.to_dict(_encoded_objs)

        else:
            # Everything else

            # types should match (unless it is None. Thats OK)
            if((type(obj) != tmpl) and (obj != None)):
                raise TypeError("'%s', depth=%d: Expected '%s'. Got '%s'"
                    % (parent_key, depth, tmpl.__name__, type(obj).__name__))

            # Pass-through
            result = obj

    else:
        raise TypeError("'%s', depth=%d: Unsupported type '%s'"
                    % (parent_key, depth, type(tmpl).__name__))

    return(result)

#-------------------------------------------------------------------------------
def do_decode(obj, tmpl, parent_key, _decoded_objs, depth = 1):

    if(type(tmpl) == list):
        # Expecting a list of items
        if(type(obj) != list):
            raise TypeError("'%s', depth=%d: Expected 'list'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Template list must have exactly one item
        if(len(tmpl) != 1):
            raise ValueError("'%s', depth=%d: Templates for lists must have exactly one item in them"
                % (parent_key, depth))

        result = []
        for item in obj:
            result.append(do_decode(item, tmpl[0], parent_key, _decoded_objs, depth+1))

    elif(type(tmpl) == tuple):
        # Expecting a tuple of items (a list is OK too...)
        if((type(obj) != tuple) and (type(obj) != list)):
            raise TypeError("'%s', depth=%d: Expected 'tuple' or 'list'. Got '%s'"
                % (parent_key, depth, type(obj).__name__))

        # Size of tuples must match
        if(len(tmpl) != len(obj)):
            raise ValueError("'%s', depth=%d: Tuple len(%d) does not match template len(%d)"
                % (parent_key, depth, len(obj), len(tmpl)))

        tmplist = []
        for idx, item in enumerate(obj):
            tmplist.append(do_decode(item, tmpl[idx], parent_key, _decoded_objs, depth+1))

        result = tuple(tmplist)

    elif(type(tmpl) == type):
        # Got to maximum depth.

        if(issubclass(tmpl, EncodableClass)):
            # Expecting an EncodableClass

            # Check if current obj looks like an EncodableClass
            if((type(obj) != dict) or ('<classtype>' not in obj)):
                raise TypeError("'%s', depth=%d: Dictionary incompatible with '%s'"
                    % (parent_key, depth, tmpl.__name__))

            if('<ref_id>' not in obj):
                raise ValueError("'%s', depth=%d: Missing <ref_id>" % (parent_key, depth))

            if(obj['<classtype>'] == '<ref>'):
                # This is a reference, not an actual class definition
                # Populate a Ref object for now. It will be resolved later with the actual
                result = Ref(obj['<ref_id>'])

            else:
                # Not a reference. This is an actual class definition

                # Figure out what specific subtype of tmpl should be created.
                subclasses = [tmpl]
                subclasses.extend(get_all_subclasses(tmpl))

                m = list(filter(lambda cls: (get_classid_str(cls) == obj['<classtype>']), subclasses))
                if(len(m) == 0):
                    raise TypeError("'%s', depth=%d: Type '%s' is incompatible with '%s'"
                        % (parent_key, depth, obj['<classtype>'], get_classid_str(tmpl)))

                cls = m[0]
                result = cls.from_dict(obj, _decoded_objs)

        else:
            # Everything else

            # types should match (unless it is None. Thats OK)
            if((type(obj) != tmpl) and (obj != None)):
                raise TypeError("'%s', depth=%d: Expected '%s'. Got '%s'"
                    % (parent_key, depth, tmpl.__name__, type(obj).__name__))

            # Pass-through
            result = obj

    else:
        raise TypeError("'%s', depth=%d: Unsupported type '%s'"
                    % (parent_key, depth, type(tmpl).__name__))

    return(result)

#-------------------------------------------------------------------------------
def do_resolve_ref(tmpl, obj, _decoded_objs):
    # Note: type checking against template is only done for Refs since everything else has
    # already been validated

    if(type(obj) == list):
        # Resolve all refs in a list
        for i,v in enumerate(obj):
            obj[i] = do_resolve_ref(tmpl[0], v, _decoded_objs)

    elif(type(obj) == tuple):
        # Resolve all refs in a tuple

        # Cast tuple back to list so it can be modified
        obj = list(obj)
        for i, v in enumerate(obj):
            obj[i] = do_resolve_ref(tmpl[i], v, _decoded_objs)
        obj = tuple(obj)

    elif(type(obj) == Ref):
        # Resolve reference

        if(obj.ref_id not in _decoded_objs):
            raise ValueError("Unresolved reference to object with ref_id %d" % Ref.ref_id)

        obj = _decoded_objs[obj.ref_id]

        # Verify that the referenced object type is compatible with the template
        subclasses = [tmpl]
        subclasses.extend(get_all_subclasses(tmpl))
        m = list(filter(lambda cls: (cls == type(obj)), subclasses))
        if(len(m) == 0):
            raise TypeError("Referenced type '%s' is incompatible with '%s'"
                % (get_classid_str(type(obj)), get_classid_str(tmpl)))

    elif(issubclass(type(obj), EncodableClass)):
        # Resolve references of sub-object
        obj._resolve_refs(_decoded_objs)

    else:
        # Everything else passes through
        pass

    return(obj)

#-------------------------------------------------------------------------------
class EncodableClass(object):

    """
    Dictionary of class members to encode
    {key : template}

    key: The name of the class member
    template: Minimal representation of the type of the member's contents

    Aggregate data types in template:
        List: Describes a list where each item in the list is of the same type
        Tuple: Describes a tuple of fixed size, containing the matching type items

    Examples:
        A String:
            {"my_string" : str}
        List of integers:
            {"my_list" : [int]}
        List of mixed tuples:
            {"my_complex_list" : [(int, str, my_class)]}
    """
    encode_schema = {}

    def to_dict(self, _encoded_objs=None):
        """
        Encodes the class, and all its child members to a dictionary
        Only encodes strictly according to what is defined in encode_schema

        The _encoded_objs parameter is for internal use only
        (tracks which objs have already been encoded)
        """

        if(_encoded_objs == None):
            # Allocate new list
            _encoded_objs = []

        D = {}
        if(self in _encoded_objs):
            # This object has already been encoded elsewhere.
            # Instead, just store a reference to the other one
            D['<classtype>'] = '<ref>'
            D['<ref_id>'] = _encoded_objs.index(self)

        else:
            # First time encountering this object.

            # Since it will be encoded here, insert into the _encoded_objs list
            ref_id = len(_encoded_objs)
            _encoded_objs.append(self)

            # Collapse all schemas from parent classes into one
            schema = self._merge_schemas()

            D['<classtype>'] = get_classid_str(type(self))
            D['<ref_id>'] = ref_id

            for key, template in schema.items():
                D[key] = do_encode(getattr(self,key), template, key, _encoded_objs)

        return(D)

    @classmethod
    def from_dict(cls, D, _decoded_objs=None):
        """
        Construct a class from a dictionary.
        Class members are populated based on what is defined in encode_schema

        The _decoded_objs parameter is for internal use only
        (tracks which objs have been decoded for later ref resolution.)
        """
        if(_decoded_objs == None):
            # Allocate new dict
            _decoded_objs = {}
            is_root = True
        else:
            is_root = False

        if(('<classtype>' not in D) or (D['<classtype>'] != get_classid_str(cls))):
            raise ValueError("Dictionary is incompatible with object '%s'" % cls.__name__)

        if('<ref_id>' not in D):
            raise ValueError("Missing <ref_id>")

        ref_id = D['<ref_id>']

        del D['<classtype>']
        del D['<ref_id>']

        # Collapse all schemas into one
        schema = cls._merge_schemas()

        self = cls.__new__(cls)

        # register the decoded object
        if(ref_id in _decoded_objs):
            # An object with the same ID was already decoded??
            raise ValueError("An object with the same <ref_id> : %d has already been decoded" % ref_id)
        _decoded_objs[ref_id] = self

        # Decode contents of self
        for key, template in schema.items():
            v = do_decode(D[key], template, key, _decoded_objs)
            setattr(self, key, v)

        if(is_root):
            # This is the root object. Finished decoding everything
            # Now, do a second pass through all objects, and resolve references
            self._resolve_refs(_decoded_objs)

        return(self)

    def _resolve_refs(self, _decoded_objs):

        # Collapse all schemas into one
        schema = type(self)._merge_schemas()

        for key, template in schema.items():
            v = getattr(self, key)
            v = do_resolve_ref(template, v, _decoded_objs)
            setattr(self, key, v)

    @classmethod
    def _merge_schemas(cls):
        """
        Combines own encode_schema with all parent classes
        Returns merged result
        """
        schema = {}
        for base_t in cls.__bases__:
            if(issubclass(base_t, EncodableClass)):
                schema.update(base_t._merge_schemas())

        schema.update(cls.encode_schema)
        return(schema)
<|endoftext|>


<commit_before>import sys
from contextlib import redirect_stdout
import io
from PyQt5.QtCore import QThread, pyqtSignal
sys.path.insert(0, './pyth')
from pyth import *

class OutputWriter:
        def __init__(self, text_edit_write):
            self.text_edit_write = text_edit_write

        def write(self, s):
            self.text_edit_write.emit(str(s))

class CodeExecutor(QThread):
    begin = pyqtSignal(name='start')
    text_edit_write = pyqtSignal(str, name='text_edit_write')
    finished = pyqtSignal(name='finished')

    def __init__(self, code, inp, debug_on, multi_line_on):
        QThread.__init__(self)
        self.code = code
        self.inp = inp
        self.debug_on = debug_on
        self.multi_line_on = multi_line_on

    def run(self):
        self.begin.emit()

        with redirect_stdout(OutputWriter(self.text_edit_write)):
            if isinstance(self.inp, list):
                for input_data in self.inp:
                    self.execute_code(input_data)
                    print()
            else:
                self.execute_code(self.inp)

        sys.stdin = sys.__stdin__
        sys.stdout = sys.__stdout__

        self.finished.emit()

    def execute_code(self, input_data):
        global safe_mode
        global environment
        global c_to_i

        sys.stdin = io.StringIO(input_data)
        error = None

        saved_env = c.deepcopy(environment)
        saved_c_to_i = c.deepcopy(c_to_i)

        try:
            safe_mode = False
            exec(general_parse(self.code), environment)
        except SystemExit:
            pass
        except Exception as e:
            error = e

        for key in saved_env:
            environment[key] = saved_env[key]
        c_to_i = saved_c_to_i
<commit_msg>Also print Debug and Error messages<commit_after>import sys
from contextlib import redirect_stdout
import io
import traceback
from PyQt5.QtCore import QThread, pyqtSignal
sys.path.insert(0, './pyth')
from pyth import *

class OutputWriter:
        def __init__(self, text_edit_write):
            self.text_edit_write = text_edit_write

        def write(self, s):
            self.text_edit_write.emit(str(s))

class CodeExecutor(QThread):
    begin = pyqtSignal(name='start')
    text_edit_write = pyqtSignal(str, name='text_edit_write')
    finished = pyqtSignal(name='finished')

    def __init__(self, code, inp, debug_on, multi_line_on):
        QThread.__init__(self)
        self.code = code
        self.inp = inp
        self.debug_on = debug_on
        self.multi_line_on = multi_line_on

        if multi_line_on:
            self.code = preprocess_multiline(self.code)

    def run(self):
        self.begin.emit()

        output_writer = OutputWriter(self.text_edit_write)
        sys.stderr = output_writer
        with redirect_stdout(output_writer):
            if isinstance(self.inp, list):
                for input_data in self.inp:
                    self.execute_code(input_data)
                    print()
            else:
                self.execute_code(self.inp)

        sys.stdin = sys.__stdin__
        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__

        self.finished.emit()

    def execute_code(self, input_data):
        global safe_mode
        global environment
        global c_to_i

        sys.stdin = io.StringIO(input_data)

        saved_env = c.deepcopy(environment)
        saved_c_to_i = c.deepcopy(c_to_i)

        try:
            safe_mode = False

            py_code_line = general_parse(self.code)
            if self.debug_on:
                print('{:=^50}'.format(' ' + str(len(self.code)) + ' chars '))
                print(self.code)
                print('='*50)
                print(py_code_line)
                print('='*50)

            exec(py_code_line, environment)
        except SystemExit:
            pass
        except Exception as error:
            traceback.print_exc()

        for key in saved_env:
            environment[key] = saved_env[key]
        c_to_i = saved_c_to_i
<|endoftext|>


<commit_before>""" This module is an interface for the open exoplanet catalogue. Its design motive is to be user friendly and readable
with speed as a secondary consideration. Therefore if your using larger complex queries and are comfortable with the
default xml.etree.ElementTree you will probably find that faster and more powerful.
"""

import xml.etree.ElementTree as ET
import glob

from astroclasses import System, Star, Planet
from os.path import basename

databaseLocation = '/Users/ryanv/Documents/git/open-exoplanet-catalogue/systems/'  # Temp

# Initialise Database
systems = {}
stars = {}
planets = {}

for filename in glob.glob(databaseLocation + '*.xml'):
    tree = ET.parse(open(filename, 'r'))
    root = tree.getroot()

    # Process the system
    assert root.tag == 'system', '{} does not contain a valid system'  # TODO remove or upgrade to raise or try

    systemParams = {}
    for systemXML in root:

        tag = systemXML.tag

        if not tag == 'star':  # TODO this is for the class to reject not the loader
            systemParams[tag] = systemXML.text

    system = System(systemParams)
    systems[system.params['name']] = system  # Add system to the index

    # Now look for stars
    starsXML = root.findall(".//star")

    for starXML in starsXML:

        starParams = {}

        for value in starXML:

            tag = value.tag

            if not tag == 'planet':  # TODO this is for the class to reject not the loader
                starParams[tag] = value.text

        star = Star(starParams)
        star.parent = system

        system._addChild(star.params['name'], star)  # Add planet to the system
        stars[star.params['name']] = star  # Add planet to the index

<commit_msg>working database<commit_after>""" This module is an interface for the open exoplanet catalogue. Its design motive is to be user friendly and readable
with speed as a secondary consideration. Therefore if your using larger complex queries and are comfortable with the
default xml.etree.ElementTree you will probably find that faster and more powerful.
"""

import xml.etree.ElementTree as ET
import glob

from astroclasses import System, Star, Planet
from os.path import basename

databaseLocation = '/Users/ryanv/Documents/git/open-exoplanet-catalogue/systems/'  # Temp

# Initialise Database
systems = {}
stars = {}
planets = {}

for filename in glob.glob(databaseLocation + '*.xml'):
    tree = ET.parse(open(filename, 'r'))
    root = tree.getroot()

    # Process the system
    assert root.tag == 'system', '{} does not contain a valid system'  # TODO remove or upgrade to raise or try

    systemParams = {}
    for systemXML in root:

        tag = systemXML.tag

        if not tag == 'star':  # TODO this is for the class to reject not the loader
            systemParams[tag] = systemXML.text

    system = System(systemParams)
    systems[system.params['name']] = system  # Add system to the index

    # Now look for stars
    starsXML = root.findall(".//star")

    for starXML in starsXML:

        starParams = {}

        for value in starXML:

            tag = value.tag

            if not tag == 'planet':  # TODO this is for the class to reject not the loader
                starParams[tag] = value.text

        star = Star(starParams)
        star.parent = system

        system._addChild(star.params['name'], star)  # Add star to the system
        stars[star.params['name']] = star  # Add star to the index

        # And finally look for planets
        planetsXML = root.findall(".//planet")

        for planetXML in planetsXML:

            planetParams = {}

            for value in planetXML:
                # TODO handle duplicate names
                tag = value.tag
                planetParams[tag] = value.text

            planet = Planet(planetParams)
            planet.parent = star

            star._addChild(planet.params['name'], planet)  # Add planet to the star
            planets[planet.params['name']] = planet  # Add planet to the index

<|endoftext|>


<commit_before>from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import models as auth

from etcetera.extras.mailer import error_mail
from etcetera.extras import models as extras
from etcetera.extras import forms as exforms
from etcetera.service import models as service

from docutils.core import publish_parts

def error_mail(request):
	error_mail(request)
	return HttpResponseRedirect('etcetera-index')

def index(request):
	posts = extras.Post.objects.all()
	context = {
		'object_list': posts,
	}
	return render_to_response(
		"index.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def profile(request, the_user):
	if not the_user == request.user:
		the_user = get_object_or_404(auth.User, username=the_user)
	try:
		the_user.profile = the_user.get_profile()
	except extras.UserProfile.DoesNotExist:
		if the_user == request.user:
			return HttpResponseRedirect(reverse('edit-profile'))
		else:
			the_user.profile = None
	# There's got to be a better way to do this. I'll figure it out sometime.
	the_user.repairs = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="repair"
	)
	the_user.installs = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="install"
	)
	the_user.maintenances = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="maintenance"
	)
	the_user.replacements = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="replacement"
	)
	context = {
		'the_user': the_user,
	}
	return render_to_response(
		"extras/profile.html",
		context,
		context_instance=RequestContext(request)
	)

def edit_profile(request):
	if request.method == 'POST':
		try:
			form = exforms.UserProfileForm(
				request.POST, request.FILES,
				instance=request.user.get_profile()
			)
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm(request.POST, request.FILES)
		if form.is_valid():
			if not form.cleaned_data['user']:
				form.cleaned_data['user'] = request.user
			form.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		try:
			form = exforms.UserProfileForm(instance=request.user.get_profile())
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm()
		except AttributeError:
			return HttpResponseRedirect(reverse('etcetera-login'))
	context = {
		'the_user': request.user,
		'form': form,
	}
	return render_to_response(
		"extras/edit-profile.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def change_password(request):
	user = request.user
	if request.method == 'POST':
		form = exforms.PasswordForm(request.POST)
		if form.is_valid():
			cd = form.cleaned_data
			user.set_password(cd['password'])
			user.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		form = exforms.PasswordForm()
	context = {
		'user': user,
		'form': form,
	}
	return render_to_response(
		"extras/change-password.html",
		context,
		context_instance=RequestContext(request)
	)
<commit_msg>Front page now only shows the latest 3 posts.<commit_after>from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import models as auth

from etcetera.extras.mailer import error_mail
from etcetera.extras import models as extras
from etcetera.extras import forms as exforms
from etcetera.service import models as service

from docutils.core import publish_parts

def error_mail(request):
	error_mail(request)
	return HttpResponseRedirect('etcetera-index')

def index(request):
	posts = extras.Post.objects.all()[:3]
	context = {
		'object_list': posts,
	}
	return render_to_response(
		"index.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def profile(request, the_user):
	if not the_user == request.user:
		the_user = get_object_or_404(auth.User, username=the_user)
	try:
		the_user.profile = the_user.get_profile()
	except extras.UserProfile.DoesNotExist:
		if the_user == request.user:
			return HttpResponseRedirect(reverse('edit-profile'))
		else:
			the_user.profile = None
	# There's got to be a better way to do this. I'll figure it out sometime.
	the_user.repairs = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="repair"
	)
	the_user.installs = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="install"
	)
	the_user.maintenances = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="maintenance"
	)
	the_user.replacements = service.WorkOrder.objects.filter(
		technician=the_user
	).filter(
		archived=False
	).filter(
		work_type="replacement"
	)
	context = {
		'the_user': the_user,
	}
	return render_to_response(
		"extras/profile.html",
		context,
		context_instance=RequestContext(request)
	)

def edit_profile(request):
	if request.method == 'POST':
		try:
			form = exforms.UserProfileForm(
				request.POST, request.FILES,
				instance=request.user.get_profile()
			)
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm(request.POST, request.FILES)
		if form.is_valid():
			if not form.cleaned_data['user']:
				form.cleaned_data['user'] = request.user
			form.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		try:
			form = exforms.UserProfileForm(instance=request.user.get_profile())
		except extras.UserProfile.DoesNotExist:
			form = exforms.UserProfileForm()
		except AttributeError:
			return HttpResponseRedirect(reverse('etcetera-login'))
	context = {
		'the_user': request.user,
		'form': form,
	}
	return render_to_response(
		"extras/edit-profile.html",
		context,
		context_instance=RequestContext(request)
	)

@login_required
def change_password(request):
	user = request.user
	if request.method == 'POST':
		form = exforms.PasswordForm(request.POST)
		if form.is_valid():
			cd = form.cleaned_data
			user.set_password(cd['password'])
			user.save()
			return HttpResponseRedirect(reverse(
				'etcetera-user',
				kwargs={'the_user': request.user},
			))
	else:
		form = exforms.PasswordForm()
	context = {
		'user': user,
		'form': form,
	}
	return render_to_response(
		"extras/change-password.html",
		context,
		context_instance=RequestContext(request)
	)
<|endoftext|>


<commit_before>from fabric.api import task, local, settings, hide, run
from pprint import pprint
from cloudmesh.cm_mongo import cm_mongo
from cloudmesh.config.cm_config import cm_config, cm_config_server
from cloudmesh.user.cm_userLDAP import cm_userLDAP
from cloudmesh.pbs.pbs_mongo import pbs_mongo
from cloudmesh.util.util import path_expand as cm_path_expand
from cloudmesh.util.util import banner
from cloudmesh.inventory import Inventory
from cloudmesh.user.cm_template import cm_template
from cloudmesh.config.ConfigDict import ConfigDict
from cloudmesh.user.cm_user import cm_user
from sh import keystone

from  yaml import dump as yaml_dump
import sys
import os.path

@task
def yaml():
    """
    creates a user from tempalte and a simplified version of the yaml file and
    puts it into mongo as a profile

    """

    with hide('running', 'status', 'output'):
        user_config = ConfigDict(filename="~/.futuregrid/me.yaml")
        t = cm_template("~/.futuregrid/etc/cloudmesh.yaml")
        result = t.replace(kind="dict", values=user_config)
        print yaml_dump(result, default_flow_style=False)
        location = cm_path_expand('~/.futuregrid/cloudmesh-new.yaml')
        yaml_file = open(location, 'w+')
        print >> yaml_file, yaml_dump(result, default_flow_style=False)
        yaml_file.close()
        print "Written new yaml file in " + location

@task
def list():
     user = cm_user()
     list_of_users = user.list_users()
     pprint (list_of_users)
     print
     print "========================="
     num = len(list_of_users)
     print str(num) + " users listed"

@task
def info(id):
    user = cm_user()
    res = user.info(id)
    pprint (res)

@task
def password():
    user_config = cm_config(filename="~/.futuregrid/cloudmesh.yaml")
    user = user_config.cloud('sierra_openstack_grizzly')['credentials']

    server_config = ConfigDict(filename="~/.futuregrid/cloudmesh_server.yaml")
    server = server_config['keystone']['sierra_openstack_grizzly']

    print(" ".join(["keystone", "--os-username", server['OS_USERNAME'],
             "--os-password", server['OS_PASSWORD'],
             "--os-tenant-name", server['OS_TENANT_NAME'],
             "--os-auth-url", server['OS_AUTH_URL'],
             "user-password-update",
             "--pass", user['OS_PASSWORD'], user['OS_USERNAME']]))

    keystone("--os-username", server['OS_USERNAME'],
             "--os-password", server['OS_PASSWORD'],
             "--os-tenant-name", server['OS_TENANT_NAME'],
             "--os-auth-url", server['OS_AUTH_URL'],
             "user-password-update",
             "--pass", user['OS_PASSWORD'], user['OS_USERNAME'])

@task
def mongo():
    yaml()
    config = cm_config(filename="~/.futuregrid/cloudmesh.yaml")
    profile = config.profile()

    element = {
               "firstname" : profile["firstname"],
               "lastname" : profile["lastname"],
               "uidNumber" : profile["uid"],
               "phone" : profile["phone"],
               "gidNumber" : profile["gid"],
               "address" : profile["address"][0],
               "cm_user_id" : config.get("cloudmesh.hpc.username"),
               "email" : profile["e_mail"]
    }

    projects = {}

    active = config.get("cloudmesh.projects.active")

    if active != ['None']:
        projects["active"] = active

    completed = config.get("cloudmesh.projects.completed")
    if completed != ['None']:
        projects["completed"] = completed

    if projects != {}:
        element["projects"] = projects

    # get keys and clean the key titles (replace '.' with '_' due to mongo restriction)
    keys = config.get("cloudmesh.keys.keylist")
    for keytitle in keys.keys():
        if "." in keytitle:
            keycontent = keys[keytitle]
            newkeytitle = keytitle.replace(".", "_")
            del keys[keytitle]
            keys[newkeytitle] = keycontent
    element['keys'] = keys

    #pprint (element)

    # hpc username as key
    username = element["cm_user_id"]
    # populate the local userinfo into the same mongo as though it were from LDAP.
    userstore = cm_userLDAP()
    userstore.updates(username, element)
<commit_msg>removing the yaml writer adding a pprint for debugging<commit_after>from fabric.api import task, local, settings, hide, run
from pprint import pprint
from cloudmesh.cm_mongo import cm_mongo
from cloudmesh.config.cm_config import cm_config, cm_config_server
from cloudmesh.user.cm_userLDAP import cm_userLDAP
from cloudmesh.pbs.pbs_mongo import pbs_mongo
from cloudmesh.util.util import path_expand as cm_path_expand
from cloudmesh.util.util import banner
from cloudmesh.inventory import Inventory
from cloudmesh.user.cm_template import cm_template
from cloudmesh.config.ConfigDict import ConfigDict
from cloudmesh.user.cm_user import cm_user
from sh import keystone

from  yaml import dump as yaml_dump
import sys
import os.path

@task
def yaml():
    """
    creates a user from tempalte and a simplified version of the yaml file and
    puts it into mongo as a profile

    """

    with hide('running', 'status', 'output'):
        user_config = ConfigDict(filename="~/.futuregrid/me.yaml")
        t = cm_template("~/.futuregrid/etc/cloudmesh.yaml")
        result = t.replace(kind="dict", values=user_config)
        print yaml_dump(result, default_flow_style=False)
        location = cm_path_expand('~/.futuregrid/cloudmesh-new.yaml')
        yaml_file = open(location, 'w+')
        print >> yaml_file, yaml_dump(result, default_flow_style=False)
        yaml_file.close()
        print "Written new yaml file in " + location

@task
def list():
     user = cm_user()
     list_of_users = user.list_users()
     pprint (list_of_users)
     print
     print "========================="
     num = len(list_of_users)
     print str(num) + " users listed"

@task
def info(id):
    user = cm_user()
    res = user.info(id)
    pprint (res)

@task
def password():
    user_config = cm_config(filename="~/.futuregrid/cloudmesh.yaml")
    user = user_config.cloud('sierra_openstack_grizzly')['credentials']

    server_config = ConfigDict(filename="~/.futuregrid/cloudmesh_server.yaml")
    server = server_config['keystone']['sierra_openstack_grizzly']

    print(" ".join(["keystone", "--os-username", server['OS_USERNAME'],
             "--os-password", server['OS_PASSWORD'],
             "--os-tenant-name", server['OS_TENANT_NAME'],
             "--os-auth-url", server['OS_AUTH_URL'],
             "user-password-update",
             "--pass", user['OS_PASSWORD'], user['OS_USERNAME']]))

    keystone("--os-username", server['OS_USERNAME'],
             "--os-password", server['OS_PASSWORD'],
             "--os-tenant-name", server['OS_TENANT_NAME'],
             "--os-auth-url", server['OS_AUTH_URL'],
             "user-password-update",
             "--pass", user['OS_PASSWORD'], user['OS_USERNAME'])

@task
def mongo():
    config = cm_config(filename="~/.futuregrid/cloudmesh.yaml")
    profile = config.profile()

    element = {
               "firstname" : profile["firstname"],
               "lastname" : profile["lastname"],
               "uidNumber" : profile["uid"],
               "phone" : profile["phone"],
               "gidNumber" : profile["gid"],
               "address" : profile["address"][0],
               "cm_user_id" : config.get("cloudmesh.hpc.username"),
               "email" : profile["e_mail"]
    }

    projects = {}

    active = config.get("cloudmesh.projects.active")

    if active != ['None']:
        projects["active"] = active

    completed = config.get("cloudmesh.projects.completed")
    if completed != ['None']:
        projects["completed"] = completed

    if projects != {}:
        element["projects"] = projects

    # get keys and clean the key titles (replace '.' with '_' due to mongo restriction)
    keys = config.get("cloudmesh.keys.keylist")
    for keytitle in keys.keys():
        if "." in keytitle:
            keycontent = keys[keytitle]
            newkeytitle = keytitle.replace(".", "_")
            del keys[keytitle]
            keys[newkeytitle] = keycontent
    element['keys'] = keys

    pprint (element)

    # hpc username as key
    username = element["cm_user_id"]
    # populate the local userinfo into the same mongo as though it were from LDAP.
    userstore = cm_userLDAP()
    userstore.updates(username, element)
<|endoftext|>


<commit_before>"""
SmartOS packages
===============

This module provides tools to manage SmartOS packages.

"""
from __future__ import with_statement

from fabric.api import *
from fabtools.files import is_file

MANAGER = 'pkgin'

def update_index(force=False):
    """
    Update pkgin package definitions.
    """
    manager = MANAGER
    if force:
        with quiet():
            sudo("%(manager)s cl" % locals())
        sudo("%(manager)s -f up" % locals())
    else:
        sudo("%(manager)s up" % locals())

def upgrade(full=False):
    """
    Upgrade all packages.
    """
    manager = MANAGER
    cmds = {'pkgin': {False: 'ug', True: 'fug'}}
    cmd = cmds[manager][full]
    sudo("%(manager)s -y %(cmd)s" % locals())

def is_installed(pkg_name):
    """
    Check if a package is installed.
    """
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = run("pkgin list | grep -qio %(pkg_name)s" % locals())
        if res.succeeded:
            return True
        return False

def install(packages, update=False, yes=None, options=None):
    """
    Install one or more packages.

    If *update* is ``True``, the package definitions will be updated
    first, using :py:func:`~fabtools.pkg.update_index`.

    Extra *yes* may be passed to ``pkgin`` to validate license if necessary.

    Extra *options* may be passed to ``pkgin`` if necessary.

    Example::

        import fabtools

        # Update index, then verbosely install a single package
        fabtools.pkg.install('redis', update=True, options='-V',)

        # Install multiple packages
        fabtools.pkg.install([
            'unzip',
            'top'
        ])

    """
    manager = MANAGER
    if update:
        update_index()
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    if isinstance(yes, str):
        sudo('yes %(yes)s | %(manager)s %(options)s in %(packages)s' % locals())
    else:
        sudo('%(manager)s %(options)s in %(packages)s' % locals())

def uninstall(packages, orphan=False, options=None):
    """
    Remove one or more packages.

    If *orphan* is ``True``, orphan dependencies will be
    removed from the system.

    Extra *options* may be passed to ``pkgin`` if necessary.
    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    if orphan:
        sudo('%(manager)s -y ar' % locals())
    sudo('%(manager)s %(options)s remove %(packages)s' % locals())

def smartos_build():
    """
    Get the build of SmartOS. Useful to determine provider for example.

    Example::

        from fabtools.pkg import smartos_build

        if smartos_build().startswith('joyent'):
            print('SmartOS Joyent')

    """
    with settings(hide('running', 'stdout')):
        return run('uname -v')

def smartos_image():
    """
    Get the SmartOS image. Useful to determine the image/dataset for example.
    Returns 'NA' if it can't be determined.

    Example::

        from fabtools.pkg import smartos_image

        if smartos_image().startswith('percona'):
            sudo('mysql -uroot -psecretpassword -e 'show databases;'')

    """
    with settings(hide('running', 'stdout')):
        if is_file('/etc/product'):
            return run('cat /etc/product | head -n 2 | tail -n 1 | awk \'{ print $2 " " $3 }\'')
        else:
            return 'NA'
<commit_msg>returns None instead of NA if smartos_* cant be determined<commit_after>"""
SmartOS packages
===============

This module provides tools to manage SmartOS packages.

"""
from __future__ import with_statement

from fabric.api import *
from fabtools.files import is_file

MANAGER = 'pkgin'

def update_index(force=False):
    """
    Update pkgin package definitions.
    """
    manager = MANAGER
    if force:
        with quiet():
            sudo("%(manager)s cl" % locals())
        sudo("%(manager)s -f up" % locals())
    else:
        sudo("%(manager)s up" % locals())

def upgrade(full=False):
    """
    Upgrade all packages.
    """
    manager = MANAGER
    cmds = {'pkgin': {False: 'ug', True: 'fug'}}
    cmd = cmds[manager][full]
    sudo("%(manager)s -y %(cmd)s" % locals())

def is_installed(pkg_name):
    """
    Check if a package is installed.
    """
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = run("pkgin list | grep -qio %(pkg_name)s" % locals())
        if res.succeeded:
            return True
        return False

def install(packages, update=False, yes=None, options=None):
    """
    Install one or more packages.

    If *update* is ``True``, the package definitions will be updated
    first, using :py:func:`~fabtools.pkg.update_index`.

    Extra *yes* may be passed to ``pkgin`` to validate license if necessary.

    Extra *options* may be passed to ``pkgin`` if necessary.

    Example::

        import fabtools

        # Update index, then verbosely install a single package
        fabtools.pkg.install('redis', update=True, options='-V',)

        # Install multiple packages
        fabtools.pkg.install([
            'unzip',
            'top'
        ])

    """
    manager = MANAGER
    if update:
        update_index()
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    if isinstance(yes, str):
        sudo('yes %(yes)s | %(manager)s %(options)s in %(packages)s' % locals())
    else:
        sudo('%(manager)s %(options)s in %(packages)s' % locals())

def uninstall(packages, orphan=False, options=None):
    """
    Remove one or more packages.

    If *orphan* is ``True``, orphan dependencies will be
    removed from the system.

    Extra *options* may be passed to ``pkgin`` if necessary.
    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    if orphan:
        sudo('%(manager)s -y ar' % locals())
    sudo('%(manager)s %(options)s remove %(packages)s' % locals())

def smartos_build():
    """
    Get the build of SmartOS. Useful to determine provider for example.

    Example::

        from fabtools.pkg import smartos_build

        if smartos_build().startswith('joyent'):
            print('SmartOS Joyent')

    """
    with settings(hide('running', 'stdout')):
        return run('uname -v')

def smartos_image():
    """
    Get the SmartOS image. Useful to determine the image/dataset for example.
    Returns None if it can't be determined.

    Example::

        from fabtools.pkg import smartos_image

        if smartos_image().startswith('percona'):
            sudo('mysql -uroot -psecretpassword -e 'show databases;'')

    """
    with settings(hide('running', 'stdout')):
        if is_file('/etc/product'):
            return run('cat /etc/product | head -n 2 | tail -n 1 | awk \'{ print $2 " " $3 }\'')
        else:
            return None
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Simple fake sendmail program. It will check that the email comes
from an authorized sender and check for spam before calling the
real sendmail. This can be useful to prevent your email delivery service
to be suspended because somebody hacked into your server and started
sending massive amounts of email.

Usually you would rename the sendmail binary and put a link to this program
as /usr/sbin/sendmail

You would also want to rename or uninstall mailx or any other means to
send email from your server (those spam sending scripts will try everything).
Dont forget to give the script exec permissions (chmod +x)

Author: Juanjo Alvarez <juanjux@gmail.com>

License: MIT https://opensource.org/licenses/mit-license.php
"""

import sys, os, antispam, time, random, string
from email import message_from_string
from subprocess import Popen, PIPE
from traceback import format_exc

pjoin = os.path.join

def get_random_fname():
    return str(int(time.time())) + '_' + \
            ''.join(random.SystemRandom().choice(
                string.ascii_uppercase + string.digits) for _ in range(4)
            )

def real_send(real_sendmail, email_text, params = None):
    params = [] if params is None else params
    ps = Popen([real_sendmail] + params, stdin=PIPE, stderr=PIPE)
    ps.stdin.write(email_text)
    (stdout, stderr) = ps.communicate()
    return ps.returncode

class SpamMessageException(Exception): pass
class UnauthorizedSenderException(Exception): pass

class EmailFilter(object):

    def __init__(self, real_sendmail = '/usr/sbin/ssmtp',
                 log_directory = '/var/log/fakesendmail',
                 notify_info = None):
        """
        Args:
            real_sendmail: the path to the real sendmail-alike binary that will be used
            to deliver message that has passed the tests

            log_directory: the directory where the subfolders with valid/invalid emails
            will be stored and the script log

            notify_info: a dictionary with the keys 'from', 'to' and 'template' with the
            delivery information for problem notifications. The template must be a string
            with {origmsgpath}, {fromaddr} and {toaddr} keys for Python' string.format
            method. With the default value of None no notifications will be sent
            (but the log will still reflect the problems).
        """

        self.real_sendmail = real_sendmail
        self.logdir = log_directory
        self.saved_email_path = None
        self.notify_info = notify_info
        self.get_params()

    def save_email(self, subdir, suffix = ''):
        assert self.email_object

        fname = get_random_fname() + suffix
        fullpath = pjoin(self.logdir, subdir, fname)
        with open(pjoin(self.logdir, 'fakesendmail.log'), 'a') as f:
            f.write(time.ctime() + '\n')
            f.write('PARAMS: ' + ','.join(self.params) + '\n')
            f.write('FILE: {0}'.format(fullpath))
            f.write('\n-------\n')

        with open(fullpath, 'w') as out:
            out.write(self.email_object.as_string())

        self.saved_email_path = fullpath

    def test_validsender(self, valid_senders):
        if valid_senders is None:
            return

        assert self.email_object

        from_ = self.email_object['from'].lower().strip()
        print from_
        if from_.lower() not in VALID_SENDERS:
            # Log it and return
            self.save_email('unauthorized_sender')
            print('ERROR: unathorized sender {0}'.format(from_))
            sys.exit(1)

    def test_spam(self, threshold):
        assert self.email_object

        detector = antispam.Detector(path = pjoin(self.logdir, 'bayesian_model', 'model.pkl'))

        if detector.score(self.email_object.as_string()) > threshold:
            self.save_email('spam')
            print('ERROR: spam email detected')
            sys.exit(1)

    def get_params(self):
        self.params = []
        self.param_addresses = []

        inline_addrs = False

        for token in sys.argv[1:]:
            if token[0] == '-':
                self.params.append(token)
                if token == '-t':
                    inline_addrs = True
                continue
            elif not inline_addrs:
                self.param_addresses.append(token)

    def notify_problem(self, error, emailpath):
        if self.notify_info:
            ninfo = self.notify_info
            msg = ninfo['template'].format(
                fromaddr = ninfo['from'],
                toaddr = ninfo['to'],
                error = error,
                origmsgpath = emailpath
            )
            real_send(self.real_sendmail, msg)

    def read_from_stdin(self):
        self.email_object = message_from_string(sys.stdin.read())

    def process_email(self, spam_threshold = 0.45, valid_senders = None):
        """
        Process the email and test for valid sender and spaminess, return
        the code that should be returned to the caller

        Args:
            spam_threshold: the value that the antispam module will use to classify
            an email as spam or not. You can also train it, check:
            http://antispam.readthedocs.io/en/latest/api.html

            valid_senders: a list of valid senders. If used the email has a "From" address
            that is not in this list, the email will not be delivered. Use the None default
            to consider all senders as valid.

        Returns:
            The return value of the real sendmail program or 1 if there was any
            other error.
        """
        assert self.email_object

        try:
            # Find if any of the valid senders is not in the valid senders
            self.test_validsender(valid_senders)

            # Find if its spam
            self.test_spam(spam_threshold)
        except:
            print format_exc()
            self.notify_problem(format_exc, self.saved_email_path)
            return 1

        # Finally deliver it calling ssmtp with the same parameters we got
        retcode = real_send(self.real_sendmail, self.email_object.as_string(),
                            self.params + self.param_addresses)
        if retcode != 0:
            self.save_email('deliver_fail', suffix = '_' + str(retcode))
        else:
            # Delivered, save a copy
            self.save_email('ok')
        return retcode

NOTIFY_TEMPLATE = """
From: {fromaddr}
To: {toaddr}
Subject: Email delivery problem detected

Problem detected with a delivery on the server. The error was:
{error}.

The original message can be find at the path:
{origmsgpath}
"""

# Yeah, change this
VALID_SENDERS = [i.lower().strip() for i in [
    'RotarySpainClub@club-rx8.com',
    'admin@rotaryspainclub.com',
    'juanjux@gmail.com',
    'juanjux+web@gmail.com',
    'juanjux+test@gmail.com',
    'juanjo@juanjoalvarez.net',
    ]
]

if __name__  == '__main__':
    notify_info = {
        'from': 'juanjux+web@gmail.com',
        'to': 'juanjux+emailproblems@gmail.com',
        'template': NOTIFY_TEMPLATE,
    }
    f = EmailFilter()
    f.read_from_stdin()
    sys.exit(f.process_email(valid_senders = VALID_SENDERS))
<commit_msg>create the log dir if it doesnt exists<commit_after>#!/usr/bin/env python
"""
Simple fake sendmail program. It will check that the email comes
from an authorized sender and check for spam before calling the
real sendmail. This can be useful to prevent your email delivery service
to be suspended because somebody hacked into your server and started
sending massive amounts of email.

Usually you would rename the sendmail binary and put a link to this program
as /usr/sbin/sendmail

You would also want to rename or uninstall mailx or any other means to
send email from your server (those spam sending scripts will try everything).
Dont forget to give the script exec permissions (chmod +x)

Author: Juanjo Alvarez <juanjux@gmail.com>

License: MIT https://opensource.org/licenses/mit-license.php
"""

import sys, os, antispam, time, random, string
from email import message_from_string
from subprocess import Popen, PIPE
from traceback import format_exc

pjoin = os.path.join

def get_random_fname():
    return str(int(time.time())) + '_' + \
            ''.join(random.SystemRandom().choice(
                string.ascii_uppercase + string.digits) for _ in range(4)
            )

def real_send(real_sendmail, email_text, params = None):
    params = [] if params is None else params
    ps = Popen([real_sendmail] + params, stdin=PIPE, stderr=PIPE)
    ps.stdin.write(email_text)
    (stdout, stderr) = ps.communicate()
    return ps.returncode

class SpamMessageException(Exception): pass
class UnauthorizedSenderException(Exception): pass

class EmailFilter(object):

    def __init__(self, real_sendmail = '/usr/sbin/ssmtp',
                 log_directory = '/var/log/fakesendmail',
                 notify_info = None):
        """
        Args:
            real_sendmail: the path to the real sendmail-alike binary that will be used
            to deliver message that has passed the tests

            log_directory: the directory where the subfolders with valid/invalid emails
            will be stored and the script log

            notify_info: a dictionary with the keys 'from', 'to' and 'template' with the
            delivery information for problem notifications. The template must be a string
            with {origmsgpath}, {fromaddr} and {toaddr} keys for Python' string.format
            method. With the default value of None no notifications will be sent
            (but the log will still reflect the problems).
        """

        self.real_sendmail = real_sendmail
        self.logdir = log_directory
        if not os.path.exists(log_directory):
            os.makedirs(log_directory)
        self.saved_email_path = None
        self.notify_info = notify_info
        self.get_params()

    def save_email(self, subdir, suffix = ''):
        assert self.email_object

        fname = get_random_fname() + suffix
        fullpath = pjoin(self.logdir, subdir, fname)
        with open(pjoin(self.logdir, 'fakesendmail.log'), 'a') as f:
            f.write(time.ctime() + '\n')
            f.write('PARAMS: ' + ','.join(self.params) + '\n')
            f.write('FILE: {0}'.format(fullpath))
            f.write('\n-------\n')

        with open(fullpath, 'w') as out:
            out.write(self.email_object.as_string())

        self.saved_email_path = fullpath

    def test_validsender(self, valid_senders):
        if valid_senders is None:
            return

        assert self.email_object

        from_ = self.email_object['from'].lower().strip()
        print from_
        if from_.lower() not in VALID_SENDERS:
            # Log it and return
            self.save_email('unauthorized_sender')
            print('ERROR: unathorized sender {0}'.format(from_))
            sys.exit(1)

    def test_spam(self, threshold):
        assert self.email_object

        detector = antispam.Detector(path = pjoin(self.logdir, 'bayesian_model', 'model.pkl'))

        if detector.score(self.email_object.as_string()) > threshold:
            self.save_email('spam')
            print('ERROR: spam email detected')
            sys.exit(1)

    def get_params(self):
        self.params = []
        self.param_addresses = []

        inline_addrs = False

        for token in sys.argv[1:]:
            if token[0] == '-':
                self.params.append(token)
                if token == '-t':
                    inline_addrs = True
                continue
            elif not inline_addrs:
                self.param_addresses.append(token)

    def notify_problem(self, error, emailpath):
        if self.notify_info:
            ninfo = self.notify_info
            msg = ninfo['template'].format(
                fromaddr = ninfo['from'],
                toaddr = ninfo['to'],
                error = error,
                origmsgpath = emailpath
            )
            real_send(self.real_sendmail, msg)

    def read_from_stdin(self):
        self.email_object = message_from_string(sys.stdin.read())

    def process_email(self, spam_threshold = 0.45, valid_senders = None):
        """
        Process the email and test for valid sender and spaminess, return
        the code that should be returned to the caller

        Args:
            spam_threshold: the value that the antispam module will use to classify
            an email as spam or not. You can also train it, check:
            http://antispam.readthedocs.io/en/latest/api.html

            valid_senders: a list of valid senders. If used the email has a "From" address
            that is not in this list, the email will not be delivered. Use the None default
            to consider all senders as valid.

        Returns:
            The return value of the real sendmail program or 1 if there was any
            other error.
        """
        assert self.email_object

        try:
            # Find if any of the valid senders is not in the valid senders
            self.test_validsender(valid_senders)

            # Find if its spam
            self.test_spam(spam_threshold)
        except:
            print format_exc()
            self.notify_problem(format_exc, self.saved_email_path)
            return 1

        # Finally deliver it calling ssmtp with the same parameters we got
        retcode = real_send(self.real_sendmail, self.email_object.as_string(),
                            self.params + self.param_addresses)
        if retcode != 0:
            self.save_email('deliver_fail', suffix = '_' + str(retcode))
        else:
            # Delivered, save a copy
            self.save_email('ok')
        return retcode

NOTIFY_TEMPLATE = """
From: {fromaddr}
To: {toaddr}
Subject: Email delivery problem detected

Problem detected with a delivery on the server. The error was:
{error}.

The original message can be find at the path:
{origmsgpath}
"""

# Yeah, change this
VALID_SENDERS = [i.lower().strip() for i in [
    'RotarySpainClub@club-rx8.com',
    'admin@rotaryspainclub.com',
    'juanjux@gmail.com',
    'juanjux+web@gmail.com',
    'juanjux+test@gmail.com',
    'juanjo@juanjoalvarez.net',
    ]
]

if __name__  == '__main__':
    notify_info = {
        'from': 'juanjux+web@gmail.com',
        'to': 'juanjux+emailproblems@gmail.com',
        'template': NOTIFY_TEMPLATE,
    }
    f = EmailFilter()
    f.read_from_stdin()
    sys.exit(f.process_email(valid_senders = VALID_SENDERS))
<|endoftext|>


<commit_before># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.

class Table:

    class InputStrContainsCarriageReturn (RuntimeError):
        pass

    class InconsistentColumnCounts (RuntimeError):
        def __init__ (self, expected_len, row):
            self.expected_len = expected_len
            self.row = row

        def __str__ (self):
            return "Expected every row to have len={:d}: {}".format(
                    self.expected_len, repr(self.row))

    def __init__ (self, tab_separated_text = None):
        self.text = tab_separated_text

        self.__raise_error_if_carriage_returns()
        self.__create_internal_structure()

    @property
    def rows (self):
        return len(self)

    @property
    def cols (self):
        return len(self.__rows[0]) if self else 0

    def body (self):
        body = iter(self)
        next(body, None)
        return body

    def add_header (self, *args):
        self.__assert_valid_length(args)
        self.__header = args

    def __len__ (self):
        if self.__header is None:
            return len(self.__rows)

        else:
            return len(self.__rows) + 1

    def __iter__ (self):
        if self.__header is not None:
            yield self.__header

        yield from self.__rows

    def __getitem__ (self, key):
        if self.__header is None:
            return self.__rows[key]

        elif key == 0:
            return self.__header

        else:
            return self.__rows[key - 1]

    def __repr__ (self):
        if self.__header is None:
            return "<{} {}>".format(self.__class__.__name__,
                                    repr(self.text))

        else:
            return "<{} {} {}>".format(self.__class__.__name__,
                                       repr(self.__header),
                                       repr(self.text))

    def __raise_error_if_carriage_returns (self):
        if self.text and "\r" in self.text:
            raise self.InputStrContainsCarriageReturn

    def __create_internal_structure (self):
        self.__header = None

        if self.text:
            self.__set_to_list_of_rows_from_text()

        else:
            self.__rows = []

    def __set_to_list_of_rows_from_text (self):
        self.__rows = [self.__split_row(r)
                       for r in self.__rows_from_text()]
        self.__raise_error_unless_col_counts_are_consistent()

    def __split_row (self, row_text):
        return tuple(row_text.split("\t"))

    def __rows_from_text (self):
        return self.text.rstrip("\n").split("\n")

    def __raise_error_unless_col_counts_are_consistent (self):
        for row in self.__rows:
            self.__assert_valid_length(row)

    def __assert_valid_length (self, row):
        if len(row) != self.cols:
            raise Table.InconsistentColumnCounts(self.cols, row)
<commit_msg>:muscle: Replace condition with inline ternary<commit_after># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.

class Table:

    class InputStrContainsCarriageReturn (RuntimeError):
        pass

    class InconsistentColumnCounts (RuntimeError):
        def __init__ (self, expected_len, row):
            self.expected_len = expected_len
            self.row = row

        def __str__ (self):
            return "Expected every row to have len={:d}: {}".format(
                    self.expected_len, repr(self.row))

    def __init__ (self, tab_separated_text = None):
        self.text = tab_separated_text

        self.__raise_error_if_carriage_returns()
        self.__create_internal_structure()

    @property
    def rows (self):
        return len(self)

    @property
    def cols (self):
        return len(self.__rows[0]) if self else 0

    def body (self):
        body = iter(self)
        next(body, None)
        return body

    def add_header (self, *args):
        self.__assert_valid_length(args)
        self.__header = args

    def __len__ (self):
        len_header = 0 if self.__header is None else 1
        return len(self.__rows) + len_header

    def __iter__ (self):
        if self.__header is not None:
            yield self.__header

        yield from self.__rows

    def __getitem__ (self, key):
        if self.__header is None:
            return self.__rows[key]

        elif key == 0:
            return self.__header

        else:
            return self.__rows[key - 1]

    def __repr__ (self):
        if self.__header is None:
            return "<{} {}>".format(self.__class__.__name__,
                                    repr(self.text))

        else:
            return "<{} {} {}>".format(self.__class__.__name__,
                                       repr(self.__header),
                                       repr(self.text))

    def __raise_error_if_carriage_returns (self):
        if self.text and "\r" in self.text:
            raise self.InputStrContainsCarriageReturn

    def __create_internal_structure (self):
        self.__header = None

        if self.text:
            self.__set_to_list_of_rows_from_text()

        else:
            self.__rows = []

    def __set_to_list_of_rows_from_text (self):
        self.__rows = [self.__split_row(r)
                       for r in self.__rows_from_text()]
        self.__raise_error_unless_col_counts_are_consistent()

    def __split_row (self, row_text):
        return tuple(row_text.split("\t"))

    def __rows_from_text (self):
        return self.text.rstrip("\n").split("\n")

    def __raise_error_unless_col_counts_are_consistent (self):
        for row in self.__rows:
            self.__assert_valid_length(row)

    def __assert_valid_length (self, row):
        if len(row) != self.cols:
            raise Table.InconsistentColumnCounts(self.cols, row)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# transform.py - Module containing flow cytometry transformations.
#
# All transformations should be of the following form:
#
#     data_out = transform(data_in, parameters)
#
# where DATA_IN and DATA_OUT are NxD numpy arrays describing N cytometry events
# observing D data dimensions, and PARAMETERS are transformation specific
# parameters.
#
# Author: John T. Sexton (john.t.sexton@rice.edu)
#          Sebastian M. Castillo-Hair (smc9@rice.edu)
# Date: 6/30/2015
#
# Requires:
#   * numpy
#   * scipy

import numpy as np
import scipy.ndimage.filters

def exponentiate(data):
    '''Exponentiate data using the following transformation:
        y = 10^(x/256)
    This transformation spaces out the data across 4 logs from 1 to 10,000.
    This is also the transformation typically applied by flow cytometer
    software.

    data    - NxD numpy array (row=event)

    returns - NxD numpy array'''

    return np.power(10, (data.astype(float)/256.0))

def _find_hist_peaks(data, labels, labels_all = None,
        min_val = 0, max_val = 1023):
    '''
    Find histogram peaks from a dataset.

    This function assumes that clustering of the data has been previously
    performed, and the labels for the different clusters are given in the
    arguments. The algorithm then proceeds as follows:
        1. For each one of the C clusters, calculate the histogram for the
            dataset.
        2. Use a 1D Gaussian filter to smooth out the histogram. The sigma
            value for the Gaussian filter is chosen based on the standard
            deviation of the fit Gaussian from the GMM for that paritcular
            segment.
        3. Identify peak as the maximum value of the Gaussian-blurred histogram.

    data        - Nx1 numpy array with the 1D data from where peaks should be
                  identified. Data values are assumed to be integer.
    labels      - Nx1 numpy array with the cluster labels of each data sample.
    labels_all  - Cx1 numpy array with all the individual cluster labels.
    min_val     - minimum possible value in data.
    max_val     - maximum possible value in data.

    returns     - Cx1 numpy array with the values of the identified peaks.
                - Cx(max_val - min_val + 1) numpy array with a smoothed
                  histogram for each cluster.
    '''

    # Check if individual labels have been provided, otherwise calculate
    if labels_all is None:
        labels_all = list(set(labels))

    # Calculate bin edges and centers
    bin_edges = np.arange(min_val, max_val + 2) - 0.5
    bin_edges[0] = -np.inf
    bin_edges[-1] = np.inf
    bin_centers = np.arange(min_val, max_val + 1)

    # Identify peaks for each cluster
    peaks = np.zeros(len(labels_all))
    hists_smooth = np.zeros([len(labels_all), len(bin_centers)])
    for i, li in enumerate(labels_all):
        # Extract data that belongs to this cluster
        data_li = data[labels == li]
        # Calculate sample mean and standard deviation
        # mu_li = np.mean(data_li)
        sigma_li = np.std(data_li)
        # Calculate histogram
        hist, __ = np.histogram(data_li, bin_edges)
        # Do Gaussian blur on histogram
        # We have found empirically that using one half of the distribution's
        # standard deviation results in a nice fit.
        hist_smooth = scipy.ndimage.filters.gaussian_filter1d(hist, sigma_li/2.)
        # Extract peak
        i_max = np.argmax(hist_smooth)
        peak = bin_centers[i_max]
        # Pack values
        peaks[i] = peak
        hists_smooth[i,:] = hist_smooth

    return peaks, hists_smooth

def _fit_mef_standards_curve(known_peaks, fit_peaks, method='log line+auto'):
    '''description'''

    if method == 'log line':
        pass
    elif method == 'exp':
        pass
    elif method == 'exp+auto':
        pass
    elif method == 'log line+auto':
        pass
    else:
        raise ValueError('Unrecognized method')

    def standards_curve(data):
        '''Transform data from Channels to Molecules of Equivalent
        Fluorophore.'''

        return data

    return standards_curve

def to_mef(data, beads, known_mef, peaks=None, sc=None):
    '''description'''

    if sc is None:
        if peaks is None:
            peaks = _find_hist_peaks(beads)

        sc = _fit_mef_standards_curve(known_mef, peaks)

    return sc(data)

def blank(data, white_cells):
    pass

def compensate(data):
    pass
<commit_msg>Added _clustering_dbscan function to transform.py.<commit_after>#!/usr/bin/python
#
# transform.py - Module containing flow cytometry transformations.
#
# All transformations should be of the following form:
#
#     data_out = transform(data_in, parameters)
#
# where DATA_IN and DATA_OUT are NxD numpy arrays describing N cytometry events
# observing D data dimensions, and PARAMETERS are transformation specific
# parameters.
#
# Author: John T. Sexton (john.t.sexton@rice.edu)
#          Sebastian M. Castillo-Hair (smc9@rice.edu)
# Date: 6/30/2015
#
# Requires:
#   * numpy
#   * scipy
#   * scikit-learn

import numpy as np
import scipy.ndimage.filters
from sklearn.cluster import DBSCAN

def exponentiate(data):
    '''Exponentiate data using the following transformation:
        y = 10^(x/256)
    This transformation spaces out the data across 4 logs from 1 to 10,000.
    This is also the transformation typically applied by flow cytometer
    software.

    data    - NxD numpy array (row=event)

    returns - NxD numpy array'''

    return np.power(10, (data.astype(float)/256.0))

def _clustering_dbscan(data, eps = 20.0, min_samples = 40):
    '''
    Find clusters in the data array using the DBSCAN method from the
    scikit-learn library.

    data        - NxD numpy array.
    eps         - Parameter for DBSCAN. Check scikit-learn documentation for
                  more info.
    min_samples - Parameter for DBSCAN. Check scikit-learn documentation for
                  more info.

    returns     - Nx1 numpy array, labeling each sample to a cluster.
    '''

    # Initialize DBSCAN object
    db = DBSCAN(eps = eps, min_samples = min_samples)
    # Fit data
    db.fit(data)
    # Extract labels
    # A value of -1 indicates no assignment to any cluster
    labels = db.labels_

    # Extract individual labels and number of labels
    labels_all = list(set(labels))
    n_labels = len(labels_all)
    n_samples = len(labels)

    # Calculate number of samples in each cluster
    n_samples_cluster = [np.sum(labels==li) for li in labels_all]

    # Check that no cluster is too small.
    # Clusters are assumed to be roughly the same size. Any cluster smaller
    # than 10 times less than the expected amount will be removed
    min_n_samples = float(n_samples)/n_labels/10.0
    labels_all_checked = []
    for i, li in enumerate(labels_all):
        if n_samples_cluster[i] < min_n_samples:
            labels[labels==li] = -1
        else:
            labels_all_checked.append(li)
    labels_all = labels_all_checked

    # Change the cluster numbers to a contiguous positive sequence
    labels_checked = -1*np.ones(len(labels))
    cn = 0
    for li in labels_all:
        labels_checked[labels==li] = cn
        cn = cn + 1
    labels = labels_checked

    assert(np.any(labels==-1) == False)

    return labels

def _find_hist_peaks(data, labels, labels_all = None,
        min_val = 0, max_val = 1023):
    '''
    Find histogram peaks from a dataset.

    This function assumes that clustering of the data has been previously
    performed, and the labels for the different clusters are given in the
    arguments. The algorithm then proceeds as follows:
        1. For each one of the C clusters, calculate the histogram for the
            dataset.
        2. Use a 1D Gaussian filter to smooth out the histogram. The sigma
            value for the Gaussian filter is chosen based on the standard
            deviation of the fit Gaussian from the GMM for that paritcular
            segment.
        3. Identify peak as the maximum value of the Gaussian-blurred histogram.

    data        - Nx1 numpy array with the 1D data from where peaks should be
                  identified. Data values are assumed to be integer.
    labels      - Nx1 numpy array with the cluster labels of each data sample.
    labels_all  - Cx1 numpy array with all the individual cluster labels.
    min_val     - minimum possible value in data.
    max_val     - maximum possible value in data.

    returns     - Cx1 numpy array with the values of the identified peaks.
                - Cx(max_val - min_val + 1) numpy array with a smoothed
                  histogram for each cluster.
    '''

    # Check if individual labels have been provided, otherwise calculate
    if labels_all is None:
        labels_all = list(set(labels))

    # Calculate bin edges and centers
    bin_edges = np.arange(min_val, max_val + 2) - 0.5
    bin_edges[0] = -np.inf
    bin_edges[-1] = np.inf
    bin_centers = np.arange(min_val, max_val + 1)

    # Identify peaks for each cluster
    peaks = np.zeros(len(labels_all))
    hists_smooth = np.zeros([len(labels_all), len(bin_centers)])
    for i, li in enumerate(labels_all):
        # Extract data that belongs to this cluster
        data_li = data[labels == li]
        # Calculate sample mean and standard deviation
        # mu_li = np.mean(data_li)
        sigma_li = np.std(data_li)
        # Calculate histogram
        hist, __ = np.histogram(data_li, bin_edges)
        # Do Gaussian blur on histogram
        # We have found empirically that using one half of the distribution's
        # standard deviation results in a nice fit.
        hist_smooth = scipy.ndimage.filters.gaussian_filter1d(hist, sigma_li/2.)
        # Extract peak
        i_max = np.argmax(hist_smooth)
        peak = bin_centers[i_max]
        # Pack values
        peaks[i] = peak
        hists_smooth[i,:] = hist_smooth

    return peaks, hists_smooth

def _fit_mef_standards_curve(known_peaks, fit_peaks, method='log line+auto'):
    '''description'''

    if method == 'log line':
        pass
    elif method == 'exp':
        pass
    elif method == 'exp+auto':
        pass
    elif method == 'log line+auto':
        pass
    else:
        raise ValueError('Unrecognized method')

    def standards_curve(data):
        '''Transform data from Channels to Molecules of Equivalent
        Fluorophore.'''

        return data

    return standards_curve

def to_mef(data, beads, known_mef, peaks=None, sc=None):
    '''description'''

    if sc is None:
        if peaks is None:
            peaks = _find_hist_peaks(beads)

        sc = _fit_mef_standards_curve(known_mef, peaks)

    return sc(data)

def blank(data, white_cells):
    pass

def compensate(data):
    pass
<|endoftext|>