<commit_before># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from builtins import str
import nacl
from nacl.secret import SecretBox

# 16-byte MAC plus a nonce is added to every message.
overhead = 16 + SecretBox.NONCE_SIZE

def encrypt(message, keyPath):
    """
    Encrypts a message given a path to a local file containing a key.

    :param message: The message to be encrypted.
    :param keyPath: A path to a file containing a 256-bit key (and nothing else).
    :type message: str
    :type keyPath: str
    :rtype: str

    A constant overhead is added to every encrypted message (for the nonce and MAC).
    >>> import tempfile
    >>> k = tempfile.mktemp()
    >>> with open(k, 'wb') as f:
    ...     f.write(nacl.utils.random(SecretBox.KEY_SIZE))
    >>> message = 'test'
    >>> len(encrypt(message, k)) == overhead + len(message)
    True
    """
    with open(keyPath) as f:
        key = f.read()
    if len(key) != SecretBox.KEY_SIZE:
        raise ValueError("Key is %d bytes, but must be exactly %d bytes" % (len(key),
                                                                            SecretBox.KEY_SIZE))
    sb = SecretBox(key)
    # We generate the nonce using secure random bits. For long enough
    # nonce size, the chance of a random nonce collision becomes
    # *much* smaller than the chance of a subtle coding error causing
    # a nonce reuse. Currently the nonce size is 192 bits--the chance
    # of a collision is astronomically low. (This approach is
    # recommended in the libsodium documentation.)
    nonce = nacl.utils.random(SecretBox.NONCE_SIZE)
    assert len(nonce) == SecretBox.NONCE_SIZE
    return bytes(sb.encrypt(message, nonce))

def decrypt(ciphertext, keyPath):
    """
    Decrypts a given message that was encrypted with the encrypt() method.

    :param ciphertext: The encrypted message (as a string).
    :param keyPath: A path to a file containing a 256-bit key (and nothing else).
    :type keyPath: str
    :rtype: str

    Raises an error if ciphertext was modified
    >>> import tempfile
    >>> k = tempfile.mktemp()
    >>> with open(k, 'w') as f:
    ...     f.write(nacl.utils.random(SecretBox.KEY_SIZE))
    >>> ciphertext = encrypt("testMessage", k)
    >>> ciphertext = chr(ord(ciphertext[0]) ^ 1) + ciphertext[1:]
    >>> decrypt(ciphertext, k)
    Traceback (most recent call last):
    ...
    CryptoError: Decryption failed. Ciphertext failed verification

    Otherwise works correctly
    >>> decrypt(encrypt("testMessage", k), k)
    'testMessage'
    """
    with open(keyPath) as f:
        key = f.read()
    if len(key) != SecretBox.KEY_SIZE:
        raise ValueError("Key is %d bytes, but must be exactly %d bytes" % (len(key),
                                                                            SecretBox.KEY_SIZE))
    sb = SecretBox(key)
    # The nonce is kept with the message.
    return sb.decrypt(ciphertext)
<commit_msg>Fix encryption tests work on bytes<commit_after># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from builtins import str
import nacl
from nacl.secret import SecretBox

# 16-byte MAC plus a nonce is added to every message.
overhead = 16 + SecretBox.NONCE_SIZE

def encrypt(message, keyPath):
    """
    Encrypts a message given a path to a local file containing a key.

    :param message: The message to be encrypted.
    :param keyPath: A path to a file containing a 256-bit key (and nothing else).
    :type message: bytes
    :type keyPath: str
    :rtype: bytes

    A constant overhead is added to every encrypted message (for the nonce and MAC).
    >>> import tempfile
    >>> k = tempfile.mktemp()
    >>> with open(k, 'wb') as f:
    ...     _ = f.write(nacl.utils.random(SecretBox.KEY_SIZE))
    >>> message = 'test'.encode('utf-8')
    >>> len(encrypt(message, k)) == overhead + len(message)
    True
    """
    with open(keyPath, 'rb') as f:
        key = f.read()
    if len(key) != SecretBox.KEY_SIZE:
        raise ValueError("Key is %d bytes, but must be exactly %d bytes" % (len(key),
                                                                            SecretBox.KEY_SIZE))
    sb = SecretBox(key)
    # We generate the nonce using secure random bits. For long enough
    # nonce size, the chance of a random nonce collision becomes
    # *much* smaller than the chance of a subtle coding error causing
    # a nonce reuse. Currently the nonce size is 192 bits--the chance
    # of a collision is astronomically low. (This approach is
    # recommended in the libsodium documentation.)
    nonce = nacl.utils.random(SecretBox.NONCE_SIZE)
    assert len(nonce) == SecretBox.NONCE_SIZE
    return bytes(sb.encrypt(message, nonce))

def decrypt(ciphertext, keyPath):
    """
    Decrypts a given message that was encrypted with the encrypt() method.

    :param ciphertext: The encrypted message (as a string).
    :param keyPath: A path to a file containing a 256-bit key (and nothing else).
    :type ciphertext: bytes
    :type keyPath: str
    :rtype: bytes

    Raises an error if ciphertext was modified
    >>> import tempfile
    >>> k = tempfile.mktemp()
    >>> with open(k, 'wb') as f:
    ...     _ = f.write(nacl.utils.random(SecretBox.KEY_SIZE))
    >>> ciphertext = encrypt("testMessage".encode('utf-8'), k)
    >>> ciphertext = b'5' + ciphertext[1:]
    >>> decrypt(ciphertext, k) # doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
    ...
    CryptoError: Decryption failed. Ciphertext failed verification

    Otherwise works correctly
    >>> decrypt(encrypt("testMessage".encode('utf-8'), k), k).decode('utf-8') # doctest: +ALLOW_UNICODE
    u'testMessage'
    """
    with open(keyPath, 'rb') as f:
        key = f.read()
    if len(key) != SecretBox.KEY_SIZE:
        raise ValueError("Key is %d bytes, but must be exactly %d bytes" % (len(key),
                                                                            SecretBox.KEY_SIZE))
    sb = SecretBox(key)
    # The nonce is kept with the message.
    return sb.decrypt(ciphertext)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Implement geodisWS."""
from roulier.transport import Transport
from datetime import datetime
import logging

log = logging.getLogger(__name__)

class GeodisTransportEdi(Transport):
    """Implement Geodis EDI"""

    def send(self, payload):
        """Call this function.

        Args:
            payload.body: XML in a string
            payload.header : auth
        Return:
            {
                response: (Requests.response)
                body: XML response (without soap)
                parts: dict of attachments
            }
        """
        body = payload['body']
        headers = payload['headers']
        message = self.transport_wrap(body, headers)
        return self.convert_to_edi(message)

    def transport_wrap(self, body, headers):
        date = datetime.now()
        return [
            ['UNB', ['UNOA', '3'],
                [headers['interchangeSender'], '22'],
                [headers['interchangeRecipient'], '22'],
                [date.strftime('%d%m%y'), date.strftime('%H:%M')],
                headers['deposit']
             ],
        ] + body + [['UNZ', '1', headers['deposit']]]

    def convert_to_edi(self, arr):
        def parse_token(token):
            if isinstance(token, list):
                return ":".join(sanitize(token))
            else:
                return sanitize(token)

        def parse_segment(segment):
            return "%s'" % "+".join([parse_token(token) for token in segment])

        def parse_lines(lines):
            return "\n".join([parse_segment(segment) for segment in lines])

        def sanitize(token):
            sanitized = (
                token
                .replace("'", "\'")
                .replace("?", "\?")
            )
            return sanitized

        return parse_lines(arr)
<commit_msg>fix depositId<commit_after># -*- coding: utf-8 -*-
"""Implement geodisWS."""
from roulier.transport import Transport
from datetime import datetime
import logging

log = logging.getLogger(__name__)

class GeodisTransportEdi(Transport):
    """Implement Geodis EDI"""

    def send(self, payload):
        """Call this function.

        Args:
            payload.body: XML in a string
            payload.header : auth
        Return:
            {
                response: (Requests.response)
                body: XML response (without soap)
                parts: dict of attachments
            }
        """
        body = payload['body']
        headers = payload['headers']
        message = self.transport_wrap(body, headers)
        return self.convert_to_edi(message)

    def transport_wrap(self, body, headers):
        date = datetime.now()
        return [
            ['UNB', ['UNOA', '3'],
                [headers['interchangeSender'], '22'],
                [headers['interchangeRecipient'], '22'],
                [date.strftime('%d%m%y'), date.strftime('%H:%M')],
                headers['depositId']
             ],
        ] + body + [['UNZ', '1', headers['depositId']]]

    def convert_to_edi(self, arr):
        def parse_token(token):
            if isinstance(token, list):
                return ":".join(sanitize(token))
            else:
                return sanitize(token)

        def parse_segment(segment):
            return "%s'" % "+".join([parse_token(token) for token in segment])

        def parse_lines(lines):
            return "\n".join([parse_segment(segment) for segment in lines])

        def sanitize(token):
            sanitized = (
                token
                .replace("'", "\'")
                .replace("?", "\?")
            )
            return sanitized

        return parse_lines(arr)
<|endoftext|>


<commit_before># Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Classes supporting creation and editing of question_groups."""

__author__ = 'John Orr (jorr@google.com)'

from common import schema_fields
from controllers.utils import BaseRESTHandler
from controllers.utils import XsrfTokenManager
from models import transforms
from models.models import QuestionDAO
from models.models import QuestionGroupDAO
from models.models import QuestionGroupDTO
import question_editor
from unit_lesson_editor import CourseOutlineRights

class QuestionGroupManagerAndEditor(question_editor.BaseDatastoreAssetEditor):
    """An editor for editing and managing question_groups."""

    def get_template_values(self, key):
        template_values = {}
        template_values['page_title'] = self.format_title('Edit Question')
        template_values['main_content'] = self.get_form(
            QuestionGroupRESTHandler, key)

        return template_values

    def get_add_question_group(self):
        self.render_page(self.get_template_values(''))

    def get_edit_question_group(self):
        self.render_page(self.get_template_values(self.request.get('key')))

class QuestionGroupRESTHandler(BaseRESTHandler):
    """REST handler for editing question_groups."""

    URI = '/rest/question_group'

    REQUIRED_MODULES = [
        'gcb-rte', 'inputex-hidden', 'inputex-select', 'inputex-string',
        'inputex-list']
    EXTRA_JS_FILES = []

    XSRF_TOKEN = 'question-group-edit'

    SCHEMA_VERSION = '1.5'

    @classmethod
    def get_schema(cls):
        """Return the InputEx schema for the question group editor."""
        question_group = schema_fields.FieldRegistry(
            'Question Group', description='question_group')

        question_group.add_property(schema_fields.SchemaField(
            'version', '', 'string', optional=True, hidden=True))
        question_group.add_property(schema_fields.SchemaField(
            'description', 'Description', 'string', optional=True))
        question_group.add_property(schema_fields.SchemaField(
            'introduction', 'Introduction', 'html', optional=True))

        item_type = schema_fields.FieldRegistry(
            'Item',
            extra_schema_dict_values={'className': 'question-group-item'})
        item_type.add_property(schema_fields.SchemaField(
            'weight', 'Weight', 'string', optional=True,
            extra_schema_dict_values={'className': 'question-group-weight'}))

        question_select_data = [
            (q.id, q.description) for q in QuestionDAO.get_all()]

        item_type.add_property(schema_fields.SchemaField(
            'question', 'Question', 'string', optional=True,
            select_data=question_select_data,
            extra_schema_dict_values={'className': 'question-group-question'}))

        item_array = schema_fields.FieldArray(
            'items', '', item_type=item_type,
            extra_schema_dict_values={
                'className': 'question-group-items',
                'sortable': 'true',
                'listAddLabel': 'Add an item',
                'listRemoveLabel': 'Delete item'})

        question_group.add_property(item_array)

        return question_group

    def get(self):
        """Respond to the REST GET verb with the contents of the group."""
        key = self.request.get('key')

        if not CourseOutlineRights.can_view(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        if key:
            question_group = QuestionGroupDAO.load(key)
            version = question_group.dict.get('version')
            if self.SCHEMA_VERSION != version:
                transforms.send_json_response(
                    self, 403, 'Cannot edit a Version %s group.' % version,
                    {'key': key})
                return
            payload_dict = question_group.dict
        else:
            payload_dict = {
                'version': self.SCHEMA_VERSION,
                'items': [{'weight': ''}, {'weight': ''}, {'weight': ''}]}

        transforms.send_json_response(
            self, 200, 'Success',
            payload_dict=payload_dict,
            xsrf_token=XsrfTokenManager.create_xsrf_token(self.XSRF_TOKEN))

    def validate(self, question_group_dict):
        """Validate the question group data sent from the form."""
        errors = []

        assert question_group_dict['version'] == self.SCHEMA_VERSION

        if not question_group_dict['description'].strip():
            errors.append('The question group must have a description.')

        if not question_group_dict['items']:
            errors.append(
                'The question group must contain at least one question.')

        items = question_group_dict['items']
        for index in range(0, len(items)):
            item = items[index]
            try:
                float(item['weight'])
            except ValueError:
                errors.append(
                    'Item %s must have a numeric weight.' % (index + 1))

        return errors

    def put(self):
        """Store a question group in the datastore in response to a PUT."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, self.XSRF_TOKEN, {'key': key}):
            return

        if not CourseOutlineRights.can_edit(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        payload = request.get('payload')
        question_group_dict = transforms.json_to_dict(
            transforms.loads(payload),
            self.get_schema().get_json_schema_dict())

        validation_errors = self.validate(question_group_dict)
        if validation_errors:
            self.validation_error('\n'.join(validation_errors), key=key)
            return

        assert self.SCHEMA_VERSION == question_group_dict.get('version')

        if key:
            question_group = QuestionGroupDTO(key, question_group_dict)
        else:
            question_group = QuestionGroupDTO(None, question_group_dict)

        QuestionGroupDAO.save(question_group)
        transforms.send_json_response(self, 200, 'Saved.')

    def delete(self):
        """Delete the question_group in response to REST request."""
        key = self.request.get('key')

        if not self.assert_xsrf_token_or_fail(
                self.request, self.XSRF_TOKEN, {'key': key}):
            return

        if not CourseOutlineRights.can_delete(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        question_group = QuestionGroupDAO.load(key)
        if not question_group:
            transforms.send_json_response(
                self, 404, 'Question Group not found.', {'key': key})
            return
        QuestionGroupDAO.delete(question_group)
        transforms.send_json_response(self, 200, 'Deleted.')

<commit_msg>Changed the title from  Edit Question to Edit Question Group for question group editing. 	Change on 2013/06/28 by juliaoh <juliaoh@google.com> ------------- Created by MOE: http://code.google.com/p/moe-java MOE_MIGRATED_REVID=48690119<commit_after># Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Classes supporting creation and editing of question_groups."""

__author__ = 'John Orr (jorr@google.com)'

from common import schema_fields
from controllers.utils import BaseRESTHandler
from controllers.utils import XsrfTokenManager
from models import transforms
from models.models import QuestionDAO
from models.models import QuestionGroupDAO
from models.models import QuestionGroupDTO
import question_editor
from unit_lesson_editor import CourseOutlineRights

class QuestionGroupManagerAndEditor(question_editor.BaseDatastoreAssetEditor):
    """An editor for editing and managing question_groups."""

    def get_template_values(self, key):
        template_values = {}
        template_values['page_title'] = self.format_title('Edit Question Group')
        template_values['main_content'] = self.get_form(
            QuestionGroupRESTHandler, key)

        return template_values

    def get_add_question_group(self):
        self.render_page(self.get_template_values(''))

    def get_edit_question_group(self):
        self.render_page(self.get_template_values(self.request.get('key')))

class QuestionGroupRESTHandler(BaseRESTHandler):
    """REST handler for editing question_groups."""

    URI = '/rest/question_group'

    REQUIRED_MODULES = [
        'gcb-rte', 'inputex-hidden', 'inputex-select', 'inputex-string',
        'inputex-list']
    EXTRA_JS_FILES = []

    XSRF_TOKEN = 'question-group-edit'

    SCHEMA_VERSION = '1.5'

    @classmethod
    def get_schema(cls):
        """Return the InputEx schema for the question group editor."""
        question_group = schema_fields.FieldRegistry(
            'Question Group', description='question_group')

        question_group.add_property(schema_fields.SchemaField(
            'version', '', 'string', optional=True, hidden=True))
        question_group.add_property(schema_fields.SchemaField(
            'description', 'Description', 'string', optional=True))
        question_group.add_property(schema_fields.SchemaField(
            'introduction', 'Introduction', 'html', optional=True))

        item_type = schema_fields.FieldRegistry(
            'Item',
            extra_schema_dict_values={'className': 'question-group-item'})
        item_type.add_property(schema_fields.SchemaField(
            'weight', 'Weight', 'string', optional=True,
            extra_schema_dict_values={'className': 'question-group-weight'}))

        question_select_data = [
            (q.id, q.description) for q in QuestionDAO.get_all()]

        item_type.add_property(schema_fields.SchemaField(
            'question', 'Question', 'string', optional=True,
            select_data=question_select_data,
            extra_schema_dict_values={'className': 'question-group-question'}))

        item_array = schema_fields.FieldArray(
            'items', '', item_type=item_type,
            extra_schema_dict_values={
                'className': 'question-group-items',
                'sortable': 'true',
                'listAddLabel': 'Add an item',
                'listRemoveLabel': 'Delete item'})

        question_group.add_property(item_array)

        return question_group

    def get(self):
        """Respond to the REST GET verb with the contents of the group."""
        key = self.request.get('key')

        if not CourseOutlineRights.can_view(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        if key:
            question_group = QuestionGroupDAO.load(key)
            version = question_group.dict.get('version')
            if self.SCHEMA_VERSION != version:
                transforms.send_json_response(
                    self, 403, 'Cannot edit a Version %s group.' % version,
                    {'key': key})
                return
            payload_dict = question_group.dict
        else:
            payload_dict = {
                'version': self.SCHEMA_VERSION,
                'items': [{'weight': ''}, {'weight': ''}, {'weight': ''}]}

        transforms.send_json_response(
            self, 200, 'Success',
            payload_dict=payload_dict,
            xsrf_token=XsrfTokenManager.create_xsrf_token(self.XSRF_TOKEN))

    def validate(self, question_group_dict):
        """Validate the question group data sent from the form."""
        errors = []

        assert question_group_dict['version'] == self.SCHEMA_VERSION

        if not question_group_dict['description'].strip():
            errors.append('The question group must have a description.')

        if not question_group_dict['items']:
            errors.append(
                'The question group must contain at least one question.')

        items = question_group_dict['items']
        for index in range(0, len(items)):
            item = items[index]
            try:
                float(item['weight'])
            except ValueError:
                errors.append(
                    'Item %s must have a numeric weight.' % (index + 1))

        return errors

    def put(self):
        """Store a question group in the datastore in response to a PUT."""
        request = transforms.loads(self.request.get('request'))
        key = request.get('key')

        if not self.assert_xsrf_token_or_fail(
                request, self.XSRF_TOKEN, {'key': key}):
            return

        if not CourseOutlineRights.can_edit(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        payload = request.get('payload')
        question_group_dict = transforms.json_to_dict(
            transforms.loads(payload),
            self.get_schema().get_json_schema_dict())

        validation_errors = self.validate(question_group_dict)
        if validation_errors:
            self.validation_error('\n'.join(validation_errors), key=key)
            return

        assert self.SCHEMA_VERSION == question_group_dict.get('version')

        if key:
            question_group = QuestionGroupDTO(key, question_group_dict)
        else:
            question_group = QuestionGroupDTO(None, question_group_dict)

        QuestionGroupDAO.save(question_group)
        transforms.send_json_response(self, 200, 'Saved.')

    def delete(self):
        """Delete the question_group in response to REST request."""
        key = self.request.get('key')

        if not self.assert_xsrf_token_or_fail(
                self.request, self.XSRF_TOKEN, {'key': key}):
            return

        if not CourseOutlineRights.can_delete(self):
            transforms.send_json_response(
                self, 401, 'Access denied.', {'key': key})
            return

        question_group = QuestionGroupDAO.load(key)
        if not question_group:
            transforms.send_json_response(
                self, 404, 'Question Group not found.', {'key': key})
            return
        QuestionGroupDAO.delete(question_group)
        transforms.send_json_response(self, 200, 'Deleted.')

<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Temporal Pooler mixin that enables detailed monitoring of history.
"""

from collections import defaultdict

import numpy

from nupic.research.monitor_mixin.metric import Metric
from nupic.research.monitor_mixin.monitor_mixin_base import MonitorMixinBase
from nupic.research.monitor_mixin.plot import Plot
from nupic.research.monitor_mixin.trace import (
  IndicesTrace, StringsTrace, BoolsTrace, MetricsTrace)

class TemporalPoolerMonitorMixin(MonitorMixinBase):
  """
  Mixin for TemporalPooler that stores a detailed history, for inspection and
  debugging.
  """

  def __init__(self, *args, **kwargs):
    super(TemporalPoolerMonitorMixin, self).__init__(*args, **kwargs)

    self._mmResetActive = True  # First iteration is always a reset

  def mmGetTraceActiveCells(self):
    """
    @return (Trace) Trace of active cells
    """
    return self._mmTraces["activeCells"]

  def mmGetTraceSequenceLabels(self):
    """
    @return (Trace) Trace of sequence labels
    """
    return self._mmTraces["sequenceLabels"]

  def mmGetTraceResets(self):
    """
    @return (Trace) Trace of resets
    """
    return self._mmTraces["resets"]

  def mmGetTraceConnectionsPerColumnMetric(self):
    """
    @return (Trace) Trace of connections per column metric
    """
    return self._mmTraces["connectionsPerColumnMetric"]

  def mmGetDataOverlap(self):
    """
    Returns 2D matrix of overlaps for sets of active cells between pairs of
    iterations. Both the rows and columns are iterations, and the values in the
    matrix are the size of overlaps between sets of active cells for those
    iterations.

    @return (numpy.array) Overlap data
    """
    self._mmComputeSequenceRepresentationData()
    return self._mmData["overlap"]

  def mmGetMetricStabilityConfusion(self):
    """
    For each iteration that doesn't follow a reset, looks at every other
    iteration for the same world that doesn't follow a reset, and computes the
    number of bits that show up in one or the other set of active cells for
    that iteration, but not both. This metric returns the distribution of those
    numbers.

    @return (Metric) Stability confusion metric
    """
    self._mmComputeSequenceRepresentationData()
    numbers = self._mmData["stabilityConfusion"]
    return Metric(self, "stability confusion", numbers)

  def mmGetMetricDistinctnessConfusion(self):
    """
    For each iteration that doesn't follow a reset, looks at every other
    iteration for every other world that doesn't follow a reset, and computes
    the number of bits that show up in both sets of active cells those that
    iteration. This metric returns the distribution of those numbers.

    @return (Metric) Distinctness confusion metric
    """
    self._mmComputeSequenceRepresentationData()
    numbers = self._mmData["distinctnessConfusion"]
    return Metric(self, "distinctness confusion", numbers)

  def mmGetPlotConnectionsPerColumn(self, title=None):
    """
    Returns plot of # connections per column.

    @return (Plot) plot
    """
    plot = Plot(self, title)
    plot.addGraph(sorted(self._connectedCounts.tolist(), reverse=True),
                  position=211,
                  xlabel="column", ylabel="# connections")
    plot.addHistogram(self._connectedCounts.tolist(),
                      position=212,
                      bins=len(self._connectedCounts) / 10,
                      xlabel="# connections", ylabel="# columns")
    return plot

  def mmGetCellActivityPlot(self, title=None, showReset=False):
    """
    Returns plot of the cell activity.
    @param title - an optional title for the figure
    @param showReset - if true, the first set of cell activities after a reset
                        will have a gray background
    @return (Plot) plot
    """
    plot = Plot(self, title)

    resetTrace = self.mmGetTraceResets().data
    activeCellTrace = self._mmTraces["activeCells"].data
    data = numpy.zeros((self._numColumns, 1))
    for i in xrange(len(activeCellTrace)):
      if showReset and resetTrace[i]:
        activity = numpy.ones((self._numColumns, 1)) / 4
      else:
        activity = numpy.zeros((self._numColumns, 1))

      activeSet = activeCellTrace[i]
      activity[list(activeSet)] = 1
      data = numpy.concatenate((data, activity), 1)

    plot.addImage(data, xlabel="Time", ylabel="Cell Activity")
    return plot

  def mmPrettyPrintDataOverlap(self):
    """
    Returns pretty-printed string representation of overlap data.
    (See `mmGetDataOverlap`.)

    @return (string) Pretty-printed data
    """
    matrix = self.mmGetDataOverlap()
    resetsTrace = self.mmGetTraceResets()

    text = ""

    for i, row in enumerate(matrix):
      if resetsTrace.data[i]:
        text += "\n"

      for j, item in enumerate(row):
        if resetsTrace.data[j]:
          text += "    "

        text += "{:4}".format(item)

      text += "\n"

    return text

  # ==============================
  # Helpers
  # ==============================

  def _mmComputeSequenceRepresentationData(self):
    if not self._sequenceRepresentationDataStale:
      return

    activeCellsTrace = self.mmGetTraceActiveCells()
    sequenceLabelsTrace = self.mmGetTraceSequenceLabels()
    resetsTrace = self.mmGetTraceResets()

    n = len(activeCellsTrace.data)
    overlap = numpy.empty((n, n), dtype=int)
    stabilityConfusion = []
    distinctnessConfusion = []

    for i in xrange(n):
      for j in xrange(i+1):
        numActiveCells = len(activeCellsTrace.data[i])
        numOverlap = len(activeCellsTrace.data[i] & activeCellsTrace.data[j])
        overlap[i][j] = numOverlap
        overlap[j][i] = numOverlap

        if (i != j and
            sequenceLabelsTrace.data[i] is not None and
            not resetsTrace.data[i] and
            sequenceLabelsTrace.data[j] is not None and
            not resetsTrace.data[j]):
          if sequenceLabelsTrace.data[i] == sequenceLabelsTrace.data[j]:
            stabilityConfusion.append(numActiveCells - numOverlap)
          else:
            distinctnessConfusion.append(numOverlap)

    self._mmData["overlap"] = overlap
    self._mmData["stabilityConfusion"] = stabilityConfusion
    self._mmData["distinctnessConfusion"] = distinctnessConfusion

    self._sequenceRepresentationDataStale = False

  # ==============================
  # Overrides
  # ==============================

  def compute(self, *args, **kwargs):
    sequenceLabel = None
    if "sequenceLabel" in kwargs:
      sequenceLabel = kwargs["sequenceLabel"]
      del kwargs["sequenceLabel"]

    activeColumns = super(TemporalPoolerMonitorMixin, self).compute(*args,
                                                                    **kwargs)
    activeColumns = set(activeColumns)
    activeCells = activeColumns  # TODO: Update when moving to a cellular TP

    self._mmTraces["activeCells"].data.append(activeCells)
    self._mmTraces["sequenceLabels"].data.append(sequenceLabel)

    self._mmTraces["resets"].data.append(self._mmResetActive)
    self._mmResetActive = False

    self._mmTraces["connectionsPerColumnMetric"].data.append(
      Metric(self, "connections per column", self._connectedCounts.tolist()))

    self._sequenceRepresentationDataStale = True

  def reset(self):
    super(TemporalPoolerMonitorMixin, self).reset()

    self._mmResetActive = True

  def mmGetDefaultTraces(self, verbosity=1):
    traces = [self.mmGetTraceActiveCells()]

    if verbosity == 1:
      traces = [trace.makeCountsTrace() for trace in traces]

    return traces + [
      self.mmGetTraceConnectionsPerColumnMetric(),
      self.mmGetTraceSequenceLabels()
    ]

  def mmGetDefaultMetrics(self, verbosity=1):
    metrics = ([Metric.createFromTrace(trace)
                for trace in self.mmGetDefaultTraces()[:-2]])

    connectionsPerColumnMetricIntial = (
      self._mmTraces["connectionsPerColumnMetric"].data[0].copy())
    connectionsPerColumnMetricIntial.title += " (initial)"
    connectionsPerColumnMetricFinal = (
      self._mmTraces["connectionsPerColumnMetric"].data[-1].copy())
    connectionsPerColumnMetricFinal.title += " (final)"

    metrics += [self.mmGetMetricStabilityConfusion(),
                self.mmGetMetricDistinctnessConfusion(),
                connectionsPerColumnMetricIntial,
                connectionsPerColumnMetricFinal]

    return metrics

  def mmClearHistory(self):
    super(TemporalPoolerMonitorMixin, self).mmClearHistory()

    self._mmTraces["activeCells"] = IndicesTrace(self, "active cells")
    self._mmTraces["sequenceLabels"] = StringsTrace(self, "sequence labels")
    self._mmTraces["resets"] = BoolsTrace(self, "resets")
    self._mmTraces["connectionsPerColumnMetric"] = MetricsTrace(
      self, "connections per column (metric)")

    self._sequenceRepresentationDataStale = True
<commit_msg>Added optional param and updated call to plot.<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Temporal Pooler mixin that enables detailed monitoring of history.
"""

from collections import defaultdict

import numpy

from nupic.research.monitor_mixin.metric import Metric
from nupic.research.monitor_mixin.monitor_mixin_base import MonitorMixinBase
from nupic.research.monitor_mixin.plot import Plot
from nupic.research.monitor_mixin.trace import (
  IndicesTrace, StringsTrace, BoolsTrace, MetricsTrace)

class TemporalPoolerMonitorMixin(MonitorMixinBase):
  """
  Mixin for TemporalPooler that stores a detailed history, for inspection and
  debugging.
  """

  def __init__(self, *args, **kwargs):
    super(TemporalPoolerMonitorMixin, self).__init__(*args, **kwargs)

    self._mmResetActive = True  # First iteration is always a reset

  def mmGetTraceActiveCells(self):
    """
    @return (Trace) Trace of active cells
    """
    return self._mmTraces["activeCells"]

  def mmGetTraceSequenceLabels(self):
    """
    @return (Trace) Trace of sequence labels
    """
    return self._mmTraces["sequenceLabels"]

  def mmGetTraceResets(self):
    """
    @return (Trace) Trace of resets
    """
    return self._mmTraces["resets"]

  def mmGetTraceConnectionsPerColumnMetric(self):
    """
    @return (Trace) Trace of connections per column metric
    """
    return self._mmTraces["connectionsPerColumnMetric"]

  def mmGetDataOverlap(self):
    """
    Returns 2D matrix of overlaps for sets of active cells between pairs of
    iterations. Both the rows and columns are iterations, and the values in the
    matrix are the size of overlaps between sets of active cells for those
    iterations.

    @return (numpy.array) Overlap data
    """
    self._mmComputeSequenceRepresentationData()
    return self._mmData["overlap"]

  def mmGetMetricStabilityConfusion(self):
    """
    For each iteration that doesn't follow a reset, looks at every other
    iteration for the same world that doesn't follow a reset, and computes the
    number of bits that show up in one or the other set of active cells for
    that iteration, but not both. This metric returns the distribution of those
    numbers.

    @return (Metric) Stability confusion metric
    """
    self._mmComputeSequenceRepresentationData()
    numbers = self._mmData["stabilityConfusion"]
    return Metric(self, "stability confusion", numbers)

  def mmGetMetricDistinctnessConfusion(self):
    """
    For each iteration that doesn't follow a reset, looks at every other
    iteration for every other world that doesn't follow a reset, and computes
    the number of bits that show up in both sets of active cells those that
    iteration. This metric returns the distribution of those numbers.

    @return (Metric) Distinctness confusion metric
    """
    self._mmComputeSequenceRepresentationData()
    numbers = self._mmData["distinctnessConfusion"]
    return Metric(self, "distinctness confusion", numbers)

  def mmGetPlotConnectionsPerColumn(self, title=None):
    """
    Returns plot of # connections per column.

    @return (Plot) plot
    """
    plot = Plot(self, title)
    plot.addGraph(sorted(self._connectedCounts.tolist(), reverse=True),
                  position=211,
                  xlabel="column", ylabel="# connections")
    plot.addHistogram(self._connectedCounts.tolist(),
                      position=212,
                      bins=len(self._connectedCounts) / 10,
                      xlabel="# connections", ylabel="# columns")
    return plot

  def mmGetCellActivityPlot(self, title=None, showReset=False,
                            resetShading=0.25):
    """ Returns plot of the cell activity.
    @param title an optional title for the figure
    @param showReset if true, the first set of cell activities after a reset
                        will have a gray background
    @param resetShading If showReset is true, this float specifies the
    intensity of the reset background with 0.0 being white and 1.0 being black
    @return (Plot) plot
    """
    plot = Plot(self, title)
    resetTrace = self.mmGetTraceResets().data
    activeCellTrace = self._mmTraces["activeCells"].data
    data = numpy.zeros((self._numColumns, 1))
    for i in xrange(len(activeCellTrace)):
      if showReset and resetTrace[i]:
        activity = numpy.ones((self._numColumns, 1)) * resetShading
      else:
        activity = numpy.zeros((self._numColumns, 1))

      activeSet = activeCellTrace[i]
      activity[list(activeSet)] = 1
      data = numpy.concatenate((data, activity), 1)

    plot.add2DArray(data, xlabel="Time", ylabel="Cell Activity")
    return plot

  def mmPrettyPrintDataOverlap(self):
    """
    Returns pretty-printed string representation of overlap data.
    (See `mmGetDataOverlap`.)

    @return (string) Pretty-printed data
    """
    matrix = self.mmGetDataOverlap()
    resetsTrace = self.mmGetTraceResets()

    text = ""

    for i, row in enumerate(matrix):
      if resetsTrace.data[i]:
        text += "\n"

      for j, item in enumerate(row):
        if resetsTrace.data[j]:
          text += "    "

        text += "{:4}".format(item)

      text += "\n"

    return text

  # ==============================
  # Helpers
  # ==============================

  def _mmComputeSequenceRepresentationData(self):
    if not self._sequenceRepresentationDataStale:
      return

    activeCellsTrace = self.mmGetTraceActiveCells()
    sequenceLabelsTrace = self.mmGetTraceSequenceLabels()
    resetsTrace = self.mmGetTraceResets()

    n = len(activeCellsTrace.data)
    overlap = numpy.empty((n, n), dtype=int)
    stabilityConfusion = []
    distinctnessConfusion = []

    for i in xrange(n):
      for j in xrange(i+1):
        numActiveCells = len(activeCellsTrace.data[i])
        numOverlap = len(activeCellsTrace.data[i] & activeCellsTrace.data[j])
        overlap[i][j] = numOverlap
        overlap[j][i] = numOverlap

        if (i != j and
            sequenceLabelsTrace.data[i] is not None and
            not resetsTrace.data[i] and
            sequenceLabelsTrace.data[j] is not None and
            not resetsTrace.data[j]):
          if sequenceLabelsTrace.data[i] == sequenceLabelsTrace.data[j]:
            stabilityConfusion.append(numActiveCells - numOverlap)
          else:
            distinctnessConfusion.append(numOverlap)

    self._mmData["overlap"] = overlap
    self._mmData["stabilityConfusion"] = stabilityConfusion
    self._mmData["distinctnessConfusion"] = distinctnessConfusion

    self._sequenceRepresentationDataStale = False

  # ==============================
  # Overrides
  # ==============================

  def compute(self, *args, **kwargs):
    sequenceLabel = None
    if "sequenceLabel" in kwargs:
      sequenceLabel = kwargs["sequenceLabel"]
      del kwargs["sequenceLabel"]

    activeColumns = super(TemporalPoolerMonitorMixin, self).compute(*args,
                                                                    **kwargs)
    activeColumns = set(activeColumns)
    activeCells = activeColumns  # TODO: Update when moving to a cellular TP

    self._mmTraces["activeCells"].data.append(activeCells)
    self._mmTraces["sequenceLabels"].data.append(sequenceLabel)

    self._mmTraces["resets"].data.append(self._mmResetActive)
    self._mmResetActive = False

    self._mmTraces["connectionsPerColumnMetric"].data.append(
      Metric(self, "connections per column", self._connectedCounts.tolist()))

    self._sequenceRepresentationDataStale = True

  def reset(self):
    super(TemporalPoolerMonitorMixin, self).reset()

    self._mmResetActive = True

  def mmGetDefaultTraces(self, verbosity=1):
    traces = [self.mmGetTraceActiveCells()]

    if verbosity == 1:
      traces = [trace.makeCountsTrace() for trace in traces]

    return traces + [
      self.mmGetTraceConnectionsPerColumnMetric(),
      self.mmGetTraceSequenceLabels()
    ]

  def mmGetDefaultMetrics(self, verbosity=1):
    metrics = ([Metric.createFromTrace(trace)
                for trace in self.mmGetDefaultTraces()[:-2]])

    connectionsPerColumnMetricIntial = (
      self._mmTraces["connectionsPerColumnMetric"].data[0].copy())
    connectionsPerColumnMetricIntial.title += " (initial)"
    connectionsPerColumnMetricFinal = (
      self._mmTraces["connectionsPerColumnMetric"].data[-1].copy())
    connectionsPerColumnMetricFinal.title += " (final)"

    metrics += [self.mmGetMetricStabilityConfusion(),
                self.mmGetMetricDistinctnessConfusion(),
                connectionsPerColumnMetricIntial,
                connectionsPerColumnMetricFinal]

    return metrics

  def mmClearHistory(self):
    super(TemporalPoolerMonitorMixin, self).mmClearHistory()

    self._mmTraces["activeCells"] = IndicesTrace(self, "active cells")
    self._mmTraces["sequenceLabels"] = StringsTrace(self, "sequence labels")
    self._mmTraces["resets"] = BoolsTrace(self, "resets")
    self._mmTraces["connectionsPerColumnMetric"] = MetricsTrace(
      self, "connections per column (metric)")

    self._sequenceRepresentationDataStale = True
<|endoftext|>


<commit_before>pred_bz2_file = "shape_predictor_68_face_landmarks.dat.bz2"
www_pred_path = "http://dlib.net/files/"+pred_bz2_file
rec_bz2_file = "dlib_face_recognition_resnet_model_v1.dat.bz2"
www_rec_path = "http://dlib.net/files/"+rec_bz2_file

def download_model(url, local_path, bz2_file):
  import os
  import urllib
  print "Downloading model from: {}".format(url)
  out_dir = os.path.dirname(local_path)
  try:
    os.makedirs(out_dir)
  except:
    pass
  bz2_local_path = os.path.join(out_dir,bz2_file)
  urllib.urlretrieve(url, bz2_local_path)
  # need to unzip
  unzip_model(bz2_local_path, local_path)

def unzip_model(bz2_model_path, local_path):
  print "Unzipping file: {}".format(bz2_model_path)
  import bz2
  bz2model = bz2.BZ2File(bz2_model_path, 'r')
  with open(local_path, 'w') as out:
    data = bz2model.read()
    out.write(data)

class DLibFeaturizer(object):

  def __init__(self, global_conf_filename, prefix="DLIBFEAT_"):
    #TODO read predictor_path, face_rec_model_path from conf
    import dlib
    import json
    import os
    self.global_conf_filename = global_conf_filename
    self.prefix = prefix
    self.verbose = 0
    self.global_conf = json.load(open(global_conf_filename, 'rt'))
    pred_path = str(self.get_param('pred_path'))
    if not pred_path:
      raise ValueError('[DLibFeaturizer: error] pred_path was not set in config file.')
    # test if file exits there
    if not os.path.exists(pred_path):
      # download file if not
      download_model(www_pred_path, pred_path, pred_bz2_file)
    self.sp = dlib.shape_predictor(pred_path)
    rec_path = str(self.get_param('rec_path'))
    if not pred_path:
      raise ValueError('[DLibFeaturizer: error] pred_path was not set in config file.')
    # test if file exits there
    if not os.path.exists(rec_path):
      # download file if not
      download_model(www_rec_path, rec_path, rec_bz2_file)
    self.facerec = dlib.face_recognition_model_v1(rec_path)

  def get_param(self, param):
    key_param = self.prefix + param
    if key_param in self.global_conf:
      return self.global_conf[key_param]
    if self.verbose:
      print '[get_param: info] could not find {} in configuration'.format(key_param)

  def featurize(self, img, d):
    #TODO: deal with B&W images
    print img.shape,len(img.shape)
    if len(img.shape):
      import skimage
      img = skimage.color.gray2rgb(img)
    from dlib import rectangle
    dlib_bbox = rectangle(d['left'], d['top'], d['right'], d['bottom'])
    shape = self.sp(img, dlib_bbox)
    return self.facerec.compute_face_descriptor(img, shape)<commit_msg>add score<commit_after>pred_bz2_file = "shape_predictor_68_face_landmarks.dat.bz2"
www_pred_path = "http://dlib.net/files/"+pred_bz2_file
rec_bz2_file = "dlib_face_recognition_resnet_model_v1.dat.bz2"
www_rec_path = "http://dlib.net/files/"+rec_bz2_file

def download_model(url, local_path, bz2_file):
  import os
  import urllib
  print "Downloading model from: {}".format(url)
  out_dir = os.path.dirname(local_path)
  try:
    os.makedirs(out_dir)
  except:
    pass
  bz2_local_path = os.path.join(out_dir,bz2_file)
  urllib.urlretrieve(url, bz2_local_path)
  # need to unzip
  unzip_model(bz2_local_path, local_path)

def unzip_model(bz2_model_path, local_path):
  print "Unzipping file: {}".format(bz2_model_path)
  import bz2
  bz2model = bz2.BZ2File(bz2_model_path, 'r')
  with open(local_path, 'w') as out:
    data = bz2model.read()
    out.write(data)

class DLibFeaturizer(object):

  def __init__(self, global_conf_filename, prefix="DLIBFEAT_"):
    #TODO read predictor_path, face_rec_model_path from conf
    import dlib
    import json
    import os
    self.global_conf_filename = global_conf_filename
    self.prefix = prefix
    self.verbose = 0
    self.global_conf = json.load(open(global_conf_filename, 'rt'))
    pred_path = str(self.get_param('pred_path'))
    if not pred_path:
      raise ValueError('[DLibFeaturizer: error] pred_path was not set in config file.')
    # test if file exits there
    if not os.path.exists(pred_path):
      # download file if not
      download_model(www_pred_path, pred_path, pred_bz2_file)
    self.sp = dlib.shape_predictor(pred_path)
    rec_path = str(self.get_param('rec_path'))
    if not pred_path:
      raise ValueError('[DLibFeaturizer: error] pred_path was not set in config file.')
    # test if file exits there
    if not os.path.exists(rec_path):
      # download file if not
      download_model(www_rec_path, rec_path, rec_bz2_file)
    self.facerec = dlib.face_recognition_model_v1(rec_path)

  def get_param(self, param):
    key_param = self.prefix + param
    if key_param in self.global_conf:
      return self.global_conf[key_param]
    if self.verbose:
      print '[get_param: info] could not find {} in configuration'.format(key_param)

  def featurize(self, img, d):
    #TODO: deal with B&W images
    #print img.shape,len(img.shape)
    if len(img.shape)==2:
      import skimage
      img = skimage.color.gray2rgb(img)
    from dlib import rectangle
    dlib_bbox = rectangle(d['left'], d['top'], d['right'], d['bottom'])
    shape = self.sp(img, dlib_bbox)
    return self.facerec.compute_face_descriptor(img, shape)<|endoftext|>


<commit_before>'''
A blast runner hardly tied to backbone folder structure
'''

# Copyright 2009 Jose Blanca, Peio Ziarsolo, COMAV-Univ. Politecnica Valencia
# This file is part of franklin.
# franklin is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# franklin is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with franklin. If not, see <http://www.gnu.org/licenses/>.

import logging
from os.path import join, exists, splitext, basename, split
from os import makedirs, symlink, remove, listdir
from franklin.backbone.specifications import (BACKBONE_DIRECTORIES,
                                              BACKBONE_BASENAMES)
from franklin.utils.cmd_utils import call
from franklin.utils.misc_utils import get_fhand
from franklin.seq.readers import guess_seq_file_format
from franklin.utils.seqio_utils import seqio
from tempfile import NamedTemporaryFile

class NullHandler(logging.Handler):
    def emit(self, record):
        pass
MOCK_HANDLER = NullHandler()
LOGGER_NAME = 'franklin'
logging.getLogger(LOGGER_NAME).addHandler(MOCK_HANDLER)

def _get_basename(fpath):
    'It returns the base name without path and extension'
    return splitext(basename(fpath))[0]

def _create_temp_fasta_file(fpath):
    'It creates a fasta file format temfile'
    fasta_fhand = NamedTemporaryFile(suffix='.fasta', mode='a')
    seqio(in_seq_fhand=open(fpath),
          out_seq_fhand=fasta_fhand,
          out_format='fasta')
    return fasta_fhand

def guess_blastdb_kind(blastdb):
    'it infers the kind of the blastdb'

    blastdir, basename_ = split(blastdb)
    for file_ in listdir(blastdir):
        if file_.startswith(basename_) and file_ != basename_:
            if splitext(file_)[1][1] == 'n':
                return 'nucl'
            elif splitext(file_)[1][1] == 'p':
                return 'prot'

def backbone_blast_runner(query_fpath, project_dir, blast_program,
                          blast_db=None, blast_db_seq=None, dbtype='nucl',
                          threads=False):
    '''It returns the blast if the results doesn't exist'''
    if blast_db is None and blast_db_seq is None:
        raise RuntimeError('It needs a blast database or seqfile')

    #create a logger
    logger = logging.getLogger(LOGGER_NAME)

    query_basename = _get_basename(query_fpath)
    blast_dir = join(project_dir, BACKBONE_DIRECTORIES['blast_dir'])

    if blast_db:
        result_dir = join(blast_dir, query_basename, _get_basename(blast_db))
    else:
        result_dir = join(blast_dir, query_basename,
                          _get_basename(blast_db_seq))
    if not exists(result_dir):
        makedirs(result_dir)
    result_fpath = join(result_dir,
                        '%s.%s.xml' % (BACKBONE_BASENAMES['blast_basename'],
                                       blast_program))
    if exists(result_fpath):
        logger.info('Using the stored blast result %s' % result_fpath)
        return result_fpath

    #the input file should be fasta
    fasta_query_fhand = None
    fasta_db_fhand = None
    if guess_seq_file_format(open(query_fpath)) != 'fasta':
        fasta_query_fhand = _create_temp_fasta_file(query_fpath)
        query_fpath = fasta_query_fhand.name

    #we have to create a database in BACKBONE_DIRECTORIES['blast_databases']
    if blast_db_seq:
        blast_db = make_backbone_blast_db(project_dir, blast_db_seq, dbtype)

    logger.info('Running the blast %s' % result_fpath)
    try:
        blast_runner(query_fpath, blast_db, blast_program, result_fpath,
                     threads=threads)
    except RuntimeError as error:
        if exists(result_fpath):
            remove(result_fpath)
            msg = '%s \n database: %s\ndatabase type: %s' % (str(error),
                                                             blast_db, dbtype)
        raise RuntimeError(msg)

    if fasta_query_fhand:
        fasta_query_fhand.close()
    if fasta_db_fhand:
        fasta_db_fhand.close()

    return result_fpath

def blast_runner(seq_fpath, blast_db, blast_type, result_fpath,
                 threads=False):
    'It runs a blast giving a file and a database path'
    #threads = get_num_threads(threads)
    cmd = ['blast2', '-d', blast_db, '-p', blast_type, '-v', '25',
           '-b', '25', '-e', '0.0001', '-m', '7',
           '-i', seq_fpath, '-o', result_fpath,
           '-a', str(threads)]

    call(cmd, raise_on_error=True, log=True)

def make_backbone_blast_db(project_dir, blast_db_seq, dbtype):
    'It formats a blastdb when need it'
    logger = logging.getLogger(LOGGER_NAME)
    #the name should be the basename of the blast_db_seq
    db_dir = join(project_dir, BACKBONE_DIRECTORIES['blast_databases'])
    if not exists(db_dir):
        makedirs(db_dir)
    db_seq_fpath = join(db_dir, _get_basename(blast_db_seq))
    if not exists(db_seq_fpath):
        #which is the name of the new databae?
        blast_db_seq_format = guess_seq_file_format(open(blast_db_seq))
        if blast_db_seq_format == 'fasta':
            symlink(blast_db_seq, db_seq_fpath)
        else:
            seqio(in_seq_fhand=open(blast_db_seq),
                  out_seq_fhand=open(db_seq_fpath, 'w'),
                  out_format='fasta')
        logger.info('Formatting the database %s' % db_seq_fpath)
        try:
            makeblastdb(db_seq_fpath, dbtype=dbtype)
        except RuntimeError:
            msg = 'Error making blastdb. db:%s\n dbtype:%s\n' % \
                                               (db_seq_fpath, dbtype)
            remove(db_seq_fpath)
            raise RuntimeError(msg)
    return db_seq_fpath

def makeblastdb(seq_fpath, dbtype):
    'It creates the blast db database'
    dbtype = 'T' if dbtype == 'prot' else 'F'
    cmd = ['formatdb', '-i', seq_fpath, '-V', '-p', dbtype, '-o']
    call(cmd, raise_on_error=True)

def blast_runner_plust(seq_fpath, blast_db, blast_type, result_fpath):
    'It runs a blast giving a file and a database path'
    cmd = [blast_type, '-db', blast_db, '-num_alignments', '25',
           '-num_descriptions', '25', '-evalue', '0.0001', '-outfmt', '5',
           '-query', seq_fpath, '-out', result_fpath]
    call(cmd, raise_on_error=True, log=True)

def makeblastdb_plus(seq_fpath, dbtype, outputdb=None):
    'It creates the blast db database'
    cmd = ['makeblastdb', '-in', seq_fpath, '-dbtype', dbtype]
    if outputdb is not None:
        cmd.extend(['-out', outputdb])
    call(cmd, raise_on_error=True)
<commit_msg>bugfix: blast runner does not deal with threads=False<commit_after>'''
A blast runner hardly tied to backbone folder structure
'''

# Copyright 2009 Jose Blanca, Peio Ziarsolo, COMAV-Univ. Politecnica Valencia
# This file is part of franklin.
# franklin is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# franklin is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with franklin. If not, see <http://www.gnu.org/licenses/>.

import logging
from os.path import join, exists, splitext, basename, split
from os import makedirs, symlink, remove, listdir
from franklin.backbone.specifications import (BACKBONE_DIRECTORIES,
                                              BACKBONE_BASENAMES)
from franklin.utils.cmd_utils import call
from franklin.utils.misc_utils import get_num_threads
from franklin.seq.readers import guess_seq_file_format
from franklin.utils.seqio_utils import seqio
from tempfile import NamedTemporaryFile

class NullHandler(logging.Handler):
    def emit(self, record):
        pass
MOCK_HANDLER = NullHandler()
LOGGER_NAME = 'franklin'
logging.getLogger(LOGGER_NAME).addHandler(MOCK_HANDLER)

def _get_basename(fpath):
    'It returns the base name without path and extension'
    return splitext(basename(fpath))[0]

def _create_temp_fasta_file(fpath):
    'It creates a fasta file format temfile'
    fasta_fhand = NamedTemporaryFile(suffix='.fasta', mode='a')
    seqio(in_seq_fhand=open(fpath),
          out_seq_fhand=fasta_fhand,
          out_format='fasta')
    return fasta_fhand

def guess_blastdb_kind(blastdb):
    'it infers the kind of the blastdb'

    blastdir, basename_ = split(blastdb)
    for file_ in listdir(blastdir):
        if file_.startswith(basename_) and file_ != basename_:
            if splitext(file_)[1][1] == 'n':
                return 'nucl'
            elif splitext(file_)[1][1] == 'p':
                return 'prot'

def backbone_blast_runner(query_fpath, project_dir, blast_program,
                          blast_db=None, blast_db_seq=None, dbtype='nucl',
                          threads=False):
    '''It returns the blast if the results doesn't exist'''
    if blast_db is None and blast_db_seq is None:
        raise RuntimeError('It needs a blast database or seqfile')

    #create a logger
    logger = logging.getLogger(LOGGER_NAME)

    query_basename = _get_basename(query_fpath)
    blast_dir = join(project_dir, BACKBONE_DIRECTORIES['blast_dir'])

    if blast_db:
        result_dir = join(blast_dir, query_basename, _get_basename(blast_db))
    else:
        result_dir = join(blast_dir, query_basename,
                          _get_basename(blast_db_seq))
    if not exists(result_dir):
        makedirs(result_dir)
    result_fpath = join(result_dir,
                        '%s.%s.xml' % (BACKBONE_BASENAMES['blast_basename'],
                                       blast_program))
    if exists(result_fpath):
        logger.info('Using the stored blast result %s' % result_fpath)
        return result_fpath

    #the input file should be fasta
    fasta_query_fhand = None
    fasta_db_fhand = None
    if guess_seq_file_format(open(query_fpath)) != 'fasta':
        fasta_query_fhand = _create_temp_fasta_file(query_fpath)
        query_fpath = fasta_query_fhand.name

    #we have to create a database in BACKBONE_DIRECTORIES['blast_databases']
    if blast_db_seq:
        blast_db = make_backbone_blast_db(project_dir, blast_db_seq, dbtype)

    logger.info('Running the blast %s' % result_fpath)
    try:
        blast_runner(query_fpath, blast_db, blast_program, result_fpath,
                     threads=threads)
    except RuntimeError as error:
        if exists(result_fpath):
            remove(result_fpath)
        msg = '%s \n database: %s\ndatabase type: %s' % (str(error),
                                                             blast_db, dbtype)
        raise RuntimeError(msg)

    if fasta_query_fhand:
        fasta_query_fhand.close()
    if fasta_db_fhand:
        fasta_db_fhand.close()

    return result_fpath

def blast_runner(seq_fpath, blast_db, blast_type, result_fpath,
                 threads=False):
    'It runs a blast giving a file and a database path'
    threads = get_num_threads(threads)
    cmd = ['blast2', '-d', blast_db, '-p', blast_type, '-v', '25',
           '-b', '25', '-e', '0.0001', '-m', '7',
           '-i', seq_fpath, '-o', result_fpath,
           '-a', str(threads)]

    call(cmd, raise_on_error=True, log=True)

def make_backbone_blast_db(project_dir, blast_db_seq, dbtype):
    'It formats a blastdb when need it'
    logger = logging.getLogger(LOGGER_NAME)
    #the name should be the basename of the blast_db_seq
    db_dir = join(project_dir, BACKBONE_DIRECTORIES['blast_databases'])
    if not exists(db_dir):
        makedirs(db_dir)
    db_seq_fpath = join(db_dir, _get_basename(blast_db_seq))
    if not exists(db_seq_fpath):
        #which is the name of the new databae?
        blast_db_seq_format = guess_seq_file_format(open(blast_db_seq))
        if blast_db_seq_format == 'fasta':
            symlink(blast_db_seq, db_seq_fpath)
        else:
            seqio(in_seq_fhand=open(blast_db_seq),
                  out_seq_fhand=open(db_seq_fpath, 'w'),
                  out_format='fasta')
        logger.info('Formatting the database %s' % db_seq_fpath)
        try:
            makeblastdb(db_seq_fpath, dbtype=dbtype)
        except RuntimeError:
            msg = 'Error making blastdb. db:%s\n dbtype:%s\n' % \
                                               (db_seq_fpath, dbtype)
            remove(db_seq_fpath)
            raise RuntimeError(msg)
    return db_seq_fpath

def makeblastdb(seq_fpath, dbtype):
    'It creates the blast db database'
    dbtype = 'T' if dbtype == 'prot' else 'F'
    cmd = ['formatdb', '-i', seq_fpath, '-V', '-p', dbtype, '-o']
    call(cmd, raise_on_error=True)

def blast_runner_plust(seq_fpath, blast_db, blast_type, result_fpath):
    'It runs a blast giving a file and a database path'
    cmd = [blast_type, '-db', blast_db, '-num_alignments', '25',
           '-num_descriptions', '25', '-evalue', '0.0001', '-outfmt', '5',
           '-query', seq_fpath, '-out', result_fpath]
    call(cmd, raise_on_error=True, log=True)

def makeblastdb_plus(seq_fpath, dbtype, outputdb=None):
    'It creates the blast db database'
    cmd = ['makeblastdb', '-in', seq_fpath, '-dbtype', dbtype]
    if outputdb is not None:
        cmd.extend(['-out', outputdb])
    call(cmd, raise_on_error=True)
<|endoftext|>


<commit_before>"""
Tests for messaging_contentstore.contentstore.
"""

from unittest import TestCase

from requests import HTTPError
from requests.adapters import HTTPAdapter
from requests_testadapter import TestSession, Resp, TestAdapter

from verified_fake.fake_contentstore import Request, FakeContentStoreApi

from messaging_contentstore.contentstore import ContentStoreApiClient

class FakeContentStoreApiAdapter(HTTPAdapter):

    """
    Adapter for FakeContentStoreApi

    This inherits directly from HTTPAdapter instead of using TestAdapter
    because it overrides everything TestAdaptor does.
    """

    def __init__(self, contentstore_api):
        self.contentstore_api = contentstore_api
        super(FakeContentStoreApiAdapter, self).__init__()

    def send(self, request, stream=False, timeout=None,
             verify=True, cert=None, proxies=None):
        # print request.path_url
        req = Request(
            request.method, request.path_url, request.body, request.headers)
        resp = self.contentstore_api.handle_request(req)
        response = Resp(resp.body, resp.code, resp.headers)
        r = self.build_response(request, response)
        # print r.status_code
        return r

make_messageset_dict = FakeContentStoreApi.make_messageset_dict
make_message_dict = FakeContentStoreApi.make_message_dict

class TestContentStoreApiClient(TestCase):
    API_URL = "http://example.com/contentstore"
    AUTH_TOKEN = "auth_token"

    def setUp(self):
        self.messageset_data = {}
        self.schedule_data = {}
        self.message_data = {}
        self.binary_content_data = {}
        self.contentstore_backend = FakeContentStoreApi(
            "contentstore/", self.AUTH_TOKEN,
            messageset_data=self.messageset_data,
            schedule_data=self.schedule_data, message_data=self.message_data,
            binary_content_data=self.binary_content_data)
        self.contentstore_backend.messagesets.endpoint_data = {}
        self.contentstore_backend.messages.endpoint_data = {}
        self.contentstore_backend.schedules.endpoint_data = {}
        self.contentstore_backend.binary_contents.endpoint_data = {}
        self.session = TestSession()
        adapter = FakeContentStoreApiAdapter(self.contentstore_backend)
        self.session.mount(self.API_URL, adapter)
        # self.session = Session()

    def make_client(self, auth_token=AUTH_TOKEN):
        return ContentStoreApiClient(
            auth_token, api_url=self.API_URL, session=self.session)

    def make_existing_messageset(self, messageset_data):
        existing_messageset = make_messageset_dict(messageset_data)
        self.contentstore_backend.messagesets.endpoint_data[
            existing_messageset[u"id"]] = existing_messageset
        return existing_messageset

    def make_existing_message(self, message_data):
        existing_message = make_message_dict(message_data)
        self.contentstore_backend.messages.endpoint_data[
            existing_message[u"id"]] = existing_message
        return existing_message

    def assert_messageset_status(self, messageset_id, exists=True):
        exists_status = (messageset_id in self.messageset_data)
        self.assertEqual(exists_status, exists)

    def assert_http_error(self, expected_status, func, *args, **kw):
        try:
            func(*args, **kw)
        except HTTPError as err:
            self.assertEqual(err.response.status_code, expected_status)
        else:
            self.fail(
                "Expected HTTPError with status %s." % (expected_status,))

    def test_assert_http_error(self):
        self.session.mount("http://bad.example.com/", TestAdapter("", 500))

        def bad_req():
            r = self.session.get("http://bad.example.com/")
            r.raise_for_status()

        # Fails when no exception is raised.
        self.assertRaises(
            self.failureException, self.assert_http_error, 404, lambda: None)

        # Fails when an HTTPError with the wrong status code is raised.
        self.assertRaises(
            self.failureException, self.assert_http_error, 404, bad_req)

        # Passes when an HTTPError with the expected status code is raised.
        self.assert_http_error(500, bad_req)

        # Non-HTTPError exceptions aren't caught.
        def raise_error():
            raise ValueError()

        self.assertRaises(ValueError, self.assert_http_error, 404, raise_error)

    def test_default_session(self):
        import requests
        contentstore = ContentStoreApiClient(self.AUTH_TOKEN)
        self.assertTrue(isinstance(contentstore.session, requests.Session))

    def test_default_api_url(self):
        contentstore = ContentStoreApiClient(self.AUTH_TOKEN)
        self.assertEqual(
            contentstore.api_url, "http://testserver/contentstore")

    def test_auth_failure(self):
        contentstore = self.make_client(auth_token="bogus_token")
        self.assert_http_error(403, contentstore.get_messagesets)

    def test_get_messageset(self):
        expected_messageset = self.make_existing_messageset({
            u"short_name": u"Full Set",
            u"notes": u"A full set of messages.",
            u"default_schedule": 1
        })
        contentstore = self.make_client()
        [messageset] = list(contentstore.get_messagesets())
        self.assertEqual(messageset, expected_messageset)

    def test_create_messageset(self):
        contentstore = self.make_client()
        new_messageset = contentstore.create_messageset({
            u"short_name": u"Full Set1",
            u"notes": u"A full and new set of messages.",
            u"default_schedule": 1
        })
        [messageset] = list(contentstore.get_messagesets())
        self.assertEqual(messageset["short_name"], new_messageset["short_name"])

    def test_get_message(self):
        expected_message = self.make_existing_message({
            "messageset": 1,
            "sequence_number": 2,
            "lang": "afr_ZA",
            "text_content": "Message two"
        })
        contentstore = self.make_client()
        [message] = list(contentstore.get_messages())
        self.assertEqual(message, expected_message)

    def test_create_message(self):
        contentstore = self.make_client()
        new_message = contentstore.create_message({
            "messageset": 1,
            "sequence_number": 2,
            "lang": "afr_ZA",
            "text_content": "Message two"
        })
        [message] = list(contentstore.get_messages())
        self.assertEqual(message["text_content"],
                         new_message["text_content"])
<commit_msg>flake8<commit_after>"""
Tests for messaging_contentstore.contentstore.
"""

from unittest import TestCase

from requests import HTTPError
from requests.adapters import HTTPAdapter
from requests_testadapter import TestSession, Resp, TestAdapter

from verified_fake.fake_contentstore import Request, FakeContentStoreApi

from messaging_contentstore.contentstore import ContentStoreApiClient

class FakeContentStoreApiAdapter(HTTPAdapter):

    """
    Adapter for FakeContentStoreApi

    This inherits directly from HTTPAdapter instead of using TestAdapter
    because it overrides everything TestAdaptor does.
    """

    def __init__(self, contentstore_api):
        self.contentstore_api = contentstore_api
        super(FakeContentStoreApiAdapter, self).__init__()

    def send(self, request, stream=False, timeout=None,
             verify=True, cert=None, proxies=None):
        # print request.path_url
        req = Request(
            request.method, request.path_url, request.body, request.headers)
        resp = self.contentstore_api.handle_request(req)
        response = Resp(resp.body, resp.code, resp.headers)
        r = self.build_response(request, response)
        # print r.status_code
        return r

make_messageset_dict = FakeContentStoreApi.make_messageset_dict
make_message_dict = FakeContentStoreApi.make_message_dict

class TestContentStoreApiClient(TestCase):
    API_URL = "http://example.com/contentstore"
    AUTH_TOKEN = "auth_token"

    def setUp(self):
        self.messageset_data = {}
        self.schedule_data = {}
        self.message_data = {}
        self.binary_content_data = {}
        self.contentstore_backend = FakeContentStoreApi(
            "contentstore/", self.AUTH_TOKEN,
            messageset_data=self.messageset_data,
            schedule_data=self.schedule_data, message_data=self.message_data,
            binary_content_data=self.binary_content_data)
        self.contentstore_backend.messagesets.endpoint_data = {}
        self.contentstore_backend.messages.endpoint_data = {}
        self.contentstore_backend.schedules.endpoint_data = {}
        self.contentstore_backend.binary_contents.endpoint_data = {}
        self.session = TestSession()
        adapter = FakeContentStoreApiAdapter(self.contentstore_backend)
        self.session.mount(self.API_URL, adapter)
        # self.session = Session()

    def make_client(self, auth_token=AUTH_TOKEN):
        return ContentStoreApiClient(
            auth_token, api_url=self.API_URL, session=self.session)

    def make_existing_messageset(self, messageset_data):
        existing_messageset = make_messageset_dict(messageset_data)
        self.contentstore_backend.messagesets.endpoint_data[
            existing_messageset[u"id"]] = existing_messageset
        return existing_messageset

    def make_existing_message(self, message_data):
        existing_message = make_message_dict(message_data)
        self.contentstore_backend.messages.endpoint_data[
            existing_message[u"id"]] = existing_message
        return existing_message

    def assert_messageset_status(self, messageset_id, exists=True):
        exists_status = (messageset_id in self.messageset_data)
        self.assertEqual(exists_status, exists)

    def assert_http_error(self, expected_status, func, *args, **kw):
        try:
            func(*args, **kw)
        except HTTPError as err:
            self.assertEqual(err.response.status_code, expected_status)
        else:
            self.fail(
                "Expected HTTPError with status %s." % (expected_status,))

    def test_assert_http_error(self):
        self.session.mount("http://bad.example.com/", TestAdapter("", 500))

        def bad_req():
            r = self.session.get("http://bad.example.com/")
            r.raise_for_status()

        # Fails when no exception is raised.
        self.assertRaises(
            self.failureException, self.assert_http_error, 404, lambda: None)

        # Fails when an HTTPError with the wrong status code is raised.
        self.assertRaises(
            self.failureException, self.assert_http_error, 404, bad_req)

        # Passes when an HTTPError with the expected status code is raised.
        self.assert_http_error(500, bad_req)

        # Non-HTTPError exceptions aren't caught.
        def raise_error():
            raise ValueError()

        self.assertRaises(ValueError, self.assert_http_error, 404, raise_error)

    def test_default_session(self):
        import requests
        contentstore = ContentStoreApiClient(self.AUTH_TOKEN)
        self.assertTrue(isinstance(contentstore.session, requests.Session))

    def test_default_api_url(self):
        contentstore = ContentStoreApiClient(self.AUTH_TOKEN)
        self.assertEqual(
            contentstore.api_url, "http://testserver/contentstore")

    def test_auth_failure(self):
        contentstore = self.make_client(auth_token="bogus_token")
        self.assert_http_error(403, contentstore.get_messagesets)

    def test_get_messageset(self):
        expected_messageset = self.make_existing_messageset({
            u"short_name": u"Full Set",
            u"notes": u"A full set of messages.",
            u"default_schedule": 1
        })
        contentstore = self.make_client()
        [messageset] = list(contentstore.get_messagesets())
        self.assertEqual(messageset, expected_messageset)

    def test_create_messageset(self):
        contentstore = self.make_client()
        new_messageset = contentstore.create_messageset({
            u"short_name": u"Full Set1",
            u"notes": u"A full and new set of messages.",
            u"default_schedule": 1
        })
        [messageset] = list(contentstore.get_messagesets())
        self.assertEqual(
            messageset["short_name"], new_messageset["short_name"])

    def test_get_message(self):
        expected_message = self.make_existing_message({
            "messageset": 1,
            "sequence_number": 2,
            "lang": "afr_ZA",
            "text_content": "Message two"
        })
        contentstore = self.make_client()
        [message] = list(contentstore.get_messages())
        self.assertEqual(message, expected_message)

    def test_create_message(self):
        contentstore = self.make_client()
        new_message = contentstore.create_message({
            "messageset": 1,
            "sequence_number": 2,
            "lang": "afr_ZA",
            "text_content": "Message two"
        })
        [message] = list(contentstore.get_messages())
        self.assertEqual(message["text_content"],
                         new_message["text_content"])
<|endoftext|>