<commit_before># -*- coding: utf-8 -*-
from hamcrest import assert_that
from hamcrest import has_entries
from hamcrest import not_none

from pyworkplace.message import Message

RECIPIENT_ID = 10000000
URL = ''

def test_send_message():

    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)

    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'Hola Mundo',
    }
    assert_that(
        message.send_message(**args),
        not_none(),
    )

    body = {
        'message': 'Hola Mundo',
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_text_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
    }
    assert_that(
        message.send_text_message(**args),
        not_none(),
    )

    body = {
        'message': {'text': 'o/ World'},
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_quick_replies_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
        'quick_replies': [
            {
                'content_type': 'text',
                'title': 'Search',
                'payload': '<POSTBACK_PAYLOAD>',
                'image_url': 'http://example.com/img/red.png',
            },
        ],
    }
    assert_that(
        message.send_quick_replies_message(**args),
        not_none(),
    )

    body = {
        'message': {
            'text': 'o/ World',
            'quick_replies': [
                {
                    'content_type': 'text',
                    'title': 'Search',
                    'payload': '<POSTBACK_PAYLOAD>',
                    'image_url': 'http://example.com/img/red.png',
                },
            ],
        },
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',

    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )
<commit_msg>!#34 Add: test multiple message quickreplies<commit_after># -*- coding: utf-8 -*-
from hamcrest import assert_that
from hamcrest import has_entries
from hamcrest import not_none

from pyworkplace.message import Message

RECIPIENT_ID = 10000000
URL = ''

def test_send_message():

    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)

    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'Hola Mundo',
    }
    assert_that(
        message.send_message(**args),
        not_none(),
    )

    body = {
        'message': 'Hola Mundo',
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_text_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
    }
    assert_that(
        message.send_text_message(**args),
        not_none(),
    )

    body = {
        'message': {'text': 'o/ World'},
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_quick_replies_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
        'quick_replies': [
            {
                'content_type': 'text',
                'title': 'Search',
                'payload': '<POSTBACK_PAYLOAD>',
                'image_url': 'http://example.com/img/red.png',
            },
        ],
    }
    assert_that(
        message.send_quick_replies_message(**args),
        not_none(),
    )

    body = {
        'message': {
            'text': 'o/ World',
            'quick_replies': [
                {
                    'content_type': 'text',
                    'title': 'Search',
                    'payload': '<POSTBACK_PAYLOAD>',
                    'image_url': 'http://example.com/img/red.png',
                },
            ],
        },
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',

    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_quick_replies_multiple_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
        'quick_replies': [
            {
                'content_type': 'text',
                'title': 'Title1',
                'payload': '<POSTBACK_PAYLOAD-1>',
                'image_url': 'http://example.com/img/1.png',
            },
            {
                'content_type': 'text',
                'title': 'Title2',
                'payload': '<POSTBACK_PAYLOAD-2>',
                'image_url': 'http://example.com/img/2.png',
            },
        ],
    }
    assert_that(
        message.send_quick_replies_message(**args),
        not_none(),
    )

    body = {
        'message': {
            'text': 'o/ World',
            'quick_replies': [
                {
                    'content_type': 'text',
                    'title': 'Title1',
                    'payload': '<POSTBACK_PAYLOAD-1>',
                    'image_url': 'http://example.com/img/1.png',
                },
                {
                    'content_type': 'text',
                    'title': 'Title2',
                    'payload': '<POSTBACK_PAYLOAD-2>',
                    'image_url': 'http://example.com/img/2.png',
                },
            ],
        },
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',

    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#  Licensed under the Apache License, Version 2.0 (the 'License'); you may
#  not use this file except in compliance with the License. You may obtain
#  a copy of the License at
#
#       https://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT
#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#  License for the specific language governing permissions and limitations
#  under the License.

from __future__ import unicode_literals, absolute_import

import unittest

from linebot.models import (
    TextMessage, ImageMessage, VideoMessage, AudioMessage,
    LocationMessage, StickerMessage, FileMessage,
)
from linebot.utils import to_snake_case

class TestMessages(unittest.TestCase):
    @staticmethod
    def rename_keys(d):
        return {to_snake_case(k): v for k, v in d.items()}

    def setUp(self):
        self.content_provider_dict = {'type': 'external',
                                      'originalContentUrl': 'https://aaa.com',
                                      'previewImageUrl': 'https://bbb.com'}

        self.content_provider_dict2 = {'type': 'external', 'originalContentUrl': 'https://aaa.com'}

        self.text_message_ev = {'type': 'text', 'id': 'message_id', 'text': 'message_text'}
        self.text_message = TextMessage(**self.rename_keys(self.text_message_ev))

        self.image_message_ev = {'id': 'message_id', 'type': 'image',
                                 'contentProvider': self.content_provider_dict}
        self.image_message = ImageMessage(**self.rename_keys(self.image_message_ev))

        self.video_message_ev = {'id': 'message_id', 'type': 'video', 'duration': 60000,
                                 'contentProvider': self.content_provider_dict}
        self.video_message = VideoMessage(**self.rename_keys(self.video_message_ev))

        self.audio_message_ev = {'id': 'message_id', 'type': 'audio', 'duration': 60000,
                                 'contentProvider': self.content_provider_dict2}
        self.audio_message = AudioMessage(**self.rename_keys(self.audio_message_ev))

        self.location_message_ev = {'type': 'location', 'id': 'message_id', 'title': 'title',
                                    'address': '〒150-0002 東京都渋谷区渋谷２丁目２１−１',
                                    'latitude': 35.65910807942215, 'longitude': 139.70372892916203}
        self.location_message = LocationMessage(**self.rename_keys(self.location_message_ev))

        self.sticker_message_ev = {'type': 'sticker', 'id': 'message_id',
                                   'packageId': '1', 'stickerId': '1'}
        self.sticker_message = StickerMessage(**self.rename_keys(self.sticker_message_ev))

        self.file_message_ev = {'type': 'file', 'id': 'message_id',
                                'fileName': 'file.txt', 'fileSize': 2138}
        self.file_message = FileMessage(**self.rename_keys(self.file_message_ev))

    def test_as_json_dict(self):
        self.assertEqual(self.text_message_ev, self.text_message.as_json_dict())
        self.assertEqual(self.image_message_ev, self.image_message.as_json_dict())
        self.assertEqual(self.video_message_ev, self.video_message.as_json_dict())
        self.assertEqual(self.audio_message_ev, self.audio_message.as_json_dict())
        self.assertEqual(self.location_message_ev, self.location_message.as_json_dict())
        self.assertEqual(self.sticker_message_ev, self.sticker_message.as_json_dict())
        self.assertEqual(self.file_message_ev, self.file_message.as_json_dict())

if __name__ == '__main__':
    unittest.main()
<commit_msg>remove json serialize test<commit_after><|endoftext|>


<commit_before>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import models, fields, api, _
from odoo.exceptions import ValidationError
import logging
_logger = logging.getLogger(__name__)

class WebsiteDocToc(models.Model):
    _name = 'website.doc.toc'
    _description = 'Documentation ToC'
    _inherit = ['website.seo.metadata', 'mail.thread']
    _order = "sequence, parent_path"
    _parent_order = "sequence, name"
    _parent_store = True

    sequence = fields.Integer(
        default=10,
    )
    name = fields.Char(
        required=True,
        # to avoid complications we disable translation
        # translate=True,
        index=True,
    )
    parent_id = fields.Many2one(
        'website.doc.toc',
        'Parent Table Of Content',
        ondelete='cascade',
        domain=[('is_article', '=', False)],
        auto_join=True,
    )
    child_ids = fields.One2many(
        'website.doc.toc',
        'parent_id',
        'Children Table Of Content',
        copy=True,
        domain=[('is_article', '=', False)],
        auto_join=True,
    )
    parent_path = fields.Char(
        index=True
    )
    is_article = fields.Boolean(
        'Is Article?',
        index=True,
    )
    article_ids = fields.One2many(
        'website.doc.toc',
        'parent_id',
        'Articles',
        domain=[('is_article', '=', True)],
        context={'default_is_article': 1},
        copy=True,
        auto_join=True,
    )
    add_google_doc = fields.Boolean(
        'Add Google Doc?',
        help="Add Google Doc after Page Content?",
    )
    google_doc_link = fields.Char(
        'Google Document Code',
    )
    google_doc_code = fields.Char(
        'Google Document Code',
    )
    google_doc_height = fields.Char(
        'Google Document Height',
        default='1050px',
    )
    content = fields.Html(
        'Content',
        sanitize=False,
    )
    google_doc = fields.Text(
        'Google Content',
        compute='_compute_google_doc',
    )
    state = fields.Selection([
        ('private', 'Is Private'),
        ('portal', 'Portal'),
        ('published', 'Published'),
    ],
        'State',
        required=True,
        default='private',
        help="If private, then it wont be accesible "
             "by portal or public users",
        index=True,
    )
    dont_show_childs = fields.Boolean(
    )
    url_suffix = fields.Char(
        compute='_compute_url',
        compute_sudo=True,
    )
    documentation_id = fields.Many2one(
        'website.doc.toc',
        'Documentation',
        compute_sudo=True,
        compute='_compute_documentation',
        help='First documentation toc',
        # lo almacenamos para mejoras de performance
        store=True,
    )
    icon = fields.Char(
        help='fa-icon name, you can use any of the icons on '
        'http://fontawesome.io/icons/, for eg. "fa-pencil-square-o"',
    )
    read_status = fields.Boolean(
        'Read Status',
        compute='_compute_read',
        # inverse='_inverse_read'
    )
    reading_percentage = fields.Integer(
        'Reading percentage',
        compute="_compute_reading_percentage",
    )

    @api.constrains('state')
    def check_published(self):
        for rec in self:
            if rec.state == 'published':
                parents_not_published = rec.search([
                    ('id', 'parent_of', rec.id), ('state', '!=', 'published')])
                if parents_not_published:
                    raise ValidationError(_(
                        'No puede publicar este articulo porque hay articulos'
                        ' padres no publicados'))
            elif rec.state == 'portal':
                parents_private = rec.search([
                    ('id', 'parent_of', rec.id), ('state', '=', 'private')])
                if parents_private:
                    raise ValidationError(_(
                        'No puede publicar este articulo porque hay articulos'
                        ' padres privados'))
                childs_published = rec.search([
                    ('id', 'child_of', rec.id), ('state', '=', 'published')])
                if childs_published:
                    raise ValidationError(_(
                        'No puede pasar a portal este articulo porque hay '
                        ' publicados'))
            else:
                childs_published = rec.search([
                    ('id', 'child_of', rec.id), ('state', '!=', 'private')])
                if childs_published:
                    raise ValidationError(_(
                        'No puede despublicr este articulo porque hay hijos'
                        ' publicados'))

    @api.multi
    def _get_doc_status(self):
        self.ensure_one()
        return self.env['website.doc.status'].search(
            [('user_id', '=', self._uid), ('article_doc_id', '=', self.id)],
            limit=1)

    @api.multi
    def _compute_reading_percentage(self):
        _logger.info('Computing reading percentage')
        for rec in self:
            child_articules = self.search(
                [('is_article', '=', True), ('id', 'child_of', rec.id)])
            if child_articules:
                total = len(child_articules)
                child_read = len(child_articules.filtered(
                    lambda p: p.read_status))
                rec.reading_percentage = int((
                    float(child_read) / float(total)) * 100)

    @api.multi
    def _compute_read(self):
        _logger.info('Computing read status')
        for rec in self:
            rec.read_status = True if rec._get_doc_status() else False

    # lo implementamos sin funcion inverse del campo porque si no requeria
    # que usuarios portal tengan permiso de escritura sobre este objeto si
    # bien era un dummy write
    @api.multi
    def inverse_read(self, read_status):
        status_obj = self.env['website.doc.status']
        for rec in self:
            status = rec._get_doc_status()
            if read_status and not status:
                vals = {
                    'user_id': rec._uid,
                    'article_doc_id': rec.id,
                }
                status_obj.create(vals)
            elif not read_status and status:
                status.unlink()

    @api.multi
    # sacamos depends para que no de error con cache y newid
    # @api.depends('parent_id')
    def _compute_url(self):
        _logger.info('Computing doc urls')
        # uuid = self._context.get('uuid')
        for rec in self:
            rec.url_suffix = '/doc/%s/%s' % (rec.documentation_id.id, rec.id)
            # if uuid:
            #     rec.url_suffix = '%s/%s/%s' % (
            #         rec.url_suffix, uuid)

    @api.depends('is_article', 'parent_id')
    def _compute_documentation(self):
        _logger.info('Computing documentation')
        for rec in self:
            # if rec.is_article:
            #     parent_toc = rec.article_toc_id
            # else:
            #     parent_toc = rec
            # the first parent is the documentation
            if not isinstance(rec.id, int):
                continue
            rec.documentation_id = rec.search([
                # ('id', 'parent_of', parent_toc.id),
                ('id', 'parent_of', rec.id),
                ('is_article', '=', False),
                ('parent_id', '=', False)], limit=1).id

    @api.depends('google_doc_code', 'google_doc_height')
    def _compute_google_doc(self):
        for rec in self:
            rec.google_doc = False
            if rec.google_doc_height and rec.google_doc_code:
                google_doc = google_doc_template % (
                    rec.google_doc_height, rec.google_doc_code)
                rec.google_doc = google_doc

    @api.constrains('parent_id')
    def _check_parent_id(self):
        if not self._check_recursion():
            raise ValidationError(
                _('Error! You cannot create recursive categories.'))

    @api.multi
    def action_open_childs(self):
        return {
            'name': _('Childs TOC'),
            'view_type': 'form',
            'view_mode': 'tree,form',
            'res_model': self._name,
            'view_id': False,
            'type': 'ir.actions.act_window',
            'domain': [('parent_id', '=', self.id)],
        }

google_doc_template = """

"""
<commit_msg>[REF]<commit_after>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import models, fields, api, _
from odoo.exceptions import ValidationError
import logging
_logger = logging.getLogger(__name__)

class WebsiteDocToc(models.Model):
    _name = 'website.doc.toc'
    _description = 'Documentation ToC'
    _inherit = ['website.seo.metadata', 'mail.thread']
    _order = "sequence, parent_path"
    _parent_order = "sequence, name"
    _parent_store = True

    sequence = fields.Integer(
        default=10,
    )
    name = fields.Char(
        required=True,
        # to avoid complications we disable translation
        # translate=True,
        index=True,
    )
    parent_id = fields.Many2one(
        'website.doc.toc',
        'Parent Table Of Content',
        ondelete='cascade',
        domain=[('is_article', '=', False)],
        auto_join=True,
    )
    child_ids = fields.One2many(
        'website.doc.toc',
        'parent_id',
        'Children Table Of Content',
        copy=True,
        domain=[('is_article', '=', False)],
        auto_join=True,
    )
    parent_path = fields.Char(
        index=True
    )
    is_article = fields.Boolean(
        'Is Article?',
        index=True,
    )
    article_ids = fields.One2many(
        'website.doc.toc',
        'parent_id',
        'Articles',
        domain=[('is_article', '=', True)],
        context={'default_is_article': 1},
        copy=True,
        auto_join=True,
    )
    add_google_doc = fields.Boolean(
        'Add Google Doc?',
        help="Add Google Doc after Page Content?",
    )
    google_doc_link = fields.Char(
        'Google Document Link',
    )
    google_doc_code = fields.Char(
        'Google Document Code',
    )
    google_doc_height = fields.Char(
        'Google Document Height',
        default='1050px',
    )
    content = fields.Html(
        'Content',
        sanitize=False,
    )
    google_doc = fields.Text(
        'Google Content',
        compute='_compute_google_doc',
    )
    state = fields.Selection([
        ('private', 'Is Private'),
        ('portal', 'Portal'),
        ('published', 'Published'),
    ],
        'State',
        required=True,
        default='private',
        help="If private, then it wont be accesible "
             "by portal or public users",
        index=True,
    )
    dont_show_childs = fields.Boolean(
    )
    url_suffix = fields.Char(
        compute='_compute_url',
        compute_sudo=True,
    )
    documentation_id = fields.Many2one(
        'website.doc.toc',
        'Documentation',
        compute_sudo=True,
        compute='_compute_documentation',
        help='First documentation toc',
        # lo almacenamos para mejoras de performance
        store=True,
    )
    icon = fields.Char(
        help='fa-icon name, you can use any of the icons on '
        'http://fontawesome.io/icons/, for eg. "fa-pencil-square-o"',
    )
    read_status = fields.Boolean(
        'Read Status',
        compute='_compute_read',
        # inverse='_inverse_read'
    )
    reading_percentage = fields.Integer(
        'Reading percentage',
        compute="_compute_reading_percentage",
    )

    @api.constrains('state')
    def check_published(self):
        for rec in self:
            if rec.state == 'published':
                parents_not_published = rec.search([
                    ('id', 'parent_of', rec.id), ('state', '!=', 'published')])
                if parents_not_published:
                    raise ValidationError(_(
                        'No puede publicar este articulo porque hay articulos'
                        ' padres no publicados'))
            elif rec.state == 'portal':
                parents_private = rec.search([
                    ('id', 'parent_of', rec.id), ('state', '=', 'private')])
                if parents_private:
                    raise ValidationError(_(
                        'No puede publicar este articulo porque hay articulos'
                        ' padres privados'))
                childs_published = rec.search([
                    ('id', 'child_of', rec.id), ('state', '=', 'published')])
                if childs_published:
                    raise ValidationError(_(
                        'No puede pasar a portal este articulo porque hay '
                        ' publicados'))
            else:
                childs_published = rec.search([
                    ('id', 'child_of', rec.id), ('state', '!=', 'private')])
                if childs_published:
                    raise ValidationError(_(
                        'No puede despublicr este articulo porque hay hijos'
                        ' publicados'))

    @api.multi
    def _get_doc_status(self):
        self.ensure_one()
        return self.env['website.doc.status'].search(
            [('user_id', '=', self._uid), ('article_doc_id', '=', self.id)],
            limit=1)

    @api.multi
    def _compute_reading_percentage(self):
        _logger.info('Computing reading percentage')
        for rec in self:
            child_articules = self.search(
                [('is_article', '=', True), ('id', 'child_of', rec.id)])
            if child_articules:
                total = len(child_articules)
                child_read = len(child_articules.filtered(
                    lambda p: p.read_status))
                rec.reading_percentage = int((
                    float(child_read) / float(total)) * 100)

    @api.multi
    def _compute_read(self):
        _logger.info('Computing read status')
        for rec in self:
            rec.read_status = True if rec._get_doc_status() else False

    # lo implementamos sin funcion inverse del campo porque si no requeria
    # que usuarios portal tengan permiso de escritura sobre este objeto si
    # bien era un dummy write
    @api.multi
    def inverse_read(self, read_status):
        status_obj = self.env['website.doc.status']
        for rec in self:
            status = rec._get_doc_status()
            if read_status and not status:
                vals = {
                    'user_id': rec._uid,
                    'article_doc_id': rec.id,
                }
                status_obj.create(vals)
            elif not read_status and status:
                status.unlink()

    @api.multi
    # sacamos depends para que no de error con cache y newid
    # @api.depends('parent_id')
    def _compute_url(self):
        _logger.info('Computing doc urls')
        # uuid = self._context.get('uuid')
        for rec in self:
            rec.url_suffix = '/doc/%s/%s' % (rec.documentation_id.id, rec.id)
            # if uuid:
            #     rec.url_suffix = '%s/%s/%s' % (
            #         rec.url_suffix, uuid)

    @api.depends('is_article', 'parent_id')
    def _compute_documentation(self):
        _logger.info('Computing documentation')
        for rec in self:
            # if rec.is_article:
            #     parent_toc = rec.article_toc_id
            # else:
            #     parent_toc = rec
            # the first parent is the documentation
            if not isinstance(rec.id, int):
                continue
            rec.documentation_id = rec.search([
                # ('id', 'parent_of', parent_toc.id),
                ('id', 'parent_of', rec.id),
                ('is_article', '=', False),
                ('parent_id', '=', False)], limit=1).id

    @api.depends('google_doc_code', 'google_doc_height')
    def _compute_google_doc(self):
        for rec in self:
            rec.google_doc = False
            if rec.google_doc_height and rec.google_doc_code:
                google_doc = google_doc_template % (
                    rec.google_doc_height, rec.google_doc_code)
                rec.google_doc = google_doc

    @api.constrains('parent_id')
    def _check_parent_id(self):
        if not self._check_recursion():
            raise ValidationError(
                _('Error! You cannot create recursive categories.'))

    @api.multi
    def action_open_childs(self):
        return {
            'name': _('Childs TOC'),
            'view_type': 'form',
            'view_mode': 'tree,form',
            'res_model': self._name,
            'view_id': False,
            'type': 'ir.actions.act_window',
            'domain': [('parent_id', '=', self.id)],
        }

google_doc_template = """

"""
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

from loguru import logger

import unittest
import os.path as op
import pytest

path_to_script = op.dirname(op.abspath(__file__))

import sys

sys.path.insert(0, op.abspath(op.join(path_to_script, "../../io3d")))
sys.path.insert(0, op.abspath(op.join(path_to_script, "../../imma")))
# import sys
# import os.path

# imcut_path =  os.path.join(path_to_script, "../../imcut/")
# sys.path.insert(0, imcut_path)
import lisa.virtual_resection
import numpy as np
import io3d

class OrganSegmentationNavigation(unittest.TestCase):

    # def test_relabel_organ_segmentation(self):
    #     import lisa.organ_segmentation
    #     import io3d
    #     # datap = io3d.datasets.generate_abdominal()
    #     datap = io3d.datasets.generate_synthetic_liver(return_dataplus=True)
    #     slab = datap["slab"]
    #     oseg = lisa.organ_segmentation.OrganSegmentation()
    #     oseg.import_dataplus(datap)
    #     self.assertGreater(np.sum(oseg.select_label("porta")), 0, "Generated porta should be bigger than 0 voxels")
    #     oseg.segmentation_relabel("porta", "liver")
    #
    #     self.assertEqual(np.sum(oseg.select_label("porta")), 0)
    #     self.assertGreater(np.sum(oseg.select_label("liver")), 100)
    #
    # def test_relabel_organ_segmentation_from_multiple_labels(self):
    #     import lisa.organ_segmentation
    #     import io3d
    #     # datap = io3d.datasets.generate_abdominal()
    #     datap = io3d.datasets.generate_synthetic_liver(return_dataplus=True)
    #     slab = datap["slab"]
    #     slab["tumor"] = 3
    #     datap["segmentation"][5:10, 5:10, 5:10] = slab["tumor"]
    #     oseg = lisa.organ_segmentation.OrganSegmentation()
    #     oseg.import_dataplus(datap)
    #     self.assertGreater(np.sum(oseg.select_label("porta")), 0, "Generated porta should be bigger than 0 voxels")
    #     oseg.segmentation_relabel(["porta", "tumor"], "liver")
    #
    #     self.assertEqual(np.sum(oseg.select_label("porta")), 0)
    #     self.assertEqual(np.sum(oseg.select_label("tumor")), 0)
    #     self.assertGreater(np.sum(oseg.select_label("liver")), 100)

    def test_bodynavigation(self):

        oseg = lisa.organ_segmentation.OrganSegmentation()
        oseg.load_data(io3d.datasets.join_path("medical", "orig", "3Dircadb1.1", "PATIENT_DICOM", get_root=True))
        oseg.get_body_navigation_structures()
        self.assertGreater(np.sum(oseg.select_label("spine")), 100)

    def test_bodynavigation_precise(self):

        oseg = lisa.organ_segmentation.OrganSegmentation()
        oseg.load_data(io3d.datasets.join_path("medical", "orig", "3Dircadb1.1", "PATIENT_DICOM", get_root=True))
        oseg.get_body_navigation_structures_precise()
        self.assertGreater(np.sum(oseg.select_label("bones")), 100)

<commit_msg>mark test as slow<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

from loguru import logger

import unittest
import os.path as op
import pytest

path_to_script = op.dirname(op.abspath(__file__))

import sys

sys.path.insert(0, op.abspath(op.join(path_to_script, "../../io3d")))
sys.path.insert(0, op.abspath(op.join(path_to_script, "../../imma")))
# import sys
# import os.path

# imcut_path =  os.path.join(path_to_script, "../../imcut/")
# sys.path.insert(0, imcut_path)
import lisa.virtual_resection
import numpy as np
import io3d

class OrganSegmentationNavigation(unittest.TestCase):

    # def test_relabel_organ_segmentation(self):
    #     import lisa.organ_segmentation
    #     import io3d
    #     # datap = io3d.datasets.generate_abdominal()
    #     datap = io3d.datasets.generate_synthetic_liver(return_dataplus=True)
    #     slab = datap["slab"]
    #     oseg = lisa.organ_segmentation.OrganSegmentation()
    #     oseg.import_dataplus(datap)
    #     self.assertGreater(np.sum(oseg.select_label("porta")), 0, "Generated porta should be bigger than 0 voxels")
    #     oseg.segmentation_relabel("porta", "liver")
    #
    #     self.assertEqual(np.sum(oseg.select_label("porta")), 0)
    #     self.assertGreater(np.sum(oseg.select_label("liver")), 100)
    #
    # def test_relabel_organ_segmentation_from_multiple_labels(self):
    #     import lisa.organ_segmentation
    #     import io3d
    #     # datap = io3d.datasets.generate_abdominal()
    #     datap = io3d.datasets.generate_synthetic_liver(return_dataplus=True)
    #     slab = datap["slab"]
    #     slab["tumor"] = 3
    #     datap["segmentation"][5:10, 5:10, 5:10] = slab["tumor"]
    #     oseg = lisa.organ_segmentation.OrganSegmentation()
    #     oseg.import_dataplus(datap)
    #     self.assertGreater(np.sum(oseg.select_label("porta")), 0, "Generated porta should be bigger than 0 voxels")
    #     oseg.segmentation_relabel(["porta", "tumor"], "liver")
    #
    #     self.assertEqual(np.sum(oseg.select_label("porta")), 0)
    #     self.assertEqual(np.sum(oseg.select_label("tumor")), 0)
    #     self.assertGreater(np.sum(oseg.select_label("liver")), 100)

    def test_bodynavigation(self):

        oseg = lisa.organ_segmentation.OrganSegmentation()
        oseg.load_data(io3d.datasets.join_path("medical", "orig", "3Dircadb1.1", "PATIENT_DICOM", get_root=True))
        oseg.get_body_navigation_structures()
        self.assertGreater(np.sum(oseg.select_label("spine")), 100)

    @pytest.mark.slow
    def test_bodynavigation_precise(self):

        oseg = lisa.organ_segmentation.OrganSegmentation()
        oseg.load_data(io3d.datasets.join_path("medical", "orig", "3Dircadb1.1", "PATIENT_DICOM", get_root=True))
        oseg.get_body_navigation_structures_precise()
        self.assertGreater(np.sum(oseg.select_label("bones")), 100)

<|endoftext|>


<commit_before>import os
import shutil
import inspect
from importlib import import_module
import numpy as np
from snntoolbox.bin.utils import initialize_simulator, run_pipeline
from snntoolbox.datasets.utils import get_dataset
from tests.conftest import pytorch_skip_if_dependency_missing
from tests.core.test_models import get_correlations

# Pytorch needs channel dimension first.
import keras
keras.backend.set_image_data_format('channels_first')

@pytorch_skip_if_dependency_missing
class TestPytorchParser:
    """Test parsing pytorch models."""

    @staticmethod
    def prepare_model(model, config):
        import torch

        path_wd = config.get('paths', 'path_wd')
        model_name = config.get('paths', 'filename_ann')
        torch.save(model.state_dict(), os.path.join(path_wd,
                                                    model_name + '.pkl'))
        # Need to copy model definition over to temp dir because only weights
        # were saved.
        from tests.parsing.models import pytorch
        source_path = inspect.getfile(pytorch)
        shutil.copyfile(source_path, os.path.join(path_wd, model_name + '.py'))

    def test_loading(self, _model_4, _config):

        import keras
        assert keras.backend.image_data_format() == 'channels_first', \
            "Pytorch to Keras parser needs image_data_format == channel_first."

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': True,
                      'parse': False,
                      'normalize': False,
                      'convert': False,
                      'simulate': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {'num_to_test': 100,
                           'batch_size': 50}}

        _config.read_dict(updates)

        initialize_simulator(_config)

        normset, testset = get_dataset(_config)

        model_lib = import_module('snntoolbox.parsing.model_libs.' +
                                  _config.get('input', 'model_lib') +
                                  '_input_lib')
        input_model = model_lib.load(_config.get('paths', 'path_wd'),
                                     _config.get('paths', 'filename_ann'))

        # Evaluate input model.
        acc = model_lib.evaluate(input_model['val_fn'],
                                 _config.getint('simulation', 'batch_size'),
                                 _config.getint('simulation', 'num_to_test'),
                                 **testset)

        assert acc >= 0.95

    def test_parsing(self, _model_4, _config):

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': True,
                      'parse': True,
                      'normalize': False,
                      'convert': False,
                      'simulate': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {'num_to_test': 100,
                           'batch_size': 50}
        }

        _config.read_dict(updates)

        initialize_simulator(_config)

        acc = run_pipeline(_config)

        assert acc[0] >= 0.95

    def test_normalizing(self, _model_4, _config):

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': True,
                      'parse': True,
                      'normalize': True,
                      'convert': False,
                      'simulate': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {'num_to_test': 100,
                           'batch_size': 50}
        }

        _config.read_dict(updates)

        initialize_simulator(_config)

        acc = run_pipeline(_config)

        assert acc[0] >= 0.95

    def test_pipeline(self, _model_4, _config):

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {
                'duration': 100,
                'num_to_test': 100,
                'batch_size': 50},
            'output': {
                'log_vars': {'activations_n_b_l', 'spiketrains_n_b_l_t'}}}

        _config.read_dict(updates)

        initialize_simulator(_config)

        acc = run_pipeline(_config)

        assert acc[0] >= 0.95

        corr = get_correlations(_config)
        assert np.all(corr[:-1] > 0.97)
        assert corr[-1] > 0.5
<commit_msg>Increased tolerance for pytorch test.<commit_after>import os
import shutil
import inspect
from importlib import import_module
import numpy as np
from snntoolbox.bin.utils import initialize_simulator, run_pipeline
from snntoolbox.datasets.utils import get_dataset
from tests.conftest import pytorch_skip_if_dependency_missing
from tests.core.test_models import get_correlations

# Pytorch needs channel dimension first.
import keras
keras.backend.set_image_data_format('channels_first')

@pytorch_skip_if_dependency_missing
class TestPytorchParser:
    """Test parsing pytorch models."""

    @staticmethod
    def prepare_model(model, config):
        import torch

        path_wd = config.get('paths', 'path_wd')
        model_name = config.get('paths', 'filename_ann')
        torch.save(model.state_dict(), os.path.join(path_wd,
                                                    model_name + '.pkl'))
        # Need to copy model definition over to temp dir because only weights
        # were saved.
        from tests.parsing.models import pytorch
        source_path = inspect.getfile(pytorch)
        shutil.copyfile(source_path, os.path.join(path_wd, model_name + '.py'))

    def test_loading(self, _model_4, _config):

        import keras
        assert keras.backend.image_data_format() == 'channels_first', \
            "Pytorch to Keras parser needs image_data_format == channel_first."

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': True,
                      'parse': False,
                      'normalize': False,
                      'convert': False,
                      'simulate': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {'num_to_test': 100,
                           'batch_size': 50}}

        _config.read_dict(updates)

        initialize_simulator(_config)

        normset, testset = get_dataset(_config)

        model_lib = import_module('snntoolbox.parsing.model_libs.' +
                                  _config.get('input', 'model_lib') +
                                  '_input_lib')
        input_model = model_lib.load(_config.get('paths', 'path_wd'),
                                     _config.get('paths', 'filename_ann'))

        # Evaluate input model.
        acc = model_lib.evaluate(input_model['val_fn'],
                                 _config.getint('simulation', 'batch_size'),
                                 _config.getint('simulation', 'num_to_test'),
                                 **testset)

        assert acc >= 0.8

    def test_parsing(self, _model_4, _config):

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': True,
                      'parse': True,
                      'normalize': False,
                      'convert': False,
                      'simulate': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {'num_to_test': 100,
                           'batch_size': 50}
        }

        _config.read_dict(updates)

        initialize_simulator(_config)

        acc = run_pipeline(_config)

        assert acc[0] >= 0.8

    def test_normalizing(self, _model_4, _config):

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': True,
                      'parse': True,
                      'normalize': True,
                      'convert': False,
                      'simulate': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {'num_to_test': 100,
                           'batch_size': 50}
        }

        _config.read_dict(updates)

        initialize_simulator(_config)

        acc = run_pipeline(_config)

        assert acc[0] >= 0.8

    def test_pipeline(self, _model_4, _config):

        self.prepare_model(_model_4, _config)

        updates = {
            'tools': {'evaluate_ann': False},
            'input': {'model_lib': 'pytorch'},
            'simulation': {
                'duration': 100,
                'num_to_test': 100,
                'batch_size': 50},
            'output': {
                'log_vars': {'activations_n_b_l', 'spiketrains_n_b_l_t'}}}

        _config.read_dict(updates)

        initialize_simulator(_config)

        acc = run_pipeline(_config)

        assert acc[0] >= 0.8

        corr = get_correlations(_config)
        assert np.all(corr[:-1] > 0.97)
        assert corr[-1] > 0.5
<|endoftext|>


<commit_before># /usr/bin/env python
"""
Trim sequences at k-mers of the given abundance, based on the given counting
hash table.  Output sequences will be placed in 'infile.abundfilt'.

% python scripts/filter-abund.py <counting.kh> <data1> [ <data2> <...> ]

Use '-h' for parameter help.
"""
import sys, screed.fasta, os
import khmer
from khmer.thread_utils import ThreadedSequenceProcessor, verbose_loader

from khmer.counting_args import build_counting_multifile_args

###

DEFAULT_CUTOFF=2

def main():
    parser = build_counting_multifile_args()
    parser.add_argument('--cutoff', '-C', dest='cutoff',
                        default=DEFAULT_CUTOFF, type=int,
                        help="Trim at k-mers below this abundance.")
    args = parser.parse_args()

    counting_ht = args.input_table
    infiles = args.input_filenames

    print 'file with ht: %s' % counting_ht

    print 'loading hashtable'
    ht = khmer.load_counting_hash(counting_ht)
    K = ht.ksize()

    print "K:", K

    ### the filtering function.
    def process_fn(record):
        name = record['name']
        seq = record['sequence']
        if 'N' in seq:
            return None, None

        trim_seq, trim_at = ht.trim_on_abundance(seq, args.cutoff)

        if trim_at >= K:
            return name, trim_seq

        return None, None

    ### the filtering loop
    for infile in infiles:
       print 'filtering', infile
       outfile = os.path.basename(infile) + '.abundfilt'
       outfp = open(outfile, 'w')

       tsp = ThreadedSequenceProcessor(process_fn)
       tsp.start(verbose_loader(infile), outfp)

       print 'output in', outfile

if __name__ == '__main__':
    main()
<commit_msg>fix shebang<commit_after>#! /usr/bin/env python
"""
Trim sequences at k-mers of the given abundance, based on the given counting
hash table.  Output sequences will be placed in 'infile.abundfilt'.

% python scripts/filter-abund.py <counting.kh> <data1> [ <data2> <...> ]

Use '-h' for parameter help.
"""
import sys, screed.fasta, os
import khmer
from khmer.thread_utils import ThreadedSequenceProcessor, verbose_loader

from khmer.counting_args import build_counting_multifile_args

###

DEFAULT_CUTOFF=2

def main():
    parser = build_counting_multifile_args()
    parser.add_argument('--cutoff', '-C', dest='cutoff',
                        default=DEFAULT_CUTOFF, type=int,
                        help="Trim at k-mers below this abundance.")
    args = parser.parse_args()

    counting_ht = args.input_table
    infiles = args.input_filenames

    print 'file with ht: %s' % counting_ht

    print 'loading hashtable'
    ht = khmer.load_counting_hash(counting_ht)
    K = ht.ksize()

    print "K:", K

    ### the filtering function.
    def process_fn(record):
        name = record['name']
        seq = record['sequence']
        if 'N' in seq:
            return None, None

        trim_seq, trim_at = ht.trim_on_abundance(seq, args.cutoff)

        if trim_at >= K:
            return name, trim_seq

        return None, None

    ### the filtering loop
    for infile in infiles:
       print 'filtering', infile
       outfile = os.path.basename(infile) + '.abundfilt'
       outfp = open(outfile, 'w')

       tsp = ThreadedSequenceProcessor(process_fn)
       tsp.start(verbose_loader(infile), outfp)

       print 'output in', outfile

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from sockjs.tornado import conn, session
from sockjs.tornado.transports import base

class ChannelSession(session.BaseSession):
	def __init__(self, conn, server, base, name):
		super(ChannelSession, self).__init__(conn, server)

		self.base = base
		self.name = name

	def send_message(self, msg, binary=False):
		self.base.send('msg,' + self.name + ',' + msg)

	def on_message(self, msg):
		self.conn.on_message(msg)

	def close(self, code=3000, message='Go away!'):
		self.base.send('uns,' + self.name)
		self._close(code, message)

	# Non-API version of the close, without sending the close message
	def _close(self, code=3000, message='Go away!'):
		super(ChannelSession, self).close(code, message)

class DummyHandler(base.BaseTransportMixin):
	name = 'multiplex'

	def __init__(self, conn_info):
		self.conn_info = conn_info

	def get_conn_info(self):
		return self.conn_info

class MultiplexConnection(conn.SockJSConnection):
	channels = dict()

	def on_open(self, info):
		self.endpoints = dict()
		self.handler = DummyHandler(self.session.conn_info)

	def on_message(self, msg):
		parts = msg.split(',', 2)
		op, chan = parts[0], parts[1]

		if chan not in self.channels:
			return

		if chan in self.endpoints:
			session = self.endpoints[chan]

			if op == 'uns':
				del self.endpoints[chan]
				session._close()
			elif op == 'msg':
				session.on_message(parts[2])
		else:
			if op == 'sub':
				session = ChannelSession(self.channels[chan],
										 self.session.server,
										 self,
										 chan)
				session.set_handler(self.handler)
				session.verify_state()

				self.endpoints[chan] = session

	def on_close(self):
		for chan in self.endpoints:
			self.endpoints[chan]._close()

	@classmethod
	def get(cls, **kwargs):
		return type('MultiplexRouter', (MultiplexConnection,), dict(channels=kwargs))
<commit_msg>another sockjs fix<commit_after>from sockjs.tornado import conn, session
from sockjs.tornado.transports import base

class ChannelSession(session.BaseSession):
	def __init__(self, conn, server, base, name):
		super(ChannelSession, self).__init__(conn, server)

		self.base = base
		self.name = name

	def send_message(self, msg, stats=True, binary=False):
		self.base.send('msg,' + self.name + ',' + msg)

	def on_message(self, msg):
		self.conn.on_message(msg)

	def close(self, code=3000, message='Go away!'):
		self.base.send('uns,' + self.name)
		self._close(code, message)

	# Non-API version of the close, without sending the close message
	def _close(self, code=3000, message='Go away!'):
		super(ChannelSession, self).close(code, message)

class DummyHandler(base.BaseTransportMixin):
	name = 'multiplex'

	def __init__(self, conn_info):
		self.conn_info = conn_info

	def get_conn_info(self):
		return self.conn_info

class MultiplexConnection(conn.SockJSConnection):
	channels = dict()

	def on_open(self, info):
		self.endpoints = dict()
		self.handler = DummyHandler(self.session.conn_info)

	def on_message(self, msg):
		parts = msg.split(',', 2)
		op, chan = parts[0], parts[1]

		if chan not in self.channels:
			return

		if chan in self.endpoints:
			session = self.endpoints[chan]

			if op == 'uns':
				del self.endpoints[chan]
				session._close()
			elif op == 'msg':
				session.on_message(parts[2])
		else:
			if op == 'sub':
				session = ChannelSession(self.channels[chan],
										 self.session.server,
										 self,
										 chan)
				session.set_handler(self.handler)
				session.verify_state()

				self.endpoints[chan] = session

	def on_close(self):
		for chan in self.endpoints:
			self.endpoints[chan]._close()

	@classmethod
	def get(cls, **kwargs):
		return type('MultiplexRouter', (MultiplexConnection,), dict(channels=kwargs))
<|endoftext|>


<commit_before>import csv
import random
import sys
import tensorflow as tf
import numpy as np

# Read in CSV file
brca = []
labelInput = []
count = 0

print("Starting random number generation...")
for i in range(1000):
    brcaInternal = []
    for j in range(100):
        brcaInternal.append(random.random())
    # print(brcaInternal)
    brca.append(brcaInternal)
    # print(brca)
    print("Generation ", i, " completed...")
print("Finished random number generation...")

# Get the length of brca
lenBRCA = len(brca[0])
halfBRCA = lenBRCA / 2

# Initialize weight and adjustment layer 1 vectors
x11 = tf.placeholder(tf.float32, [None,lenBRCA])
b11 = tf.Variable(tf.zeros([halfBRCA]))
b12 = tf.Variable(tf.zeros([lenBRCA]))
W11 = tf.get_variable('W11', shape=[lenBRCA, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W12 = tf.get_variable('W12', shape=[halfBRCA, lenBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))

z11 = tf.matmul(x11, W11) + b11
y11 = tf.nn.relu(z11)
z12 = tf.matmul(y11, W12) + b12
y12 = tf.nn.relu(z12)

# Print array dimensions
print("\nThe shape of x11 is: ", x11.get_shape())
print("The shape of b11 is: ", b11.get_shape())
print("The shape of b12 is: ", b12.get_shape())
print("The shape of W11 is: ", W11.get_shape())
print("The shape of W12 is: ", W12.get_shape())
print("The shape of y11 is: ", y11.get_shape())

# Initialize weight and adjustment layer 2 vector
W21 = tf.get_variable('W21', shape=[halfBRCA, 100], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W22 = tf.get_variable('W22', shape=[100, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
b21 = tf.Variable(tf.zeros([100]))
b22 = tf.Variable(tf.zeros([halfBRCA]))

z21 = tf.matmul(y11, W21) + b21
y21 = tf.nn.relu(z21)
z22 = tf.matmul(y21, W22) + b22
y22 = tf.nn.relu(z22)

# Print array dimensions
print("\nThe shape of b21 is: ", b21.get_shape())
print("The shape of b22 is: ", b22.get_shape())
print("The shape of W21 is: ", W21.get_shape())
print("The shape of W22 is: ", W22.get_shape())
print("The shape of y21 is: ", y21.get_shape())

#Output Layer
Wo = tf.get_variable('Wo', shape=[100, 2], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
bo = tf.Variable(tf.zeros([2]))
zo = tf.matmul(y21, Wo) + bo
yo = tf.nn.softmax(zo)
labelTensor = tf.placeholder(tf.float32, [None, 2]);

print("\nThe shape of bo is: ", bo.get_shape())
print("The shape of Wo is: ", Wo.get_shape())
print("The shape of yo is: ", yo.get_shape())

# Calculate square difference
square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
# square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
square_difference2 = tf.reduce_sum(tf.square(y11 - y22))
# square_difference2 = tf.reduce_sum(tf.square(y11 - y22))

# Output layer loss
loss3 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labelTensor, logits=zo))

# Optimization
train_step1 = tf.train.AdamOptimizer().minimize(square_difference1)
train_step2 = tf.train.AdamOptimizer().minimize(square_difference2)
train_step3 = tf.train.AdamOptimizer().minimize(loss3)

# Start tensorflow session
sess = tf.InteractiveSession()
# tf.global_variables_initializer().run()
init_op = tf.initialize_all_variables()
sess.run(init_op)

# Train autoencoder layer 1
print("Starting layer 1 training...")
for j in range(1):
    for i in range(len(brca)):
        # Convert brca array into numpy array for tensorflow
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        print("Trained sample ", i, "...")
sess.run(train_step1, feed_dict={x11: inputArray})
print("Ending layer 1 training...")

# print("\nW11\n", sess.run(W11))
# print("\nb11\n", sess.run(b11))
# print("\nW12\n", sess.run(W12))
# print("\nb12\n", sess.run(b12))

# Train autoencoder layer 2
print("Starting layer 2 training...")
for j in range(1):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        sess.run(train_step2, feed_dict={x11: inputArray})
        print("Trained sample ", i, "...")
print("Ending layer 2 training...")

# print("\nW21\n", sess.run(W21))
# print("\nb21\n", sess.run(b21))
# print("\nW12\n", sess.run(W22))
# print("\nb12\n", sess.run(b22))

# Train autoencoder output layer
print("Starting output layer training...")
for j in range(1):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
        # print(labelInput[i])
        if brca[i][0] > 0.5:
            labelInput = [1.0, 0.0]
        else:
            labelInput = [0.0, 1.0]
        labelArray = np.array(labelInput, dtype = float).reshape(1, len(labelInput))
        sess.run(train_step3, feed_dict={x11: inputArray, labelTensor: labelArray})
        print("Trained sample ", i, "...")
print("Ending output layer training...")

# print("\nWo\n", sess.run(Wo))
# print("\nbo\n", sess.run(bo))

# Print output of each layer
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    # print("\n")
    # print("y11 tensor, sample ", i, ": \n", sess.run(y11, feed_dict={x11: inputArray}))
    # print("y21 tensor, sample ", i, ": \n", sess.run(y21, feed_dict={x11: inputArray}))
    # print("yo tensor, sample ", i, ": \n", sess.run(yo, feed_dict={x11: inputArray}))

print("Collecting results...")
outputList = []
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    outputList.append(sess.run(yo, feed_dict={x11: inputArray}))
    print("Collected ", i, "...")

num1 = 0
num2 = 0
iterations = 0
for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1
    iterations += 1

print("Number of 1: ", num1)
print("Number of 2: ", num2)

# Calculate difference between input and ouput
# accuracy1 = tf.reduce_sum(tf.square(x11 - y12))
accuracy1 = tf.reduce_sum(tf.square(x11 - z12))
# accuracy2 = tf.reduce_sum(tf.square(y11 - y22))
accuracy2 = tf.reduce_sum(tf.square(y11 - z22))

# Print difference
inputArray = np.array(brca[0], dtype=float).reshape(1, lenBRCA)
print("Squared difference for layer 1: ", sess.run(accuracy1, feed_dict={x11: inputArray}))
print("Squared difference for layer 2: ", sess.run(accuracy2, feed_dict={x11: inputArray}))

<commit_msg>Scaling test 10<commit_after>import csv
import random
import sys
import tensorflow as tf
import numpy as np

# Read in CSV file
brca = []
labelInput = []
count = 0

print("Starting random number generation...")
for i in range(1000):
    brcaInternal = []
    for j in range(10):
        brcaInternal.append(random.random())
    # print(brcaInternal)
    brca.append(brcaInternal)
    # print(brca)
    print("Generation ", i, " completed...")
print("Finished random number generation...")

# Get the length of brca
lenBRCA = len(brca[0])
halfBRCA = lenBRCA / 2

# Initialize weight and adjustment layer 1 vectors
x11 = tf.placeholder(tf.float32, [None,lenBRCA])
b11 = tf.Variable(tf.zeros([halfBRCA]))
b12 = tf.Variable(tf.zeros([lenBRCA]))
W11 = tf.get_variable('W11', shape=[lenBRCA, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W12 = tf.get_variable('W12', shape=[halfBRCA, lenBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))

z11 = tf.matmul(x11, W11) + b11
y11 = tf.nn.relu(z11)
z12 = tf.matmul(y11, W12) + b12
y12 = tf.nn.relu(z12)

# Print array dimensions
print("\nThe shape of x11 is: ", x11.get_shape())
print("The shape of b11 is: ", b11.get_shape())
print("The shape of b12 is: ", b12.get_shape())
print("The shape of W11 is: ", W11.get_shape())
print("The shape of W12 is: ", W12.get_shape())
print("The shape of y11 is: ", y11.get_shape())

# Initialize weight and adjustment layer 2 vector
W21 = tf.get_variable('W21', shape=[halfBRCA, 100], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W22 = tf.get_variable('W22', shape=[100, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
b21 = tf.Variable(tf.zeros([100]))
b22 = tf.Variable(tf.zeros([halfBRCA]))

z21 = tf.matmul(y11, W21) + b21
y21 = tf.nn.relu(z21)
z22 = tf.matmul(y21, W22) + b22
y22 = tf.nn.relu(z22)

# Print array dimensions
print("\nThe shape of b21 is: ", b21.get_shape())
print("The shape of b22 is: ", b22.get_shape())
print("The shape of W21 is: ", W21.get_shape())
print("The shape of W22 is: ", W22.get_shape())
print("The shape of y21 is: ", y21.get_shape())

#Output Layer
Wo = tf.get_variable('Wo', shape=[100, 2], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
bo = tf.Variable(tf.zeros([2]))
zo = tf.matmul(y21, Wo) + bo
yo = tf.nn.softmax(zo)
labelTensor = tf.placeholder(tf.float32, [None, 2]);

print("\nThe shape of bo is: ", bo.get_shape())
print("The shape of Wo is: ", Wo.get_shape())
print("The shape of yo is: ", yo.get_shape())

# Calculate square difference
square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
# square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
square_difference2 = tf.reduce_sum(tf.square(y11 - y22))
# square_difference2 = tf.reduce_sum(tf.square(y11 - y22))

# Output layer loss
loss3 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labelTensor, logits=zo))

# Optimization
train_step1 = tf.train.AdamOptimizer().minimize(square_difference1)
train_step2 = tf.train.AdamOptimizer().minimize(square_difference2)
train_step3 = tf.train.AdamOptimizer().minimize(loss3)

# Start tensorflow session
sess = tf.InteractiveSession()
# tf.global_variables_initializer().run()
init_op = tf.initialize_all_variables()
sess.run(init_op)

# Train autoencoder layer 1
print("Starting layer 1 training...")
for j in range(1):
    for i in range(len(brca)):
        # Convert brca array into numpy array for tensorflow
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        print("Trained sample ", i, "...")
sess.run(train_step1, feed_dict={x11: inputArray})
print("Ending layer 1 training...")

# print("\nW11\n", sess.run(W11))
# print("\nb11\n", sess.run(b11))
# print("\nW12\n", sess.run(W12))
# print("\nb12\n", sess.run(b12))

# Train autoencoder layer 2
print("Starting layer 2 training...")
for j in range(1):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        sess.run(train_step2, feed_dict={x11: inputArray})
        print("Trained sample ", i, "...")
print("Ending layer 2 training...")

# print("\nW21\n", sess.run(W21))
# print("\nb21\n", sess.run(b21))
# print("\nW12\n", sess.run(W22))
# print("\nb12\n", sess.run(b22))

# Train autoencoder output layer
print("Starting output layer training...")
for j in range(1):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
        # print(labelInput[i])
        if brca[i][0] > 0.5:
            labelInput = [1.0, 0.0]
        else:
            labelInput = [0.0, 1.0]
        labelArray = np.array(labelInput, dtype = float).reshape(1, len(labelInput))
        sess.run(train_step3, feed_dict={x11: inputArray, labelTensor: labelArray})
        print("Trained sample ", i, "...")
print("Ending output layer training...")

# print("\nWo\n", sess.run(Wo))
# print("\nbo\n", sess.run(bo))

# Print output of each layer
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    # print("\n")
    # print("y11 tensor, sample ", i, ": \n", sess.run(y11, feed_dict={x11: inputArray}))
    # print("y21 tensor, sample ", i, ": \n", sess.run(y21, feed_dict={x11: inputArray}))
    # print("yo tensor, sample ", i, ": \n", sess.run(yo, feed_dict={x11: inputArray}))

print("Collecting results...")
outputList = []
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    outputList.append(sess.run(yo, feed_dict={x11: inputArray}))
    print("Collected ", i, "...")

num1 = 0
num2 = 0
iterations = 0
for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1
    iterations += 1

print("Number of 1: ", num1)
print("Number of 2: ", num2)

# Calculate difference between input and ouput
# accuracy1 = tf.reduce_sum(tf.square(x11 - y12))
accuracy1 = tf.reduce_sum(tf.square(x11 - z12))
# accuracy2 = tf.reduce_sum(tf.square(y11 - y22))
accuracy2 = tf.reduce_sum(tf.square(y11 - z22))

# Print difference
inputArray = np.array(brca[0], dtype=float).reshape(1, lenBRCA)
print("Squared difference for layer 1: ", sess.run(accuracy1, feed_dict={x11: inputArray}))
print("Squared difference for layer 2: ", sess.run(accuracy2, feed_dict={x11: inputArray}))

<|endoftext|>


<commit_before>import pytest
import os
from SCNIC.general import simulate_correls
from SCNIC.between_correls import between_correls

@pytest.fixture()
def args():
    class Arguments(object):
        def __init__(self):
            self.table1 = "table1.biom"
            self.table2 = "table2.biom"
            self.output = "out_dir"
            self.correl_method = "spearman"
            self.p_adjust = "bh"
            self.min_sample = None
            self.min_p = None
            self.min_r = None
            self.sparcc_filter = False
            self.force = False
            self.procs = 1

    return Arguments()

def test_between_correls(args, tmpdir):
    table1 = simulate_correls()
    table2 = simulate_correls()
    loc = tmpdir.mkdir("with_correls_test")
    table1.to_json("madebyme", open(str(loc)+"/table1.biom", 'w'))
    table2.to_json("madebyme", open(str(loc) + "/table2.biom", 'w'))
    os.chdir(str(loc))
    between_correls(args)
    files = os.listdir(str(loc)+'/out_dir')
    assert "correls.txt" in files
    assert "crossnet.gml" in files
<commit_msg>Add module.py test and modify some others for better coverage. Remove files not used.<commit_after>import pytest
import os
from SCNIC.general import simulate_correls
from SCNIC.between_correls import between_correls

@pytest.fixture()
def args():
    class Arguments(object):
        def __init__(self):
            self.table1 = "table1.biom"
            self.table2 = "table2.biom"
            self.output = "out_dir"
            self.correl_method = "spearman"
            self.p_adjust = "bh"
            self.min_sample = None
            self.min_p = None
            self.min_r = None
            self.sparcc_filter = True
            self.force = False
            self.procs = 1

    return Arguments()

def test_between_correls(args, tmpdir):
    table1 = simulate_correls()
    table2 = simulate_correls()
    loc = tmpdir.mkdir("with_correls_test")
    table1.to_json("madebyme", open(str(loc)+"/table1.biom", 'w'))
    table2.to_json("madebyme", open(str(loc) + "/table2.biom", 'w'))
    os.chdir(str(loc))
    between_correls(args)
    files = os.listdir(str(loc)+'/out_dir')
    assert "correls.txt" in files
    assert "crossnet.gml" in files
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

   Copyright 2018 Open Energy Efficiency, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

"""
import json
import numpy as np
import pandas as pd
import pytest

from eemeter.caltrack.hourly import (
    caltrack_hourly_fit_feature_processor,
    caltrack_hourly_prediction_feature_processor,
    fit_caltrack_hourly_model_segment,
    fit_caltrack_hourly_model,
)
from eemeter.features import (
    compute_time_features,
    compute_temperature_features,
    compute_usage_per_day_feature,
    merge_features,
)

@pytest.fixture
def segmented_data():
    index = pd.date_range(start="2017-01-01", periods=24, freq="H", tz="UTC")
    time_features = compute_time_features(index)
    segmented_data = pd.DataFrame(
        {
            "hour_of_week": time_features.hour_of_week,
            "temperature_mean": np.linspace(0, 100, 24),
            "meter_value": np.linspace(10, 70, 24),
            "weight": np.ones((24,)),
        },
        index=index,
    )
    return segmented_data

@pytest.fixture
def occupancy_lookup():
    index = pd.Categorical(range(168))
    occupancy = pd.Series([i % 2 == 0 for i in range(168)], index=index)
    return pd.DataFrame(
        {"dec-jan-feb-weighted": occupancy, "jan-feb-mar-weighted": occupancy}
    )

@pytest.fixture
def temperature_bins():
    bins = pd.Series([True, True, True], index=[30, 60, 90])
    return pd.DataFrame({"dec-jan-feb-weighted": bins, "jan-feb-mar-weighted": bins})

def test_caltrack_hourly_fit_feature_processor(
    segmented_data, occupancy_lookup, temperature_bins
):
    result = caltrack_hourly_fit_feature_processor(
        "dec-jan-feb-weighted", segmented_data, occupancy_lookup, temperature_bins
    )
    assert list(result.columns) == [
        "meter_value",
        "hour_of_week",
        "occupancy",
        "bin_0",
        "bin_1",
        "bin_2",
        "bin_3",
        "weight",
    ]
    assert result.shape == (24, 8)
    assert round(result.sum().sum(), 2) == 5928.0

def test_caltrack_hourly_prediction_feature_processor(
    segmented_data, occupancy_lookup, temperature_bins
):
    result = caltrack_hourly_prediction_feature_processor(
        "dec-jan-feb-weighted", segmented_data, occupancy_lookup, temperature_bins
    )
    assert list(result.columns) == [
        "hour_of_week",
        "occupancy",
        "bin_0",
        "bin_1",
        "bin_2",
        "bin_3",
        "weight",
    ]
    assert result.shape == (24, 7)
    assert round(result.sum().sum(), 2) == 4968.0

@pytest.fixture
def segmented_design_matrices(segmented_data, occupancy_lookup, temperature_bins):
    return {
        "dec-jan-feb-weighted": caltrack_hourly_fit_feature_processor(
            "dec-jan-feb-weighted", segmented_data, occupancy_lookup, temperature_bins
        )
    }

def test_fit_caltrack_hourly_model_segment(segmented_design_matrices):
    segment_name = "dec-jan-feb-weighted"
    segment_data = segmented_design_matrices[segment_name]
    segment_model = fit_caltrack_hourly_model_segment(segment_name, segment_data)
    assert segment_model.formula == (
        "meter_value ~ C(hour_of_week) - 1 + bin_0:C(occupancy)"
        " + bin_1:C(occupancy) + bin_2:C(occupancy) + bin_3:C(occupancy)"
    )
    assert segment_model.segment_name == "dec-jan-feb-weighted"
    assert len(segment_model.model_params.keys()) == 32
    assert segment_model.model is not None
    assert segment_model.warnings is not None
    prediction = segment_model.predict(segment_data)
    assert round(prediction.sum(), 2) == 960.0

@pytest.fixture
def temps():
    index = pd.date_range(start="2017-01-01", periods=24, freq="H", tz="UTC")
    temps = pd.Series(np.linspace(0, 100, 24), index=index)
    return temps

def test_fit_caltrack_hourly_model(
    segmented_design_matrices, occupancy_lookup, temperature_bins, temps
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices, occupancy_lookup, temperature_bins
    )

    assert segmented_model.segment_models is not None
    prediction = segmented_model.predict(temps.index, temps).result

def test_serialize_caltrack_hourly_model(
    segmented_design_matrices, occupancy_lookup, temperature_bins, temps
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices, occupancy_lookup, temperature_bins
    )
    assert json.dumps(segmented_model.json())

@pytest.fixture
def segmented_data_nans():
    num_periods = 200
    index = pd.date_range(start="2017-01-01", periods=num_periods, freq="H", tz="UTC")
    time_features = compute_time_features(index)
    segmented_data = pd.DataFrame(
        {
            "hour_of_week": time_features.hour_of_week,
            "temperature_mean": np.linspace(0, 100, num_periods),
            "meter_value": np.linspace(10, 70, num_periods),
            "weight": np.ones((num_periods,)),
        },
        index=index,
    )
    return segmented_data

@pytest.fixture
def occupancy_lookup_nans():
    index = pd.Categorical(range(168))
    occupancy = pd.Series([i % 2 == 0 for i in range(168)], index=index)
    occupancy_nans = pd.Series([np.nan for i in range(168)], index=index)
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": occupancy,
            "jan-feb-mar-weighted": occupancy,
            "apr-may-jun-weighted": occupancy_nans,
        }
    )

@pytest.fixture
def temperature_bins_nans():
    bins = pd.Series([True, True, True], index=[30, 60, 90])
    bins_nans = pd.Series([False, False, False], index=[30, 60, 90])
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": bins,
            "jan-feb-mar-weighted": bins,
            "apr-may-jun-weighted": bins_nans,
        }
    )

@pytest.fixture
def segmented_design_matrices_nans(
    segmented_data_nans, occupancy_lookup_nans, temperature_bins_nans
):
    return {
        "dec-jan-feb-weighted": caltrack_hourly_fit_feature_processor(
            "dec-jan-feb-weighted",
            segmented_data_nans,
            occupancy_lookup_nans,
            temperature_bins_nans,
        ),
        "apr-may-jun-weighted": caltrack_hourly_fit_feature_processor(
            "apr-may-jun-weighted",
            segmented_data_nans,
            occupancy_lookup_nans,
            temperature_bins_nans,
        ),
    }

def test_fit_caltrack_hourly_model_nans_less_than_week_predict(
    segmented_design_matrices_nans,
    occupancy_lookup_nans,
    temperature_bins_nans,
    temps_extended,
    temps,
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices_nans, occupancy_lookup_nans, temperature_bins_nans
    )

    assert segmented_model.segment_models is not None
    assert segmented_model.segment_models[0].model is not None
    assert segmented_model.segment_models[1].model is None
    assert (
        segmented_model.segment_models[1].warnings[0].qualified_name
        == "eemeter.fit_caltrack_hourly_model_segment.no_nonnull_data"
    )
    prediction = segmented_model.predict(temps.index, temps).result
    assert prediction.shape[0] == 24
    assert prediction["predicted_usage"].sum().round() == 955.0

@pytest.fixture
def segmented_data_nans_less_than_week():
    num_periods = 4
    index = pd.date_range(start="2017-01-01", periods=num_periods, freq="H", tz="UTC")
    time_features = compute_time_features(index)
    segmented_data = pd.DataFrame(
        {
            "hour_of_week": time_features.hour_of_week,
            "temperature_mean": np.linspace(0, 100, num_periods),
            "meter_value": np.linspace(10, 70, num_periods),
            "weight": np.ones((num_periods,)),
        },
        index=index,
    )
    return segmented_data

@pytest.fixture
def occupancy_lookup_nans_less_than_week():
    index = pd.Categorical(range(168))
    occupancy = pd.Series([i % 2 == 0 for i in range(168)], index=index)
    occupancy_nans_less_than_week = pd.Series([np.nan for i in range(168)], index=index)
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": occupancy,
            "jan-feb-mar-weighted": occupancy,
            "apr-may-jun-weighted": occupancy_nans_less_than_week,
        }
    )

@pytest.fixture
def temperature_bins_nans_less_than_week():
    bins = pd.Series([True, True, True], index=[30, 60, 90])
    bins_nans_less_than_week = pd.Series([False, False, False], index=[30, 60, 90])
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": bins,
            "jan-feb-mar-weighted": bins,
            "apr-may-jun-weighted": bins_nans_less_than_week,
        }
    )

@pytest.fixture
def segmented_design_matrices_nans_less_than_week(
    segmented_data_nans_less_than_week,
    occupancy_lookup_nans_less_than_week,
    temperature_bins_nans_less_than_week,
):
    return {
        "dec-jan-feb-weighted": caltrack_hourly_fit_feature_processor(
            "dec-jan-feb-weighted",
            segmented_data_nans_less_than_week,
            occupancy_lookup_nans_less_than_week,
            temperature_bins_nans_less_than_week,
        ),
        "apr-may-jun-weighted": caltrack_hourly_fit_feature_processor(
            "apr-may-jun-weighted",
            segmented_data_nans_less_than_week,
            occupancy_lookup_nans_less_than_week,
            temperature_bins_nans_less_than_week,
        ),
    }

@pytest.fixture
def temps_extended():
    index = pd.date_range(start="2017-01-01", periods=168, freq="H", tz="UTC")
    temps = pd.Series(1, index=index)
    return temps

def test_fit_caltrack_hourly_model_nans_less_than_week_fit(
    segmented_design_matrices_nans_less_than_week,
    occupancy_lookup_nans_less_than_week,
    temperature_bins_nans_less_than_week,
    temps_extended,
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices_nans_less_than_week,
        occupancy_lookup_nans_less_than_week,
        temperature_bins_nans_less_than_week,
    )

    assert segmented_model.segment_models is not None
    assert segmented_model.segment_models[0].model is not None
    assert segmented_model.segment_models[1].model is None
    assert (
        segmented_model.segment_models[1].warnings[0].qualified_name
        == "eemeter.fit_caltrack_hourly_model_segment.no_nonnull_data"
    )
    prediction = segmented_model.predict(temps_extended.index, temps_extended).result
    assert prediction.shape[0] == 168
    assert prediction.dropna().shape[0] == 4
<commit_msg>Update test to work in py27<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

   Copyright 2018 Open Energy Efficiency, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

"""
import json
import numpy as np
import pandas as pd
import pytest

from eemeter.caltrack.hourly import (
    caltrack_hourly_fit_feature_processor,
    caltrack_hourly_prediction_feature_processor,
    fit_caltrack_hourly_model_segment,
    fit_caltrack_hourly_model,
)
from eemeter.features import (
    compute_time_features,
    compute_temperature_features,
    compute_usage_per_day_feature,
    merge_features,
)

@pytest.fixture
def segmented_data():
    index = pd.date_range(start="2017-01-01", periods=24, freq="H", tz="UTC")
    time_features = compute_time_features(index)
    segmented_data = pd.DataFrame(
        {
            "hour_of_week": time_features.hour_of_week,
            "temperature_mean": np.linspace(0, 100, 24),
            "meter_value": np.linspace(10, 70, 24),
            "weight": np.ones((24,)),
        },
        index=index,
    )
    return segmented_data

@pytest.fixture
def occupancy_lookup():
    index = pd.Categorical(range(168))
    occupancy = pd.Series([i % 2 == 0 for i in range(168)], index=index)
    return pd.DataFrame(
        {"dec-jan-feb-weighted": occupancy, "jan-feb-mar-weighted": occupancy}
    )

@pytest.fixture
def temperature_bins():
    bins = pd.Series([True, True, True], index=[30, 60, 90])
    return pd.DataFrame({"dec-jan-feb-weighted": bins, "jan-feb-mar-weighted": bins})

def test_caltrack_hourly_fit_feature_processor(
    segmented_data, occupancy_lookup, temperature_bins
):
    result = caltrack_hourly_fit_feature_processor(
        "dec-jan-feb-weighted", segmented_data, occupancy_lookup, temperature_bins
    )
    assert list(result.columns) == [
        "meter_value",
        "hour_of_week",
        "occupancy",
        "bin_0",
        "bin_1",
        "bin_2",
        "bin_3",
        "weight",
    ]
    assert result.shape == (24, 8)
    assert round(result.sum().sum(), 2) == 5928.0

def test_caltrack_hourly_prediction_feature_processor(
    segmented_data, occupancy_lookup, temperature_bins
):
    result = caltrack_hourly_prediction_feature_processor(
        "dec-jan-feb-weighted", segmented_data, occupancy_lookup, temperature_bins
    )
    assert list(result.columns) == [
        "hour_of_week",
        "occupancy",
        "bin_0",
        "bin_1",
        "bin_2",
        "bin_3",
        "weight",
    ]
    assert result.shape == (24, 7)
    assert round(result.sum().sum(), 2) == 4968.0

@pytest.fixture
def segmented_design_matrices(segmented_data, occupancy_lookup, temperature_bins):
    return {
        "dec-jan-feb-weighted": caltrack_hourly_fit_feature_processor(
            "dec-jan-feb-weighted", segmented_data, occupancy_lookup, temperature_bins
        )
    }

def test_fit_caltrack_hourly_model_segment(segmented_design_matrices):
    segment_name = "dec-jan-feb-weighted"
    segment_data = segmented_design_matrices[segment_name]
    segment_model = fit_caltrack_hourly_model_segment(segment_name, segment_data)
    assert segment_model.formula == (
        "meter_value ~ C(hour_of_week) - 1 + bin_0:C(occupancy)"
        " + bin_1:C(occupancy) + bin_2:C(occupancy) + bin_3:C(occupancy)"
    )
    assert segment_model.segment_name == "dec-jan-feb-weighted"
    assert len(segment_model.model_params.keys()) == 32
    assert segment_model.model is not None
    assert segment_model.warnings is not None
    prediction = segment_model.predict(segment_data)
    assert round(prediction.sum(), 2) == 960.0

@pytest.fixture
def temps():
    index = pd.date_range(start="2017-01-01", periods=24, freq="H", tz="UTC")
    temps = pd.Series(np.linspace(0, 100, 24), index=index)
    return temps

def test_fit_caltrack_hourly_model(
    segmented_design_matrices, occupancy_lookup, temperature_bins, temps
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices, occupancy_lookup, temperature_bins
    )

    assert segmented_model.segment_models is not None
    prediction = segmented_model.predict(temps.index, temps).result

def test_serialize_caltrack_hourly_model(
    segmented_design_matrices, occupancy_lookup, temperature_bins, temps
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices, occupancy_lookup, temperature_bins
    )
    assert json.dumps(segmented_model.json())

@pytest.fixture
def segmented_data_nans():
    num_periods = 200
    index = pd.date_range(start="2017-01-01", periods=num_periods, freq="H", tz="UTC")
    time_features = compute_time_features(index)
    segmented_data = pd.DataFrame(
        {
            "hour_of_week": time_features.hour_of_week,
            "temperature_mean": np.linspace(0, 100, num_periods),
            "meter_value": np.linspace(10, 70, num_periods),
            "weight": np.ones((num_periods,)),
        },
        index=index,
    )
    return segmented_data

@pytest.fixture
def occupancy_lookup_nans():
    index = pd.Categorical(range(168))
    occupancy = pd.Series([i % 2 == 0 for i in range(168)], index=index)
    occupancy_nans = pd.Series([np.nan for i in range(168)], index=index)
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": occupancy,
            "jan-feb-mar-weighted": occupancy,
            "apr-may-jun-weighted": occupancy_nans,
        }
    )

@pytest.fixture
def temperature_bins_nans():
    bins = pd.Series([True, True, True], index=[30, 60, 90])
    bins_nans = pd.Series([False, False, False], index=[30, 60, 90])
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": bins,
            "jan-feb-mar-weighted": bins,
            "apr-may-jun-weighted": bins_nans,
        }
    )

@pytest.fixture
def segmented_design_matrices_nans(
    segmented_data_nans, occupancy_lookup_nans, temperature_bins_nans
):
    return {
        "dec-jan-feb-weighted": caltrack_hourly_fit_feature_processor(
            "dec-jan-feb-weighted",
            segmented_data_nans,
            occupancy_lookup_nans,
            temperature_bins_nans,
        ),
        "apr-may-jun-weighted": caltrack_hourly_fit_feature_processor(
            "apr-may-jun-weighted",
            segmented_data_nans,
            occupancy_lookup_nans,
            temperature_bins_nans,
        ),
    }

def test_fit_caltrack_hourly_model_nans_less_than_week_predict(
    segmented_design_matrices_nans,
    occupancy_lookup_nans,
    temperature_bins_nans,
    temps_extended,
    temps,
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices_nans, occupancy_lookup_nans, temperature_bins_nans
    )

    assert segmented_model.segment_models is not None
    assert segmented_model.model_lookup['jan'].model is not None
    assert segmented_model.model_lookup['may'].model is None
    assert (
        segmented_model.model_lookup['may'].warnings[0].qualified_name
        == "eemeter.fit_caltrack_hourly_model_segment.no_nonnull_data"
    )
    prediction = segmented_model.predict(temps.index, temps).result
    assert prediction.shape[0] == 24
    assert prediction["predicted_usage"].sum().round() == 955.0

@pytest.fixture
def segmented_data_nans_less_than_week():
    num_periods = 4
    index = pd.date_range(start="2017-01-01", periods=num_periods, freq="H", tz="UTC")
    time_features = compute_time_features(index)
    segmented_data = pd.DataFrame(
        {
            "hour_of_week": time_features.hour_of_week,
            "temperature_mean": np.linspace(0, 100, num_periods),
            "meter_value": np.linspace(10, 70, num_periods),
            "weight": np.ones((num_periods,)),
        },
        index=index,
    )
    return segmented_data

@pytest.fixture
def occupancy_lookup_nans_less_than_week():
    index = pd.Categorical(range(168))
    occupancy = pd.Series([i % 2 == 0 for i in range(168)], index=index)
    occupancy_nans_less_than_week = pd.Series([np.nan for i in range(168)], index=index)
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": occupancy,
            "jan-feb-mar-weighted": occupancy,
            "apr-may-jun-weighted": occupancy_nans_less_than_week,
        }
    )

@pytest.fixture
def temperature_bins_nans_less_than_week():
    bins = pd.Series([True, True, True], index=[30, 60, 90])
    bins_nans_less_than_week = pd.Series([False, False, False], index=[30, 60, 90])
    return pd.DataFrame(
        {
            "dec-jan-feb-weighted": bins,
            "jan-feb-mar-weighted": bins,
            "apr-may-jun-weighted": bins_nans_less_than_week,
        }
    )

@pytest.fixture
def segmented_design_matrices_nans_less_than_week(
    segmented_data_nans_less_than_week,
    occupancy_lookup_nans_less_than_week,
    temperature_bins_nans_less_than_week,
):
    return {
        "dec-jan-feb-weighted": caltrack_hourly_fit_feature_processor(
            "dec-jan-feb-weighted",
            segmented_data_nans_less_than_week,
            occupancy_lookup_nans_less_than_week,
            temperature_bins_nans_less_than_week,
        ),
        "apr-may-jun-weighted": caltrack_hourly_fit_feature_processor(
            "apr-may-jun-weighted",
            segmented_data_nans_less_than_week,
            occupancy_lookup_nans_less_than_week,
            temperature_bins_nans_less_than_week,
        ),
    }

@pytest.fixture
def temps_extended():
    index = pd.date_range(start="2017-01-01", periods=168, freq="H", tz="UTC")
    temps = pd.Series(1, index=index)
    return temps

def test_fit_caltrack_hourly_model_nans_less_than_week_fit(
    segmented_design_matrices_nans_less_than_week,
    occupancy_lookup_nans_less_than_week,
    temperature_bins_nans_less_than_week,
    temps_extended,
):
    segmented_model = fit_caltrack_hourly_model(
        segmented_design_matrices_nans_less_than_week,
        occupancy_lookup_nans_less_than_week,
        temperature_bins_nans_less_than_week,
    )

    assert segmented_model.segment_models is not None
    assert segmented_model.segment_models[0].model is not None
    assert segmented_model.segment_models[1].model is None
    assert (
        segmented_model.segment_models[1].warnings[0].qualified_name
        == "eemeter.fit_caltrack_hourly_model_segment.no_nonnull_data"
    )
    prediction = segmented_model.predict(temps_extended.index, temps_extended).result
    assert prediction.shape[0] == 168
    assert prediction.dropna().shape[0] == 4
<|endoftext|>