<commit_before>import os
import StringIO

from PIL import Image

from django.db import models
from django.db.models import fields
from django.utils.crypto import get_random_string
from django.core.files.storage import default_storage
from django.core.files.uploadedfile import InMemoryUploadedFile

from .utils import from_dotted_path

IMAGE_VARIANTS = []

class YADTImageField(fields.Field):
    concrete = False

    def __init__(self, variants=None, cachebust=False, track_exists=False, fallback=False, format='jpeg', filename_prefix=lambda x: x.pk):
        super(YADTImageField, self).__init__()

        self.variants = {}
        self.cachebust = cachebust
        self.track_exists = track_exists
        self.filename_prefix = filename_prefix

        variants = variants or {}
        for name, config in variants.iteritems():
            if name == 'original':
                raise ValueError("'original' is a reserved variant name")

            if config['format'] not in ('jpeg', 'png'):
                raise ValueError(
                    "'%s' is not a valid format" % config['format']
                )

            self.variants[name] = YADTVariantConfig(
                self,
                name,
                config['format'],
                kwargs=config.get('kwargs', None),
                pipeline=config.get('pipeline', ()),
                fallback=config.get('fallback', False),
            )

        self.variants['original'] = YADTVariantConfig(
            self,
            'original',
            format=format,
            original=True,
            fallback=fallback,
        )

    def contribute_to_class(self, cls, name):
        # Set up this field...
        self.attname = name
        self.name = name
        self.model = cls
        self.column = None

        setattr(cls, name, Descriptor(self))

        cls._meta.add_field(self, virtual=True)

        # Now set up several other management fields
        self.cachebusting_field = None
        self.exists_field = None

        self.upload_to = os.path.join(
            'yadt',
            '%s.%s' % (
                self.model._meta.app_label,
                self.model._meta.object_name,
            ),
            self.name,
        )

        if self.cachebust:
            self.cachebusting_field = models.CharField(
                max_length=8,
                default='',
                blank=True,
            )

            cls.add_to_class('%s_hash' % name, self.cachebusting_field)

        if self.track_exists:
            self.exists_field = models.BooleanField(default=False)

            cls.add_to_class('%s_exists' % name, self.exists_field)

    def db_type(self, connection):
        return None

class YADTVariantConfig(object):
    def __init__(self, field, name, format, kwargs=None, fallback=None, original=False, pipeline=()):
        self.field = field
        self.name = name

        self.kwargs = kwargs or {}
        self.format = format
        self.fallback = fallback
        self.original = original
        self.pipeline = pipeline

        for x in self.pipeline:
            name = x['name']

            # Allow fields to not require cumbersome prefix in most cases
            if '.' not in name:
                name = 'django_yadt.processors.%s' % x['name']

            x['fn'] = from_dotted_path(name)

        IMAGE_VARIANTS.append(self)

    def fallback_filename(self):
        return os.path.join(
            '%s.%s' % (
                self.field.model._meta.app_label,
                self.field.model._meta.object_name,
            ),
            self.field.name,
            '%s.%s' % (self.name, self.format),
        )

class Descriptor(object):
    def __init__(self, field):
        self.field = field

    def __get__(self, instance=None, owner=None):
        if instance is None:
            return YADTClassImage(self.field)

        return YADTImage(self.field, instance)

##

class YADTImage(object):
    def __init__(self, field, instance):
        self.field = field
        self.instance = instance
        self.variants = {}

        for name, config in self.field.variants.iteritems():
            self.variants[name] = YADTImageFile(name, config, self, instance)
        self.__dict__.update(self.variants)

    def __repr__(self):
        return u"<YADTImage: %s.%s.%s (%s)>" % (
            self.field.model._meta.app_label,
            self.field.model._meta.object_name,
            self.field.name,
            self.field.upload_to,
        )

    def save(self, *args, **kwargs):
        try:
            return self.original.save(*args, **kwargs)
        finally:
            self.mark_exists(True)

    def exists(self):
        if not self.field.exists_field:
            return self.original.exists()

        return getattr(self.instance, self.field.exists_field.name)

    def refresh(self):
        for variant in self.variants.values():
            if not variant.config.original:
                variant.refresh()

    def cachebust(self):
        if self.field.cachebusting_field:
            return setattr(
                self.instance,
                self.field.cachebusting_field.name,
                get_random_string(self.field.cachebusting_field.max_length),
            )

    def mark_exists(self, exists):
        if self.field.exists_field:
            return setattr(self.instance, self.field.exists_field.name, exists)

    def delete(self):
        for variant in self.variants.values():
            variant.delete()
        self.cachebust()
        self.mark_exists(False)

class YADTImageFile(object):
    def __init__(self, name, config, image, instance):
        self.name = name
        self.image = image
        self.config = config
        self.instance = instance

    def __repr__(self):
        return u"<YADTImageFile: %s>" % self._filename()

    @property
    def url(self):
        url = default_storage.url(self._filename())

        if self.image.field.cachebusting_field:
            suffix = getattr(
                self.instance,
                self.image.field.cachebusting_field.name,
            )

            if suffix:
                # If URL already has a querystring, append an anonymous param.
                if '?' in url:
                    url += '&_=%s' % suffix
                else:
                    url += '?%s' % suffix

        return url

    def exists(self):
        return default_storage.exists(self._filename())

    def save(self, content):
        self.delete()

        # Ensure correct content type in S3 (etc.)
        content.content_type = 'image/%s' % self.config.format

        expected = self._filename()
        result = default_storage.save(expected, content)

        assert expected == result, "Image was not stored at the " \
            "location we wanted (%r vs %r)" % (expected, result)

        if self.config.original:
            self.image.refresh()

        self.image.cachebust()

    def open(self, mode='rb'):
        return default_storage.open(self._filename())

    def delete(self):
        return default_storage.delete(self._filename())

    def refresh(self):
        if self.config.original:
            raise NotImplementedError("Cannot refresh the original image")

        im = Image.open(self.image.original.open())

        if im.format == 'PNG':
            original_im = im.copy()
            original_im = original_im.convert('RGBA')
            im = Image.new('RGBA', original_im.size, (255, 255, 255, 255))
            im.paste(original_im, (0, 0), original_im)
        else:
            im = im.convert('RGB')

        # Apply processing pipeline
        for x in self.config.pipeline:
            im = x['fn'](im, x)

        fileobj = StringIO.StringIO()
        im.save(fileobj, self.config.format, **self.config.kwargs)

        self.save(InMemoryUploadedFile(
            fileobj,
            None,
            self._filename(),
            'application/octet-stream',
            fileobj.len,
            None,
        ))

    def _filename(self):
        # Calculate the filename dynamically to accomodate the case where
        # ``self.instance`` was initially assigned when it didn't have a
        # primary key and was subsequently given one.

        return os.path.join(
            self.image.field.upload_to,
            self.name,
            '%s.%s' % (
                self.image.field.filename_prefix(self.instance),
                self.config.format,
            ),
        )

##

class YADTClassImage(object):
    def __init__(self, field):
        self.field = field

        self.variants = {}

        for name, config in self.field.variants.iteritems():
            self.variants[name] = YADTClassVariant(name, config, self)
        self.__dict__.update(self.variants)

    def __repr__(self):
        return u"<YADTClassImage: %s.%s.%s (%s)>" % (
            self.field.model._meta.app_label,
            self.field.model._meta.object_name,
            self.field.name,
            self.field.upload_to,
        )

class YADTClassVariant(object):
    def __init__(self, name, config, image):
        self.name = name
        self.image = image
        self.config = config

    def refresh_all(self, generator=False):
        if self.config.original:
            raise NotImplementedError("Cannot refresh the original image")

        for instance in self.image.field.model._default_manager.order_by():
            image = getattr(instance, self.image.field.name)

            if image.original.exists():
                getattr(image, self.name).refresh()

            yield image

        self.invalidate_all()

    def invalidate_all(self):
        if self.image.field.cachebusting_field:
            field = self.image.field.cachebusting_field

            field.model.objects.update(**{
                field.name: get_random_string(field.max_length)
            })

    def cachebust(self):
        if self.field.cachebusting_field:
            return setattr(
                self.instance,
                self.field.cachebusting_field.name,
                get_random_string(self.field.cachebusting_field.max_length),
            )

<commit_msg>Return the filename, like .save() normally does<commit_after>import os
import StringIO

from PIL import Image

from django.db import models
from django.db.models import fields
from django.utils.crypto import get_random_string
from django.core.files.storage import default_storage
from django.core.files.uploadedfile import InMemoryUploadedFile

from .utils import from_dotted_path

IMAGE_VARIANTS = []

class YADTImageField(fields.Field):
    concrete = False

    def __init__(self, variants=None, cachebust=False, track_exists=False, fallback=False, format='jpeg', filename_prefix=lambda x: x.pk):
        super(YADTImageField, self).__init__()

        self.variants = {}
        self.cachebust = cachebust
        self.track_exists = track_exists
        self.filename_prefix = filename_prefix

        variants = variants or {}
        for name, config in variants.iteritems():
            if name == 'original':
                raise ValueError("'original' is a reserved variant name")

            if config['format'] not in ('jpeg', 'png'):
                raise ValueError(
                    "'%s' is not a valid format" % config['format']
                )

            self.variants[name] = YADTVariantConfig(
                self,
                name,
                config['format'],
                kwargs=config.get('kwargs', None),
                pipeline=config.get('pipeline', ()),
                fallback=config.get('fallback', False),
            )

        self.variants['original'] = YADTVariantConfig(
            self,
            'original',
            format=format,
            original=True,
            fallback=fallback,
        )

    def contribute_to_class(self, cls, name):
        # Set up this field...
        self.attname = name
        self.name = name
        self.model = cls
        self.column = None

        setattr(cls, name, Descriptor(self))

        cls._meta.add_field(self, virtual=True)

        # Now set up several other management fields
        self.cachebusting_field = None
        self.exists_field = None

        self.upload_to = os.path.join(
            'yadt',
            '%s.%s' % (
                self.model._meta.app_label,
                self.model._meta.object_name,
            ),
            self.name,
        )

        if self.cachebust:
            self.cachebusting_field = models.CharField(
                max_length=8,
                default='',
                blank=True,
            )

            cls.add_to_class('%s_hash' % name, self.cachebusting_field)

        if self.track_exists:
            self.exists_field = models.BooleanField(default=False)

            cls.add_to_class('%s_exists' % name, self.exists_field)

    def db_type(self, connection):
        return None

class YADTVariantConfig(object):
    def __init__(self, field, name, format, kwargs=None, fallback=None, original=False, pipeline=()):
        self.field = field
        self.name = name

        self.kwargs = kwargs or {}
        self.format = format
        self.fallback = fallback
        self.original = original
        self.pipeline = pipeline

        for x in self.pipeline:
            name = x['name']

            # Allow fields to not require cumbersome prefix in most cases
            if '.' not in name:
                name = 'django_yadt.processors.%s' % x['name']

            x['fn'] = from_dotted_path(name)

        IMAGE_VARIANTS.append(self)

    def fallback_filename(self):
        return os.path.join(
            '%s.%s' % (
                self.field.model._meta.app_label,
                self.field.model._meta.object_name,
            ),
            self.field.name,
            '%s.%s' % (self.name, self.format),
        )

class Descriptor(object):
    def __init__(self, field):
        self.field = field

    def __get__(self, instance=None, owner=None):
        if instance is None:
            return YADTClassImage(self.field)

        return YADTImage(self.field, instance)

##

class YADTImage(object):
    def __init__(self, field, instance):
        self.field = field
        self.instance = instance
        self.variants = {}

        for name, config in self.field.variants.iteritems():
            self.variants[name] = YADTImageFile(name, config, self, instance)
        self.__dict__.update(self.variants)

    def __repr__(self):
        return u"<YADTImage: %s.%s.%s (%s)>" % (
            self.field.model._meta.app_label,
            self.field.model._meta.object_name,
            self.field.name,
            self.field.upload_to,
        )

    def save(self, *args, **kwargs):
        try:
            return self.original.save(*args, **kwargs)
        finally:
            self.mark_exists(True)

    def exists(self):
        if not self.field.exists_field:
            return self.original.exists()

        return getattr(self.instance, self.field.exists_field.name)

    def refresh(self):
        for variant in self.variants.values():
            if not variant.config.original:
                variant.refresh()

    def cachebust(self):
        if self.field.cachebusting_field:
            return setattr(
                self.instance,
                self.field.cachebusting_field.name,
                get_random_string(self.field.cachebusting_field.max_length),
            )

    def mark_exists(self, exists):
        if self.field.exists_field:
            return setattr(self.instance, self.field.exists_field.name, exists)

    def delete(self):
        for variant in self.variants.values():
            variant.delete()
        self.cachebust()
        self.mark_exists(False)

class YADTImageFile(object):
    def __init__(self, name, config, image, instance):
        self.name = name
        self.image = image
        self.config = config
        self.instance = instance

    def __repr__(self):
        return u"<YADTImageFile: %s>" % self._filename()

    @property
    def url(self):
        url = default_storage.url(self._filename())

        if self.image.field.cachebusting_field:
            suffix = getattr(
                self.instance,
                self.image.field.cachebusting_field.name,
            )

            if suffix:
                # If URL already has a querystring, append an anonymous param.
                if '?' in url:
                    url += '&_=%s' % suffix
                else:
                    url += '?%s' % suffix

        return url

    def exists(self):
        return default_storage.exists(self._filename())

    def save(self, content):
        self.delete()

        # Ensure correct content type in S3 (etc.)
        content.content_type = 'image/%s' % self.config.format

        expected = self._filename()
        result = default_storage.save(expected, content)

        assert expected == result, "Image was not stored at the " \
            "location we wanted (%r vs %r)" % (expected, result)

        if self.config.original:
            self.image.refresh()

        self.image.cachebust()

        return result

    def open(self, mode='rb'):
        return default_storage.open(self._filename())

    def delete(self):
        return default_storage.delete(self._filename())

    def refresh(self):
        if self.config.original:
            raise NotImplementedError("Cannot refresh the original image")

        im = Image.open(self.image.original.open())

        if im.format == 'PNG':
            original_im = im.copy()
            original_im = original_im.convert('RGBA')
            im = Image.new('RGBA', original_im.size, (255, 255, 255, 255))
            im.paste(original_im, (0, 0), original_im)
        else:
            im = im.convert('RGB')

        # Apply processing pipeline
        for x in self.config.pipeline:
            im = x['fn'](im, x)

        fileobj = StringIO.StringIO()
        im.save(fileobj, self.config.format, **self.config.kwargs)

        self.save(InMemoryUploadedFile(
            fileobj,
            None,
            self._filename(),
            'application/octet-stream',
            fileobj.len,
            None,
        ))

    def _filename(self):
        # Calculate the filename dynamically to accomodate the case where
        # ``self.instance`` was initially assigned when it didn't have a
        # primary key and was subsequently given one.

        return os.path.join(
            self.image.field.upload_to,
            self.name,
            '%s.%s' % (
                self.image.field.filename_prefix(self.instance),
                self.config.format,
            ),
        )

##

class YADTClassImage(object):
    def __init__(self, field):
        self.field = field

        self.variants = {}

        for name, config in self.field.variants.iteritems():
            self.variants[name] = YADTClassVariant(name, config, self)
        self.__dict__.update(self.variants)

    def __repr__(self):
        return u"<YADTClassImage: %s.%s.%s (%s)>" % (
            self.field.model._meta.app_label,
            self.field.model._meta.object_name,
            self.field.name,
            self.field.upload_to,
        )

class YADTClassVariant(object):
    def __init__(self, name, config, image):
        self.name = name
        self.image = image
        self.config = config

    def refresh_all(self, generator=False):
        if self.config.original:
            raise NotImplementedError("Cannot refresh the original image")

        for instance in self.image.field.model._default_manager.order_by():
            image = getattr(instance, self.image.field.name)

            if image.original.exists():
                getattr(image, self.name).refresh()

            yield image

        self.invalidate_all()

    def invalidate_all(self):
        if self.image.field.cachebusting_field:
            field = self.image.field.cachebusting_field

            field.model.objects.update(**{
                field.name: get_random_string(field.max_length)
            })

    def cachebust(self):
        if self.field.cachebusting_field:
            return setattr(
                self.instance,
                self.field.cachebusting_field.name,
                get_random_string(self.field.cachebusting_field.max_length),
            )

<|endoftext|>


<commit_before>__title__ = 'dj-bitcoin'
__version__ = '0.0.1'
__author__ = 'Ryan Pineo'
__license__ = 'MIT'
__copyright__ = 'Copyright 2016 The SilverLogic'
<commit_msg>version bump<commit_after>__title__ = 'dj-bitcoin'
__version__ = '0.0.2'
__author__ = 'Ryan Pineo'
__license__ = 'MIT'
__copyright__ = 'Copyright 2016 The SilverLogic'
<|endoftext|>


<commit_before>"""
Unit tests for the excel_ui module.

"""

import os
import collections
import unittest

import numpy as np
import pandas as pd
try:
    import pandas.testing as tm       # for pandas >= 0.20.1
except ImportError:
    import pandas.util.testing as tm  # for pandas <= 0.19.2

import FlowCal

class TestReadTable(unittest.TestCase):
    """
    Class to test excel_ui.read_table()

    """
    def setUp(self):
        # Name of the file to read
        self.filename = 'test/test_excel_ui.xlsx'

    def test_read_table(self):
        """
        Test for proper loading of a table from an Excel sheet.

        """
        # Sheet to read
        sheetname = "Instruments"
        # Column to use as index labels
        index_col = "ID"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        expected_index = pd.Series([u'FC001', u'FC002'], name='ID')
        expected_columns = [u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       index=expected_index,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(self.filename,
                                            sheetname=sheetname,
                                            index_col=index_col)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_xls(self):
        """
        Test for proper loading of a table from an old-format Excel sheet.

        """
        xls_filename = 'test/test_excel_ui.xls'

        # Sheet to read
        sheetname = "Instruments"
        # Column to use as index labels
        index_col = "ID"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        expected_index = pd.Series([u'FC001', u'FC002'], name='ID')
        expected_columns = [u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       index=expected_index,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(xls_filename,
                                            sheetname=sheetname,
                                            index_col=index_col)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_no_index_col(self):
        """
        Test proper loading of a table when no index column is specified.

        """
        # Sheet to read
        sheetname = "Instruments"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'ID'] = u'FC001'
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'ID'] = u'FC002'
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        expected_columns = [u'ID',
                            u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(self.filename,
                                            sheetname=sheetname)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_with_empty_row(self):
        """
        Test for proper loading of a table that includes an empty row.

        """
        # Sheet to read
        sheetname = "Instruments (empty row)"
        # Column to use as index labels
        index_col = "ID"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Some other flow cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-A'
        row[u'Side Scatter Channel'] = u'SSC-A'
        row[u'Fluorescence Channels'] = u'FL1-A, FL2-A, FL3-A'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        expected_index = pd.Series([u'FC001', u'FC002', u'FC003'], name='ID')
        expected_columns = [u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       index=expected_index,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(self.filename,
                                            sheetname=sheetname,
                                            index_col=index_col)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_duplicated_id_error(self):
        """
        Test for error when table contains duplicated index values.

        """
        # Sheet to read
        sheetname = "Instruments (duplicated)"
        # Column to use as index labels
        index_col = "ID"

        # Call function
        self.assertRaises(ValueError,
                          FlowCal.excel_ui.read_table,
                          self.filename,
                          sheetname,
                          index_col)

    def test_read_table_list_argument_error(self):
        """
        Test for error when `sheetname` is a list.

        """
        # Sheet to read
        sheetname = ["Instruments", "Instruments (duplicated)"]
        # Column to use as index labels
        index_col = "ID"

        # Call function
        self.assertRaises(TypeError,
                          FlowCal.excel_ui.read_table,
                          self.filename,
                          sheetname,
                          index_col)

    def test_read_table_none_argument_error(self):
        """
        Test for error when `sheetname` is None.

        """
        # Sheet to read
        sheetname = None
        # Column to use as index labels
        index_col = "ID"

        # Call function
        self.assertRaises(TypeError,
                          FlowCal.excel_ui.read_table,
                          self.filename,
                          sheetname,
                          index_col)

class TestRegexMEFValues(unittest.TestCase):
    """
    Class to test the regex for finding MEF Values columns.

    """
    def setUp(self):
        # Test dataset
        # Each tuple consists of a string and the corresponding match solution.
        # If the string is not expected to match, the solution is None.
        # If the string is expected to match, the solution is the expected
        # contents of group 1.
        self.test_data = [('FL1 MEF Values', 'FL1'),
                          (' FL1 MEF Values ', 'FL1'),
                          ('   FL1 MEF Values', 'FL1'),
                          ('FL1-H MEF Values', 'FL1-H'),
                          ('mCherry-A MEF Values', 'mCherry-A'),
                          ('MEF Values', None),
                          (' MEF Values', None),
                          ('  MEF Values', None),
                          (' MEF Values ', None),
                          ('test', None),
                          (' test', None),
                          ('  test', None),
                          (' test ', None),
                          ('test MEF Values', 'test'),
                          ('Parameter 14', None),
                          ('1', None),
                          ('1 MEF Values', '1'),
                          ('13', None),
                          ('13 MEF Values', '13'),
                          ('Parameter 14MEF Values', None),
                          ('Parameter 14 MEFValues', None),
                          ('Parameter 14 MEF Values', 'Parameter 14'),
                          ('Parameter 14 MEF  Values', 'Parameter 14'),
                          ('Parameter 14  MEF Values', 'Parameter 14'),
                          ('Parameter  14 MEF Values', 'Parameter  14'),
                          ('Parameter   14 MEF Values', 'Parameter   14'),
                          ('  here is another test of ', None),
                          ('  here is another test of MEF Values', 'here is another test of'),
                          ('  test test MEF Values', 'test test'),
                          ('520nm Light Intensity (umol/(m^2*s))', None),
                          ('520nm Light Intensity (umol/(m^2*s)) MEF Values', '520nm Light Intensity (umol/(m^2*s))'),
                          ]

    def test_match(self):
        # Get compiled regex
        r = FlowCal.excel_ui.re_mef_values
        # Iterate over test data
        for s, sol in self.test_data:
            # Match
            m = r.match(s)
            # Check
            if sol is None:
                self.assertIsNone(m)
            else:
                self.assertEqual(sol, m.group(1))

class TestRegexUnits(unittest.TestCase):
    """
    Class to test the regex for finding Units columns.

    """
    def setUp(self):
        # Test dataset
        # Each tuple consists of a string and the corresponding match solution.
        # If the string is not expected to match, the solution is None.
        # If the string is expected to match, the solution is the expected
        # contents of group 1.
        self.test_data = [('FL1 Units', 'FL1'),
                          (' FL1 Units ', 'FL1'),
                          ('   FL1 Units', 'FL1'),
                          ('FL1-H Units', 'FL1-H'),
                          ('mCherry-A Units', 'mCherry-A'),
                          ('Units', None),
                          (' Units', None),
                          ('  Units', None),
                          (' Units ', None),
                          ('test', None),
                          (' test', None),
                          ('  test', None),
                          (' test ', None),
                          ('test Units', 'test'),
                          ('Parameter 14', None),
                          ('1', None),
                          ('1 Units', '1'),
                          ('13', None),
                          ('13 Units', '13'),
                          ('Parameter 14Units', None),
                          ('Parameter 14 Units', 'Parameter 14'),
                          ('Parameter  14 Units', 'Parameter  14'),
                          ('Parameter   14 Units', 'Parameter   14'),
                          ('  here is another test of ', None),
                          ('  here is another test of Units', 'here is another test of'),
                          ('  test test Units', 'test test'),
                          ('520nm Light Intensity (umol/(m^2*s))', None),
                          ('520nm Light Intensity (umol/(m^2*s)) Units', '520nm Light Intensity (umol/(m^2*s))'),
                          ]

    def test_match(self):
        # Get compiled regex
        r = FlowCal.excel_ui.re_units
        # Iterate over test data
        for s, sol in self.test_data:
            # Match
            m = r.match(s)
            # Check
            if sol is None:
                self.assertIsNone(m)
            else:
                self.assertEqual(sol, m.group(1))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add unit test for excel_ui.write_workbook().<commit_after>"""
Unit tests for the excel_ui module.

"""

import os
import collections
import unittest

import numpy as np
import pandas as pd
try:
    import pandas.testing as tm       # for pandas >= 0.20.1
except ImportError:
    import pandas.util.testing as tm  # for pandas <= 0.19.2

import FlowCal

class TestReadTable(unittest.TestCase):
    """
    Class to test excel_ui.read_table()

    """
    def setUp(self):
        # Name of the file to read
        self.filename = 'test/test_excel_ui.xlsx'

    def test_read_table(self):
        """
        Test for proper loading of a table from an Excel sheet.

        """
        # Sheet to read
        sheetname = "Instruments"
        # Column to use as index labels
        index_col = "ID"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        expected_index = pd.Series([u'FC001', u'FC002'], name='ID')
        expected_columns = [u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       index=expected_index,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(self.filename,
                                            sheetname=sheetname,
                                            index_col=index_col)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_xls(self):
        """
        Test for proper loading of a table from an old-format Excel sheet.

        """
        xls_filename = 'test/test_excel_ui.xls'

        # Sheet to read
        sheetname = "Instruments"
        # Column to use as index labels
        index_col = "ID"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        expected_index = pd.Series([u'FC001', u'FC002'], name='ID')
        expected_columns = [u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       index=expected_index,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(xls_filename,
                                            sheetname=sheetname,
                                            index_col=index_col)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_no_index_col(self):
        """
        Test proper loading of a table when no index column is specified.

        """
        # Sheet to read
        sheetname = "Instruments"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'ID'] = u'FC001'
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'ID'] = u'FC002'
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        expected_columns = [u'ID',
                            u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(self.filename,
                                            sheetname=sheetname)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_with_empty_row(self):
        """
        Test for proper loading of a table that includes an empty row.

        """
        # Sheet to read
        sheetname = "Instruments (empty row)"
        # Column to use as index labels
        index_col = "ID"

        # Expected output
        expected_output_list = []
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        expected_output_list.append(row)
        row = {}
        row[u'Description'] = u'Some other flow cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-A'
        row[u'Side Scatter Channel'] = u'SSC-A'
        row[u'Fluorescence Channels'] = u'FL1-A, FL2-A, FL3-A'
        row[u'Time Channel'] = u'Time'
        expected_output_list.append(row)
        expected_index = pd.Series([u'FC001', u'FC002', u'FC003'], name='ID')
        expected_columns = [u'Description',
                            u'Forward Scatter Channel',
                            u'Side Scatter Channel',
                            u'Fluorescence Channels',
                            u'Time Channel']

        expected_output = pd.DataFrame(expected_output_list,
                                       index=expected_index,
                                       columns=expected_columns)

        # Read table
        table = FlowCal.excel_ui.read_table(self.filename,
                                            sheetname=sheetname,
                                            index_col=index_col)

        # Compare
        tm.assert_frame_equal(table, expected_output)

    def test_read_table_duplicated_id_error(self):
        """
        Test for error when table contains duplicated index values.

        """
        # Sheet to read
        sheetname = "Instruments (duplicated)"
        # Column to use as index labels
        index_col = "ID"

        # Call function
        self.assertRaises(ValueError,
                          FlowCal.excel_ui.read_table,
                          self.filename,
                          sheetname,
                          index_col)

    def test_read_table_list_argument_error(self):
        """
        Test for error when `sheetname` is a list.

        """
        # Sheet to read
        sheetname = ["Instruments", "Instruments (duplicated)"]
        # Column to use as index labels
        index_col = "ID"

        # Call function
        self.assertRaises(TypeError,
                          FlowCal.excel_ui.read_table,
                          self.filename,
                          sheetname,
                          index_col)

    def test_read_table_none_argument_error(self):
        """
        Test for error when `sheetname` is None.

        """
        # Sheet to read
        sheetname = None
        # Column to use as index labels
        index_col = "ID"

        # Call function
        self.assertRaises(TypeError,
                          FlowCal.excel_ui.read_table,
                          self.filename,
                          sheetname,
                          index_col)

class TestWriteWorkbook(unittest.TestCase):
    """
    Class to test excel_ui.write_workbook()

    """
    def setUp(self):
        file_dir = os.path.dirname(os.path.abspath(__file__))
        self.test_file = os.path.join(file_dir, 'test_write_workbook.xlsx')

    def test_write_workbook(self):
        """
        Test proper writing of Excel workbook.

        """
        # Make test table
        df_list = []
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer'
        row[u'Forward Scatter Channel'] = u'FSC-H'
        row[u'Side Scatter Channel'] = u'SSC-H'
        row[u'Fluorescence Channels'] = u'FL1-H, FL2-H, FL3-H'
        row[u'Time Channel'] = u'Time'
        df_list.append(row)
        row = {}
        row[u'Description'] = u'Moake\'s Flow Cytometer (new acquisition card)'
        row[u'Forward Scatter Channel'] = u'FSC'
        row[u'Side Scatter Channel'] = u'SSC'
        row[u'Fluorescence Channels'] = u'FL1, FL2, FL3'
        row[u'Time Channel'] = u'TIME'
        df_list.append(row)

        index_col = 'ID'
        df_idx = pd.Series([u'FC001', u'FC002'], name=index_col)

        df_columns = [u'Description',
                      u'Forward Scatter Channel',
                      u'Side Scatter Channel',
                      u'Fluorescence Channels',
                      u'Time Channel']

        table = pd.DataFrame(df_list,
                             index=df_idx,
                             columns=df_columns)

        # Write table to file
        sheetname = 'Instruments'
        FlowCal.excel_ui.write_workbook(filename=self.test_file,
                                        table_list=[(sheetname, table)])

        # Read table from file
        read_table = FlowCal.excel_ui.read_table(filename=self.test_file,
                                                 sheetname=sheetname,
                                                 index_col=index_col)

        # Compare tables
        tm.assert_frame_equal(table, read_table)

    def tearDown(self):
        os.remove(self.test_file)

class TestRegexMEFValues(unittest.TestCase):
    """
    Class to test the regex for finding MEF Values columns.

    """
    def setUp(self):
        # Test dataset
        # Each tuple consists of a string and the corresponding match solution.
        # If the string is not expected to match, the solution is None.
        # If the string is expected to match, the solution is the expected
        # contents of group 1.
        self.test_data = [('FL1 MEF Values', 'FL1'),
                          (' FL1 MEF Values ', 'FL1'),
                          ('   FL1 MEF Values', 'FL1'),
                          ('FL1-H MEF Values', 'FL1-H'),
                          ('mCherry-A MEF Values', 'mCherry-A'),
                          ('MEF Values', None),
                          (' MEF Values', None),
                          ('  MEF Values', None),
                          (' MEF Values ', None),
                          ('test', None),
                          (' test', None),
                          ('  test', None),
                          (' test ', None),
                          ('test MEF Values', 'test'),
                          ('Parameter 14', None),
                          ('1', None),
                          ('1 MEF Values', '1'),
                          ('13', None),
                          ('13 MEF Values', '13'),
                          ('Parameter 14MEF Values', None),
                          ('Parameter 14 MEFValues', None),
                          ('Parameter 14 MEF Values', 'Parameter 14'),
                          ('Parameter 14 MEF  Values', 'Parameter 14'),
                          ('Parameter 14  MEF Values', 'Parameter 14'),
                          ('Parameter  14 MEF Values', 'Parameter  14'),
                          ('Parameter   14 MEF Values', 'Parameter   14'),
                          ('  here is another test of ', None),
                          ('  here is another test of MEF Values', 'here is another test of'),
                          ('  test test MEF Values', 'test test'),
                          ('520nm Light Intensity (umol/(m^2*s))', None),
                          ('520nm Light Intensity (umol/(m^2*s)) MEF Values', '520nm Light Intensity (umol/(m^2*s))'),
                          ]

    def test_match(self):
        # Get compiled regex
        r = FlowCal.excel_ui.re_mef_values
        # Iterate over test data
        for s, sol in self.test_data:
            # Match
            m = r.match(s)
            # Check
            if sol is None:
                self.assertIsNone(m)
            else:
                self.assertEqual(sol, m.group(1))

class TestRegexUnits(unittest.TestCase):
    """
    Class to test the regex for finding Units columns.

    """
    def setUp(self):
        # Test dataset
        # Each tuple consists of a string and the corresponding match solution.
        # If the string is not expected to match, the solution is None.
        # If the string is expected to match, the solution is the expected
        # contents of group 1.
        self.test_data = [('FL1 Units', 'FL1'),
                          (' FL1 Units ', 'FL1'),
                          ('   FL1 Units', 'FL1'),
                          ('FL1-H Units', 'FL1-H'),
                          ('mCherry-A Units', 'mCherry-A'),
                          ('Units', None),
                          (' Units', None),
                          ('  Units', None),
                          (' Units ', None),
                          ('test', None),
                          (' test', None),
                          ('  test', None),
                          (' test ', None),
                          ('test Units', 'test'),
                          ('Parameter 14', None),
                          ('1', None),
                          ('1 Units', '1'),
                          ('13', None),
                          ('13 Units', '13'),
                          ('Parameter 14Units', None),
                          ('Parameter 14 Units', 'Parameter 14'),
                          ('Parameter  14 Units', 'Parameter  14'),
                          ('Parameter   14 Units', 'Parameter   14'),
                          ('  here is another test of ', None),
                          ('  here is another test of Units', 'here is another test of'),
                          ('  test test Units', 'test test'),
                          ('520nm Light Intensity (umol/(m^2*s))', None),
                          ('520nm Light Intensity (umol/(m^2*s)) Units', '520nm Light Intensity (umol/(m^2*s))'),
                          ]

    def test_match(self):
        # Get compiled regex
        r = FlowCal.excel_ui.re_units
        # Iterate over test data
        for s, sol in self.test_data:
            # Match
            m = r.match(s)
            # Check
            if sol is None:
                self.assertIsNone(m)
            else:
                self.assertEqual(sol, m.group(1))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" WSGIユーティリティ """

from brocade import handler, application
import cgi

class WSGI_Handler(handler.BaseHandler):
	""" リクエストハンドラ（WSGI版） """

	def __init__(self, environ, start_response, default_language = "ja"):
		super(WSGI_Handler, self).__init__(default_language)

		# 引数
		self.__environ        = environ.copy()
		self.__start_response = start_response

	def _param_get(self):
		""" GETパラメータを取得 """
		return self.__param(False)

	def _param_post(self):
		""" POSTパラメータを取得 """
		return self.__param(True)

	def __param(self, post = False):
		""" パラメータを取得

		@param post: GETデータを取得するならFalse, POSTデータを取得するならTrue
		@return: FieldStorage
		"""
		fp = None
		environ = self.__environ
		keep_blank_values = True

		if post:
			# POSTならクエリ文字列をクリア
			environ = environ.copy()
			environ["QUERY_STRING"] = ""
			if "wsgi.input" in environ:
				fp = environ["wsgi.input"]

		field_storage = cgi.FieldStorage(fp = fp, environ = environ, keep_blank_values = keep_blank_values)
		return WSGI_Parameters(field_storage)

	def get_env(self, name, default = ""):
		""" 指定の環境変数を取得

		@param name: 変数名
		@param default: 取得できない場合のデフォルト値
		@return: 変数値またはデフォルト値
		"""
		return self.__environ.get(name, default)

	def start(self, status = 200):
		""" レスポンスヘッダ出力

		@param status: ステータス
		"""
		from . import httputils
		self.__start_response(httputils.get_status_value(status), self.build_http_headers())

class WSGI_Parameters(handler.BaseParameters):
	""" パラメータ（WSGI版） """

	def __init__(self, field_storage):
		self.__field_storage = field_storage

	def value(self, name, default = None):
		""" 単一のパラメータ値を取得

		@param name: パラメータ名
		@param default: デフォルトパラメータ
		@return: パラメータ値
		"""
		return self.__field_storage.getfirst(name, default)

	def values(self, name):
		""" パラメータ値のリストを取得

		@param name: パラメータ名
		@return: パラメータ値のリスト（値がない場合は空のリスト）
		"""
		return self.__field_storage.getlist(name)

	def file(self, name):
		""" アップロードファイルを取得

		@param name: パラメータ名
		@return: filename, fileを属性に持つオブジェクト
		"""
		field_storage = self.__field_storage
		if not name in field_storage:
			return None

		f = field_storage[name]
		if isinstance(f, list):
			f = f[0]

		if not self.__is_file(f):
			return None

		return f

	def files(self, name):
		""" アップロードファイルのリストを取得

		@param name: パラメータ名
		@return: filename, fileを属性に持つオブジェクトのリスト
		"""
		field_storage = self.__field_storage
		if not name in field_storage:
			return []

		files = field_storage[name]
		if not isinstance(files, list):
			files = [files]

		if not self.__is_file(files[0]):
			return []

		return files

	@staticmethod
	def __is_file(f):
		""" 引数がファイルか？

		@param f: チェックする引数
		@return: Yes/No
		"""
		if not hasattr(f, "filename"):
			return False
		if not hasattr(f, "file"):
			return False

		if f.file == None:
			return False

		return True

class WSGI_Application(application.BaseApplication):
	""" アプリケーション（WSGI版） """

	def __call__(self, environ, start_response):
		""" リクエスト処理 """
#		environ["HTTP_ACCEPT_CHARSET"] = "iso-8859-5, unicode-1-1;q=0.8"
#		environ["HTTP_ACCEPT_CHARSET"] = "Shift_JIS,utf-8;q=0.7,*;q=0.7"

		uri = environ.get("PATH_INFO", "")

		(handler, args) = self._get_matched_data(uri)
		handler_instance = handler(environ, start_response)
		yield handler_instance(*args)
<commit_msg>R: location of importing cgi<commit_after># -*- coding: utf-8 -*-
""" WSGIユーティリティ """

from brocade import handler, application

class WSGI_Handler(handler.BaseHandler):
	""" リクエストハンドラ（WSGI版） """

	def __init__(self, environ, start_response, default_language = "ja"):
		super(WSGI_Handler, self).__init__(default_language)

		# 引数
		self.__environ        = environ.copy()
		self.__start_response = start_response

	def _param_get(self):
		""" GETパラメータを取得 """
		return self.__param(False)

	def _param_post(self):
		""" POSTパラメータを取得 """
		return self.__param(True)

	def __param(self, post = False):
		""" パラメータを取得

		@param post: GETデータを取得するならFalse, POSTデータを取得するならTrue
		@return: FieldStorage
		"""
		import cgi

		fp = None
		environ = self.__environ
		keep_blank_values = True

		if post:
			# POSTならクエリ文字列をクリア
			environ = environ.copy()
			environ["QUERY_STRING"] = ""
			if "wsgi.input" in environ:
				fp = environ["wsgi.input"]

		field_storage = cgi.FieldStorage(fp = fp, environ = environ, keep_blank_values = keep_blank_values)
		return WSGI_Parameters(field_storage)

	def get_env(self, name, default = ""):
		""" 指定の環境変数を取得

		@param name: 変数名
		@param default: 取得できない場合のデフォルト値
		@return: 変数値またはデフォルト値
		"""
		return self.__environ.get(name, default)

	def start(self, status = 200):
		""" レスポンスヘッダ出力

		@param status: ステータス
		"""
		from . import httputils
		self.__start_response(httputils.get_status_value(status), self.build_http_headers())

class WSGI_Parameters(handler.BaseParameters):
	""" パラメータ（WSGI版） """

	def __init__(self, field_storage):
		self.__field_storage = field_storage

	def value(self, name, default = None):
		""" 単一のパラメータ値を取得

		@param name: パラメータ名
		@param default: デフォルトパラメータ
		@return: パラメータ値
		"""
		return self.__field_storage.getfirst(name, default)

	def values(self, name):
		""" パラメータ値のリストを取得

		@param name: パラメータ名
		@return: パラメータ値のリスト（値がない場合は空のリスト）
		"""
		return self.__field_storage.getlist(name)

	def file(self, name):
		""" アップロードファイルを取得

		@param name: パラメータ名
		@return: filename, fileを属性に持つオブジェクト
		"""
		field_storage = self.__field_storage
		if not name in field_storage:
			return None

		f = field_storage[name]
		if isinstance(f, list):
			f = f[0]

		if not self.__is_file(f):
			return None

		return f

	def files(self, name):
		""" アップロードファイルのリストを取得

		@param name: パラメータ名
		@return: filename, fileを属性に持つオブジェクトのリスト
		"""
		field_storage = self.__field_storage
		if not name in field_storage:
			return []

		files = field_storage[name]
		if not isinstance(files, list):
			files = [files]

		if not self.__is_file(files[0]):
			return []

		return files

	@staticmethod
	def __is_file(f):
		""" 引数がファイルか？

		@param f: チェックする引数
		@return: Yes/No
		"""
		if not hasattr(f, "filename"):
			return False
		if not hasattr(f, "file"):
			return False

		if f.file == None:
			return False

		return True

class WSGI_Application(application.BaseApplication):
	""" アプリケーション（WSGI版） """

	def __call__(self, environ, start_response):
		""" リクエスト処理 """
#		environ["HTTP_ACCEPT_CHARSET"] = "iso-8859-5, unicode-1-1;q=0.8"
#		environ["HTTP_ACCEPT_CHARSET"] = "Shift_JIS,utf-8;q=0.7,*;q=0.7"

		uri = environ.get("PATH_INFO", "")

		(handler, args) = self._get_matched_data(uri)
		handler_instance = handler(environ, start_response)
		yield handler_instance(*args)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.
from default import Test, db, with_context
from pybossa.view.facebook import manage_user
from pybossa.model.user import User

class TestFacebook(Test):
    def test_manage_user_with_email(self):
        """Test FACEBOOK manage_user works."""
        with self.flask_app.test_request_context('/'):
            # First with a new user
            user_data = dict(id=1, username='facebook',
                             email='f@f.com', name='name')
            token = 't'
            user = manage_user(token, user_data, None)
            assert user.email_addr == user_data['email'], user
            assert user.name == user_data['username'], user
            assert user.fullname == user_data['name'], user
            assert user.facebook_user_id == user_data['id'], user

            # Second with the same user
            user = manage_user(token, user_data, None)
            assert user.email_addr == user_data['email'], user
            assert user.name == user_data['username'], user
            assert user.fullname == user_data['name'], user
            assert user.facebook_user_id == user_data['id'], user

            # Finally with a user that already is in the system
            user_data = dict(id=10, username=self.name,
                             email=self.email_addr, name=self.fullname)
            token = 'tA'
            user = manage_user(token, user_data, None)
            assert user.facebook_user_id == 10, err_msg

    @with_context
    def test_manage_user_without_email(self):
        """Test FACEBOOK manage_user without e-mail works."""
        # First with a new user
        user_data = dict(id=1, username='facebook', name='name')
        token = 't'
        user = manage_user(token, user_data, None)
        assert user.email_addr == user_data['email'], user
        assert user.name == user_data['username'], user
        assert user.fullname == user_data['name'], user
        assert user.facebook_user_id == user_data['id'], user

        # Second with the same user
        user = manage_user(token, user_data, None)
        assert user.email_addr == user_data['email'], user
        assert user.name == user_data['username'], user
        assert user.fullname == user_data['name'], user
        assert user.facebook_user_id == user_data['id'], user

        # Finally with a user that already is in the system
        user_data = dict(id=10, username=self.name,
                         email=self.email_addr, name=self.fullname)
        token = 'tA'
        user = manage_user(token, user_data, None)
        err_msg = "It should return the same user"
        assert user.facebook_user_id == 10, err_msg
<commit_msg>Refactor<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.
from default import Test, db, with_context
from pybossa.view.facebook import manage_user
from pybossa.model.user import User

class TestFacebook(Test):
    @with_context
    def test_manage_user_with_email(self):
        """Test FACEBOOK manage_user works."""
        # First with a new user
        user_data = dict(id=1, username='facebook',
                         email='f@f.com', name='name')
        token = 't'
        user = manage_user(token, user_data, None)
        assert user.email_addr == user_data['email'], user
        assert user.name == user_data['username'], user
        assert user.fullname == user_data['name'], user
        assert user.facebook_user_id == user_data['id'], user

        # Second with the same user
        user = manage_user(token, user_data, None)
        assert user.email_addr == user_data['email'], user
        assert user.name == user_data['username'], user
        assert user.fullname == user_data['name'], user
        assert user.facebook_user_id == user_data['id'], user

        # Finally with a user that already is in the system
        user_data = dict(id=10, username=self.name,
                         email=self.email_addr, name=self.fullname)
        token = 'tA'
        user = manage_user(token, user_data, None)
        err_msg = "It should return the same user"
        assert user.facebook_user_id == 10, err_msg

    @with_context
    def test_manage_user_without_email(self):
        """Test FACEBOOK manage_user without e-mail works."""
        # First with a new user
        user_data = dict(id=1, username='facebook', name='name')
        token = 't'
        user = manage_user(token, user_data, None)
        assert user.email_addr == user_data['email'], user
        assert user.name == user_data['username'], user
        assert user.fullname == user_data['name'], user
        assert user.facebook_user_id == user_data['id'], user

        # Second with the same user
        user = manage_user(token, user_data, None)
        assert user.email_addr == user_data['email'], user
        assert user.name == user_data['username'], user
        assert user.fullname == user_data['name'], user
        assert user.facebook_user_id == user_data['id'], user

        # Finally with a user that already is in the system
        user_data = dict(id=10, username=self.name,
                         email=self.email_addr, name=self.fullname)
        token = 'tA'
        user = manage_user(token, user_data, None)
        err_msg = "It should return the same user"
        assert user.facebook_user_id == 10, err_msg
<|endoftext|>


<commit_before>#! /usr/bin/env python
'''
A script to create a simple, browser-based countdown timer.

usage: python browsertimer.py
'''
# Imports
from bokeh.layouts import column, row
from bokeh import plotting as bkplt
from bokeh.models import ColumnDataSource, Button, Slider, CustomJS

# Set default/starting values
default_minutes = 5
default_seconds = 0
seconds_left = default_minutes*60 + default_seconds
# ColumnDataSource data values must be iterable,
# so the values below are placed inside single-element lists
color = ['#78c400']
text_color = ['#ffffff']
start_time = [default_minutes*60 + default_seconds]
time_remaining = list(start_time)
time_string = ['%02d:%02d' % (default_minutes, default_seconds)]
# Create data source for timer plot
source=ColumnDataSource(data=dict(start_time=start_time,
                                  start_mins=[default_minutes],
                                  start_secs=[default_seconds],
                                  time_remaining=time_remaining,
                                  time_string=time_string,
                                  fill_color=color,
                                  text_color=text_color,
                                  interval_id=[0]))

# No tools required for this one
tools = ''

# JS Callbacks
run_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled',true);
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled');
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    function countdown() {
        start_time = data['start_time'];
        time_remaining = data['time_remaining'];
        time_string = data['time_string'];
        text_color = data['text_color'];
        fill_color = data['fill_color'];
        if (time_remaining[0] == 0) {
            if (text_color[0] == '#ffffff') {
                text_color[0] = '#ff0000';
            }else {
                text_color[0] = '#ffffff';
            }
        }else{
            time_remaining[0]--;
            time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
            if (time_remaining[0] <= start_time[0]/10) {
                fill_color[0] = '#ff0000';
            }
        }
        source.trigger('change');
    }
    disable_button(start_button);
    enable_button(stop_button);
    enable_button(reset_button);
    interval_id[0] = setInterval(countdown, 1000);
    source.trigger('change');
""")

stop_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled',true);
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled');
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    clearInterval(interval_id[0]);
    enable_button(start_button);
    disable_button(stop_button);
    enable_button(reset_button);
""")

reset_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled',true);
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled');
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    start_time = data['start_time'];
    time_remaining = data['time_remaining'];
    time_string = data['time_string'];
    text_color = data['text_color'];
    fill_color = data['fill_color'];
    text_color[0] = '#ffffff';
    fill_color[0] = '78c400';
    if (interval_id[0] != 0) {
        clearInterval(interval_id[0]);
    }
    time_remaining[0] = start_time[0];
    time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
    enable_button(start_button);
    disable_button(stop_button);
    source.trigger('change');
""")

set_start_time_JS = CustomJS(args=dict(source=source), code="""
    var data = source.get('data');
    var input_mins = mins_slider.get('value');
    var input_secs = secs_slider.get('value');
    time_string = data['time_string'];
    time_remaining = data['time_remaining'];
    start_button_id = start_button.get('id');
    start_button_element = document.querySelector('#modelid_' + start_button_id + '>button');
    start_time = data['start_time'];
    start_time[0] = (input_mins * 60) + (input_secs);
    if (start_button_element.hasAttribute('disabled')) {
    } else {
        time_remaining[0] = start_time[0];
        time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
    }
    source.trigger('change');
""")

# Create plot: a color block, with text centered inside
p1 = bkplt.figure(x_range=(-8, 8), y_range=(-5, 5),
                  plot_width=900, plot_height=600,
                  title=None, tools=tools)
p1.rect(x=[0], y=[0],
        width=16, height=10,
        fill_color='fill_color',
        line_color=None,
        name='block',
        source=source)
p1.text(x=[0], y=[0],
        text='time_string',
        text_color='text_color',
        alpha=0.75,
        text_font_size='128pt',
        text_baseline='middle',
        text_align='center',
        name='timer',
        source=source)

# Remove axes, labels & tick lines
p1.ygrid.grid_line_color = None
p1.xgrid.grid_line_color = None
p1.axis.axis_line_color  = None
p1.axis.major_label_text_color = None
p1.axis.major_tick_line_color = None
p1.axis.minor_tick_line_color = None

# Sliders
minutes_slider = Slider(start=0, end=99,
                        value=default_minutes, step=1,
                        title="Minutes",
                        callback=set_start_time_JS)
seconds_slider = Slider(start=0, end=59,
                        value=default_seconds, step=1,
                        title="Seconds",
                        callback=set_start_time_JS)
set_start_time_JS.args['mins_slider'] = minutes_slider
set_start_time_JS.args['secs_slider'] = seconds_slider

# Buttons
start_button = Button(label="Start", callback=run_timer_JS)
run_timer_JS.args['start_button'] = stop_timer_JS.args['start_button'] = reset_timer_JS.args['start_button'] = set_start_time_JS.args['start_button'] = start_button
stop_button = Button(label="Stop", disabled=True, callback=stop_timer_JS)
run_timer_JS.args['stop_button'] = stop_timer_JS.args['stop_button'] = reset_timer_JS.args['stop_button'] = stop_button
reset_button = Button(label="Reset", callback=reset_timer_JS)
run_timer_JS.args['reset_button'] = stop_timer_JS.args['reset_button'] = reset_timer_JS.args['reset_button'] = reset_button

# Layout plot & widgets
layout = column(row(minutes_slider, seconds_slider),
                row(start_button, stop_button, reset_button),
                p1)

# Show figure
bkplt.output_file('browsertimer.html', title="Countdown Timer")
bkplt.show(layout)
<commit_msg>added # to fill color in reset_timer_JS callback, to fix bug that prevented timer from rendering in Firefox.<commit_after>#! /usr/bin/env python
'''
A script to create a simple, browser-based countdown timer.

usage: python browsertimer.py
'''
# Imports
from bokeh.layouts import column, row
from bokeh import plotting as bkplt
from bokeh.models import ColumnDataSource, Button, Slider, CustomJS

# Set default/starting values
default_minutes = 5
default_seconds = 0
seconds_left = default_minutes*60 + default_seconds
# ColumnDataSource data values must be iterable,
# so the values below are placed inside single-element lists
color = ['#78c400']
text_color = ['#ffffff']
start_time = [default_minutes*60 + default_seconds]
time_remaining = list(start_time)
time_string = ['%02d:%02d' % (default_minutes, default_seconds)]
# Create data source for timer plot
source=ColumnDataSource(data=dict(start_time=start_time,
                                  start_mins=[default_minutes],
                                  start_secs=[default_seconds],
                                  time_remaining=time_remaining,
                                  time_string=time_string,
                                  fill_color=color,
                                  text_color=text_color,
                                  interval_id=[0]))

# No tools required for this one
tools = ''

# JS Callbacks
run_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled',true);
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled');
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    function countdown() {
        start_time = data['start_time'];
        time_remaining = data['time_remaining'];
        time_string = data['time_string'];
        text_color = data['text_color'];
        fill_color = data['fill_color'];
        if (time_remaining[0] == 0) {
            if (text_color[0] == '#ffffff') {
                text_color[0] = '#ff0000';
            }else {
                text_color[0] = '#ffffff';
            }
        }else{
            time_remaining[0]--;
            time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
            if (time_remaining[0] <= start_time[0]/10) {
                fill_color[0] = '#ff0000';
            }
        }
        source.trigger('change');
    }
    disable_button(start_button);
    enable_button(stop_button);
    enable_button(reset_button);
    interval_id[0] = setInterval(countdown, 1000);
    source.trigger('change');
""")

stop_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled',true);
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled');
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    clearInterval(interval_id[0]);
    enable_button(start_button);
    disable_button(stop_button);
    enable_button(reset_button);
""")

reset_timer_JS = CustomJS(args=dict(source=source), code="""
    function disable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.setAttribute('disabled',true);
    }
    function enable_button(button) {
        button_id = button.get('id');
        button_element = document.querySelector('#modelid_' + button_id + '>button');
        button_element.removeAttribute('disabled');
    }
    var data = source.get('data');
    interval_id = data['interval_id'];
    start_time = data['start_time'];
    time_remaining = data['time_remaining'];
    time_string = data['time_string'];
    text_color = data['text_color'];
    fill_color = data['fill_color'];
    text_color[0] = '#ffffff';
    fill_color[0] = '#78c400';
    if (interval_id[0] != 0) {
        clearInterval(interval_id[0]);
    }
    time_remaining[0] = start_time[0];
    time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
    enable_button(start_button);
    disable_button(stop_button);
    source.trigger('change');
""")

set_start_time_JS = CustomJS(args=dict(source=source), code="""
    var data = source.get('data');
    var input_mins = mins_slider.get('value');
    var input_secs = secs_slider.get('value');
    time_string = data['time_string'];
    time_remaining = data['time_remaining'];
    start_button_id = start_button.get('id');
    start_button_element = document.querySelector('#modelid_' + start_button_id + '>button');
    start_time = data['start_time'];
    start_time[0] = (input_mins * 60) + (input_secs);
    if (start_button_element.hasAttribute('disabled')) {
    } else {
        time_remaining[0] = start_time[0];
        time_string[0] = ('0' + Math.floor(time_remaining[0] / 60)).slice(-2) + ':' + ('0' + Math.floor(time_remaining[0] % 60)).slice(-2);
    }
    source.trigger('change');
""")

# Create plot: a color block, with text centered inside
p1 = bkplt.figure(x_range=(-8, 8), y_range=(-5, 5),
                  plot_width=900, plot_height=600,
                  title=None, tools=tools)
p1.rect(x=[0], y=[0],
        width=16, height=10,
        fill_color='fill_color',
        line_color=None,
        name='block',
        source=source)
p1.text(x=[0], y=[0],
        text='time_string',
        text_color='text_color',
        alpha=0.75,
        text_font_size='128pt',
        text_baseline='middle',
        text_align='center',
        name='timer',
        source=source)

# Remove axes, labels & tick lines
p1.ygrid.grid_line_color = None
p1.xgrid.grid_line_color = None
p1.axis.axis_line_color  = None
p1.axis.major_label_text_color = None
p1.axis.major_tick_line_color = None
p1.axis.minor_tick_line_color = None

# Sliders
minutes_slider = Slider(start=0, end=99,
                        value=default_minutes, step=1,
                        title="Minutes",
                        callback=set_start_time_JS)
seconds_slider = Slider(start=0, end=59,
                        value=default_seconds, step=1,
                        title="Seconds",
                        callback=set_start_time_JS)
set_start_time_JS.args['mins_slider'] = minutes_slider
set_start_time_JS.args['secs_slider'] = seconds_slider

# Buttons
start_button = Button(label="Start", callback=run_timer_JS)
run_timer_JS.args['start_button'] = stop_timer_JS.args['start_button'] = reset_timer_JS.args['start_button'] = set_start_time_JS.args['start_button'] = start_button
stop_button = Button(label="Stop", disabled=True, callback=stop_timer_JS)
run_timer_JS.args['stop_button'] = stop_timer_JS.args['stop_button'] = reset_timer_JS.args['stop_button'] = stop_button
reset_button = Button(label="Reset", callback=reset_timer_JS)
run_timer_JS.args['reset_button'] = stop_timer_JS.args['reset_button'] = reset_timer_JS.args['reset_button'] = reset_button

# Layout plot & widgets
layout = column(row(minutes_slider, seconds_slider),
                row(start_button, stop_button, reset_button),
                p1)

# Show figure
bkplt.output_file('browsertimer.html', title="Countdown Timer")
bkplt.show(layout)
<|endoftext|>


<commit_before>#     Copyright 2020. ThingsBoard
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

from os import path, listdir
from inspect import getmembers, isclass
from importlib import util
from logging import getLogger
from simplejson import dumps, loads
from re import search, match, compile
from jsonpath_rw import jsonpath, parse

log = getLogger("service")

class TBUtility:

    @staticmethod
    def decode(message):
        content = loads(message.payload.decode("utf-8"))
        return content

    @staticmethod
    def validate_converted_data(data):
        json_data = dumps(data)
        if not data.get("deviceName") or data.get("deviceName") is None:
            log.error('deviceName is empty in data %s', json_data)
            return False
        if not data.get("deviceType") or data.get("deviceType") is None:
            log.error('deviceType is empty in data: %s', json_data)
            return False
        if (data.get("attributes") is None and data.get("telemetry") is None) or (not data.get("attributes") and not data.get("telemetry")):
            log.error('No telemetry and attributes in data: %s', json_data)
            return False
        return True

    @staticmethod
    def topic_to_regex(topic):
        return topic.replace("+", "[^/]+").replace("#", ".+")

    @staticmethod
    def regex_to_topic(regex):
        return regex.replace("[^/]+", "+").replace(".+", "#")

    @staticmethod
    def check_and_import(extension_type, module_name):
        extensions_paths = (path.abspath(path.dirname(path.dirname(__file__)) + '/connectors/'.replace('/', path.sep) + extension_type.lower()),
                            '/var/lib/thingsboard_gateway/'.replace('/', path.sep) + extension_type.lower(),
                            path.abspath(path.dirname(path.dirname(__file__)) + '/extensions/'.replace('/', path.sep) + extension_type.lower()))
        try:
            for extension_path in extensions_paths:
                if path.exists(extension_path):
                    for file in listdir(extension_path):
                        if not file.startswith('__') and file.endswith('.py'):
                            try:
                                module_spec = util.spec_from_file_location(module_name, extension_path + path.sep + file)
                                log.debug(module_spec)
                                if module_spec is None:
                                    log.error('Module: {} not found'.format(module_name))
                                    continue
                                else:
                                    module = util.module_from_spec(module_spec)
                                    log.debug(str(module))
                                    try:
                                        module_spec.loader.exec_module(module)
                                    except Exception as e:
                                        log.exception(e)
                                    for extension_class in getmembers(module, isclass):
                                        if module_name in extension_class:
                                            log.debug("Import %s from %s.", module_name, extension_path)
                                            return extension_class[1]
                            except ImportError:
                                continue
                            except Exception as e:
                                log.exception(e)
                else:
                    log.error("Import %s failed, path %s doesn't exist", module_name, extension_path)
        except Exception as e:
            log.exception(e)

    @staticmethod
    def get_value(expression, body={}, value_type="string", get_tag=False, expression_instead_none=False):
        if isinstance(body, str):
            body = loads(body)
        if not expression:
            return ''
        positions = search(r'\$\{(?:(.*))\}', expression)
        if positions is not None:
            p1 = positions.regs[-1][0]
            p2 = positions.regs[-1][1]
        else:
            p1 = 0
            p2 = len(expression)
        target_str = str(expression[p1:p2])
        if get_tag:
            return target_str
        full_value = None
        try:
            if value_type == "string":
                try:
                    full_value = expression[0: min(abs(p1 - 2), 0)] + body[target_str.split()[0]] + expression[p2 + 1:len(expression)]
                except KeyError:
                    pass
                if full_value is None:
                    try:
                        jsonpath_expression = parse(target_str)
                        jsonpath_match = jsonpath_expression.find(body)
                        if jsonpath_match is not None:
                            full_value = jsonpath_match[0].value
                    except Exception:
                        pass
                if full_value is None:
                    try:
                        full_value = search(expression, body).group(0)
                    except Exception:
                        full_value = None
                if full_value is None and expression_instead_none:
                    full_value = expression
            else:
                if full_value is None:
                    try:
                        jsonpath_expression = parse(target_str)
                        jsonpath_match = jsonpath_expression.find(body)
                        if jsonpath_match is not None:
                            full_value = jsonpath_match[0].value
                    except Exception:
                        full_value = None
                if full_value is None:
                    try:
                        full_value = search(expression, body).group(0)
                    except Exception:
                        full_value = None
                if full_value is None:
                    try:
                        full_value = body.get(target_str.split()[0])
                    except Exception as e:
                        log.exception(e)
        except TypeError:
            if full_value is None:
                log.error('Value is None - Cannot find the pattern: %s in %s. Expression will be interpreted as value.', target_str, dumps(body))
                full_value = expression
        except Exception as e:
            log.exception(e)
            return None
        return full_value
<commit_msg>TBUtility improvements and refactoring<commit_after>#     Copyright 2020. ThingsBoard
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

from os import path, listdir
from inspect import getmembers, isclass
from importlib import util
from logging import getLogger
from simplejson import dumps, loads
from re import search
from jsonpath_rw import parse

log = getLogger("service")

class TBUtility:

    @staticmethod
    def decode(message):
        content = loads(message.payload.decode("utf-8"))
        return content

    @staticmethod
    def validate_converted_data(data):
        json_data = dumps(data)
        if not data.get("deviceName") or data.get("deviceName") is None:
            log.error('deviceName is empty in data %s', json_data)
            return False
        if not data.get("deviceType") or data.get("deviceType") is None:
            log.error('deviceType is empty in data: %s', json_data)
            return False
        if not data.get("attributes") and not data.get("telemetry"):
            log.error('No telemetry and attributes in data: %s', json_data)
            return False
        return True

    @staticmethod
    def topic_to_regex(topic):
        return topic.replace("+", "[^/]+").replace("#", ".+")

    @staticmethod
    def regex_to_topic(regex):
        return regex.replace("[^/]+", "+").replace(".+", "#")

    @staticmethod
    def check_and_import(extension_type, module_name):
        extensions_paths = (path.abspath(path.dirname(path.dirname(__file__)) + '/connectors/'.replace('/', path.sep) + extension_type.lower()),
                            '/var/lib/thingsboard_gateway/'.replace('/', path.sep) + extension_type.lower(),
                            path.abspath(path.dirname(path.dirname(__file__)) + '/extensions/'.replace('/', path.sep) + extension_type.lower()))
        try:
            for extension_path in extensions_paths:
                if path.exists(extension_path):
                    for file in listdir(extension_path):
                        if not file.startswith('__') and file.endswith('.py'):
                            try:
                                module_spec = util.spec_from_file_location(module_name, extension_path + path.sep + file)
                                log.debug(module_spec)
                                if module_spec is None:
                                    log.error('Module: %s not found', module_name)
                                    continue
                                else:
                                    module = util.module_from_spec(module_spec)
                                    log.debug(str(module))
                                    module_spec.loader.exec_module(module)
                                    for extension_class in getmembers(module, isclass):
                                        if module_name in extension_class:
                                            log.debug("Import %s from %s.", module_name, extension_path)
                                            return extension_class[1]
                            except ImportError:
                                continue
                            except Exception as e:
                                log.exception(e)
                else:
                    log.error("Import %s failed, path %s doesn't exist", module_name, extension_path)
        except Exception as e:
            log.exception(e)

    @staticmethod
    def get_value(expression, body=None, value_type="string", get_tag=False, expression_instead_none=False):
        if isinstance(body, str):
            body = loads(body)
        if not expression:
            return ''
        positions = search(r'\$\{(?:(.*))\}', expression)
        if positions is not None:
            p1 = positions.regs[-1][0]
            p2 = positions.regs[-1][1]
        else:
            p1 = 0
            p2 = len(expression)
        target_str = str(expression[p1:p2])
        if get_tag:
            return target_str
        full_value = None
        try:
            if isinstance(body, dict) and target_str.split()[0] in body:
                if value_type.lower() == "string":
                    full_value = expression[0: max(abs(p1 - 2), 0)] + body[target_str.split()[0]] + expression[p2 + 1:len(expression)]
                else:
                    full_value = body.get(target_str.split()[0])
            elif isinstance(body, dict):
                jsonpath_expression = parse(target_str)
                jsonpath_match = jsonpath_expression.find(body)
                if jsonpath_match:
                    full_value = jsonpath_match[0].value
            elif isinstance(body, (str, bytes)):
                search_result = search(expression, body)
                if search_result.groups():
                    full_value = search_result.group(0)
            elif expression_instead_none:
                full_value = expression
        except Exception as e:
            log.exception(e)
        finally:
            return full_value
<|endoftext|>