<commit_before>import drake
import mistune
import re

non_unicode_word_character = re.compile('\W')

def id(text):
  return re.sub(non_unicode_word_character, '-', text.lower())

class HeaderIdsRenderer(mistune.Renderer):

  def header(self, text, level, raw=None):
    header = super().header(text, level, raw)
    position = 3 # 1 + len('h%d' % level)
    return header[0:position] + ' id="' + id(text) + '"' + header[position:]

class Renderer(drake.Builder):

  def __init__(self, source, target = None):
    self.__source = source
    self.__target = target or drake.Node(
      self.__source.name_relative.with_extension('html')
    )
    drake.Builder.__init__(self, [self.__source], [self.__target])

  def execute(self):
    self.output('Render %s' % self.__target)
    renderer = HeaderIdsRenderer()
    markdown = mistune.Markdown(
      renderer = renderer,
      parse_inline_html = True,
    )
    with open(str(self.__source.path())) as input:
      with open(str(self.__target.path()), 'w') as output:
        output.write(markdown(input.read()))
    return True

class Source(drake.Node):

  def __init__(self, *args, **kwargs):
    super().__init__(*args, **kwargs)
    Renderer(self)

drake.Node.extensions['md'] = Source
<commit_msg>markdown: Handle replacements before and after processing the markdown.<commit_after>import drake
import mistune
import re

non_unicode_word_character = re.compile('\W')

def id(text):
  return re.sub(non_unicode_word_character, '-', text.lower())

class HeaderIdsRenderer(mistune.Renderer):

  def header(self, text, level, raw = None):
    header = super().header(text, level, raw)
    position = 3 # 1 + len('h%d' % level)
    return header[0:position] + ' id="' + id(text) + '"' + header[position:]

class Renderer(drake.Builder):

  def __init__(
    self, source,
    target = None,
    replace_before = {
      '_blank': '_blank',
    },
    replace_after = {
      '<code>--': '<code>‑‑',
      '"': '\"',
    },
  ):
    self.__source = source
    self.__target = target or drake.Node(
      self.__source.name_relative.with_extension('html')
    )
    self.__replace_before = replace_before
    self.__replace_after = replace_after
    drake.Builder.__init__(self, [self.__source], [self.__target])

  def execute(self):
    self.output('Render %s' % self.__target)
    renderer = HeaderIdsRenderer()
    markdown = mistune.Markdown(
      renderer = renderer,
      parse_inline_html = True,
    )
    with open(str(self.__source.path())) as input:
      with open(str(self.__target.path()), 'w') as output:
        text = input.read()
        for k, v in self.__replace_before.items():
          text = text.replace(k, v)
        html = markdown(text)
        for k, v in self.__replace_after.items():
          html = html.replace(k, v)
        output.write(html)
    return True

class Source(drake.Node):

  def __init__(self, *args, **kwargs):
    super().__init__(*args, **kwargs)
    Renderer(self)

drake.Node.extensions['md'] = Source
<|endoftext|>


<commit_before>__author__ = 'sachinpatney'

import json
import urllib2
import base64

from common import IMonaTask
from common import BuildNotifier
from common import sync_read_status_file
from common import Timeline

class VSO_API_Templates:
    getBuilds = "https://{0}.visualstudio.com/defaultcollection/{1}/_apis/build/builds?api-version={2}"

class VSO(IMonaTask):

    mona = None

    def get_auth(self):
        d = sync_read_status_file()
        return [d['vso_username'], d['vso_password']]

    def is_broken(self, build):
        if build['status'] == 'failed':
            return True
        return False

    def get_broken_builds(self, data):
        brokenBuilds = []

        for build in data['value']:
            if self.is_broken(build):
                brokenBuilds.append(build)
            else:  # We only want broken builds after last success
                break
        return brokenBuilds

    def get_build_info(self):
        request = urllib2.Request(VSO_API_Templates.getBuilds.format('pbix','powerbiclients','1.0'))
        auth = self.get_auth()
        base64string = base64.encodestring('%s:%s' % (auth[0], auth[1])).replace('\n', '')
        request.add_header("Authorization", "Basic %s" % base64string)
        result = urllib2.urlopen(request)

        return json.loads(result.read())

    def __run__(self, time):
        broken = self.get_broken_builds(self.get_build_info())

        if len(broken) == 0:
            if BuildNotifier.build_was_broken():
                BuildNotifier.update_build_status(False)
                BuildNotifier.notify_all_clear()
                print 'Sent all clear notification'
            else:
                print 'Was not broken previously too, so do nothing new'
        else:
            if not BuildNotifier.build_was_broken():
                culprits = []
                for b in broken:
                    culprits.append(b['requests'][0]['requestedFor'])
                BuildNotifier.notify_build_break(culprits)
                BuildNotifier.update_build_status(True)
                Timeline.add_item('Mona', 'BUILD BREAK',
                                  '{0} broke the build. Change was requested by {1}'.format(
                                      broken[0]['buildNumber'],
                                      broken[0]['requests'][0]['requestedFor']))
                print 'Sent build break notification'
            else:
                print 'Was broken previously too, so do nothing'<commit_msg>updating script<commit_after>__author__ = 'sachinpatney'

import json
import urllib2
import base64

from common import IMonaTask
from common import BuildNotifier
from common import sync_read_status_file
from common import Timeline

class VSO_API_Templates:
    getBuilds = "https://{0}.visualstudio.com/defaultcollection/{1}/_apis/build/builds?api-version={2}"

class VSO(IMonaTask):

    mona = None

    def get_auth(self):
        d = sync_read_status_file()
        return [d['vso_username'], d['vso_password']]

    def is_broken(self, build):
        if build['status'] == 'failed':
            return True
        return False

    def get_broken_builds(self, data):
        brokenBuilds = []

        for build in data['value']:
            if self.is_broken(build):
                brokenBuilds.append(build)
            #else:  # We only want broken builds after last success
                #break
        return brokenBuilds

    def get_build_info(self):
        request = urllib2.Request(VSO_API_Templates.getBuilds.format('pbix','powerbiclients','1.0'))
        auth = self.get_auth()
        base64string = base64.encodestring('%s:%s' % (auth[0], auth[1])).replace('\n', '')
        request.add_header("Authorization", "Basic %s" % base64string)
        result = urllib2.urlopen(request)

        return json.loads(result.read())

    def __run__(self, time):
        broken = self.get_broken_builds(self.get_build_info())

        if len(broken) == 0:
            if BuildNotifier.build_was_broken():
                BuildNotifier.update_build_status(False)
                BuildNotifier.notify_all_clear()
                print 'Sent all clear notification'
            else:
                print 'Was not broken previously too, so do nothing new'
        else:
            if not BuildNotifier.build_was_broken():
                culprits = []
                for b in broken:
                    culprits.append(b['requests'][0]['requestedFor'])
                BuildNotifier.notify_build_break(culprits)
                BuildNotifier.update_build_status(True)
                Timeline.add_item('Mona', 'BUILD BREAK',
                                  '{0} broke the build. Change was requested by {1}'.format(
                                      broken[0]['buildNumber'],
                                      broken[0]['requests'][0]['requestedFor']))
                print 'Sent build break notification'
            else:
                print 'Was broken previously too, so do nothing'<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db import models, transaction
from django.utils.translation import ugettext as _
from vkontakte_api.models import VkontakteTimelineManager, VkontakteModel, VkontakteContentError, VkontakteError
from vkontakte_api.decorators import fetch_all
from vkontakte_groups.models import Group
from vkontakte_users.models import User, ParseUsersMixin
from datetime import datetime
import logging

log = logging.getLogger('vkontakte_board')

class CommentManager(models.Manager):
    pass

class TopicManager(models.Manager):
    pass

class BoardRemoteManager(VkontakteTimelineManager, ParseUsersMixin):

    def parse_response(self, response, extra_fields=None):
        if isinstance(response, dict):
            self.parse_response_users(response)
            return super(BoardRemoteManager, self).parse_response_list(response[self.response_instances_fieldname], extra_fields)
        else:
            raise VkontakteContentError('Vkontakte response should be dict')

class TopicRemoteManager(BoardRemoteManager):

    response_instances_fieldname = 'topics'

    @transaction.commit_on_success
    @fetch_all(default_count=100)
    def fetch(self, group, ids=None, extended=False, order=None, offset=0, count=100, preview=0, preview_length=90, **kwargs):
        #gid
        #ID группы, список тем которой необходимо получить.
        kwargs['gid'] = group.remote_id
        #tids
        #Список идентификаторов тем, которые необходимо получить (не более 100). По умолчанию возвращаются все темы. Если указан данный параметр, игнорируются параметры order, offset и count (возвращаются все запрошенные темы в указанном порядке).
        if ids and isinstance(ids, (list, tuple)):
            kwargs['tids'] = ','.join(map(lambda i: str(i), ids))
        #extended
        #Если указать в качестве этого параметра 1, то будет возвращена информация о пользователях, являющихся создателями тем или оставившими в них последнее сообщение. По умолчанию 0.
        kwargs['extended'] = int(extended)
        #order
        #Порядок, в котором необходимо вернуть список тем. Возможные значения:
        #1 - по убыванию даты обновления,
        #2 - по убыванию даты создания,
        #-1 - по возрастанию даты обновления,
        #-2 - по возрастанию даты создания.
        #По умолчанию темы возвращаются в порядке, установленном администратором группы. "Прилепленные" темы при любой сортировке возвращаются первыми в списке.
        if order:
            kwargs['order'] = int(order)
        #offset
        #Смещение, необходимое для выборки определенного подмножества тем.
        kwargs['offset'] = int(offset)
        #count
        #Количество тем, которое необходимо получить (но не более 100). По умолчанию 40.
        kwargs['count'] = int(count)
        #preview
        #Набор флагов, определяющий, необходимо ли вернуть вместе с информацией о темах текст первых и последних сообщений в них. Является суммой флагов:
        #1 - вернуть первое сообщение в каждой теме (поле first_comment),
        #2 - вернуть последнее сообщение в каждой теме (поле last_comment). По умолчанию 0 (не возвращать текст сообщений).
        kwargs['preview'] = int(preview)
        #preview_length
        #Количество символов, по которому нужно обрезать первое и последнее сообщение. Укажите 0, если Вы не хотите обрезать сообщение. (по умолчанию 90).
        kwargs['preview_length'] = int(preview_length)

        kwargs['extra_fields'] = {'group_id': group.pk}
        return super(TopicRemoteManager, self).fetch(**kwargs)

class CommentRemoteManager(BoardRemoteManager):

    response_instances_fieldname = 'comments'

    @transaction.commit_on_success
    @fetch_all(default_count=100)
    def fetch(self, topic, extended=False, offset=0, count=100, sort='asc', need_likes=True, before=None, after=None, **kwargs):
        if count > 100:
            raise ValueError("Attribute 'count' can not be more than 100")
        if sort not in ['asc','desc']:
            raise ValueError("Attribute 'sort' should be equal to 'asc' or 'desc'")
        if sort == 'asc' and after:
            raise ValueError("Attribute `sort` should be equal to 'desc' with defined `after` attribute")
        if before and not after:
            raise ValueError("Attribute `before` should be specified with attribute `after`")
        if before and before < after:
            raise ValueError("Attribute `before` should be later, than attribute `after`")

        #gid
        #ID группы, к обсуждениям которой относится указанная тема.
        kwargs['gid'] = topic.group.remote_id
        #tid
        #ID темы в группе
        kwargs['tid'] = topic.remote_id.split('_')[1]
        #extended
        #Если указать в качестве этого параметра 1, то будет возвращена информация о пользователях, являющихся авторами сообщений. По умолчанию 0.
        kwargs['extended'] = int(extended)
        #offset
        #Смещение, необходимое для выборки определенного подмножества сообщений.
        kwargs['offset'] = int(offset)
        # sort
        # порядок сортировки комментариев:
        # asc - хронологический
        # desc - антихронологический
        kwargs['sort'] = sort
        #count
        #Количество сообщений, которое необходимо получить (но не более 100). По умолчанию 20.
        kwargs['count'] = int(count)
        # need_likes
        # 1 - будет возвращено дополнительное поле likes. По умолчанию поле likes не возвращается.
        kwargs['need_likes'] = int(need_likes)

        # special parameters
        kwargs['after'] = after
        kwargs['before'] = before

        kwargs['extra_fields'] = {'topic_id': topic.pk}
        try:
            return super(CommentRemoteManager, self).fetch(**kwargs)
        except VkontakteError, e:
            if e.code == 100 and 'invalid tid' in e.description:
                log.error("Impossible to fetch comments for unexisted topic ID=%s" % topic.remote_id)
                return self.model.objects.none()
            else:
                raise e

class BoardAbstractModel(VkontakteModel):
    class Meta:
        abstract = True

    methods_namespace = 'board'
    slug_prefix = 'topic'

    # TODO: reduce max_length to 20, after changing Comment.remote_id generation rules
    remote_id = models.CharField(u'ID', max_length='50', help_text=u'Уникальный идентификатор', unique=True)

class Topic(BoardAbstractModel):
    class Meta:
        verbose_name = u'Дискуссия групп Вконтакте'
        verbose_name_plural = u'Дискуссии групп Вконтакте'

    remote_pk_field = 'tid'

    group = models.ForeignKey(Group, verbose_name=u'Группа', related_name='topics')

    title = models.CharField(u'Заголовок', max_length=500)
    created = models.DateTimeField(u'Дата создания', db_index=True)
    updated = models.DateTimeField(u'Дата последнего сообщения', null=True, db_index=True)

    created_by = models.ForeignKey(User, related_name='topics_created', verbose_name=u'Пользователь, создавший тему')
    updated_by = models.ForeignKey(User, related_name='topics_updated', verbose_name=u'Пользователь, оставивший последнее сообщение')

    is_closed = models.BooleanField(u'Закрыта?', help_text=u'Тема является закрытой (в ней нельзя оставлять сообщения)')
    is_fixed = models.BooleanField(u'Прикреплена?', help_text=u'Тема является прилепленной (находится в начале списка тем)')

    comments_count = models.PositiveIntegerField(u'Число сообщений в теме', default=0, db_index=True)

    first_comment = models.TextField(u'Текст первого сообщения')
    last_comment = models.TextField(u'Текст последнего сообщения')

    objects = TopicManager()
    remote = TopicRemoteManager(remote_pk=('remote_id',), methods={
        'get': 'getTopics',
    })

    @property
    def slug(self):
        return self.slug_prefix + str(self.remote_id)

    def __unicode__(self):
        return self.title

    def parse(self, response):
        self.created_by = User.objects.get_or_create(remote_id=response.pop('created_by'))[0]
        self.updated_by = User.objects.get_or_create(remote_id=response.pop('updated_by'))[0]
        if 'comments' in response:
            response['comments_count'] = response.pop('comments')

        super(Topic, self).parse(response)

        if '_' not in str(self.remote_id):
            self.remote_id = '-%s_%s' % (self.group.remote_id, self.remote_id)

    @transaction.commit_on_success
    def fetch_comments(self, *args, **kwargs):
        return Comment.remote.fetch(topic=self, *args, **kwargs)

class Comment(BoardAbstractModel):
    class Meta:
        verbose_name = u'Коммментарий дискуссии групп Вконтакте'
        verbose_name_plural = u'Коммментарии дискуссий групп Вконтакте'

    topic = models.ForeignKey(Topic, verbose_name=u'Тема', related_name='comments')
    author = models.ForeignKey(User, related_name='topics_comments', verbose_name=u'Aвтор комментария')
    date = models.DateTimeField(help_text=u'Дата создания', db_index=True)
    text = models.TextField(u'Текст сообщения')
    #attachments - присутствует только если у сообщения есть прикрепления, содержит массив объектов (фотографии, ссылки и т.п.). Более подробная информация представлена на странице Описание поля attachments

    # TODO: implement with tests
#    likes = models.PositiveIntegerField(u'Кол-во лайков', default=0)

    objects = CommentManager()
    remote = CommentRemoteManager(remote_pk=('remote_id',), methods={
        'get': 'getComments',
    })

    @property
    def slug(self):
        return self.slug_prefix + str(self.topic.remote_id) + '?post=' + self.remote_id.split('_')[2]

    def parse(self, response):
        self.author = User.objects.get_or_create(remote_id=response.pop('from_id'))[0]
        # TODO: add parsing attachments and polls
        if 'attachments' in response:
            response.pop('attachments')
        if 'poll' in response:
            response.pop('poll')

        super(Comment, self).parse(response)

        if '_' not in str(self.remote_id):
            # TODO: remove second _ from remote_id, make in 111_111 for comments
            self.remote_id = '%s_%s' % (self.topic.remote_id, self.remote_id)<commit_msg>added checks for right encoding for comment<commit_after># -*- coding: utf-8 -*-
from django.db import models, transaction
from django.utils.translation import ugettext as _
from vkontakte_api.models import VkontakteTimelineManager, VkontakteModel, VkontakteContentError, VkontakteError
from vkontakte_api.decorators import fetch_all
from vkontakte_groups.models import Group
from vkontakte_users.models import User, ParseUsersMixin
from datetime import datetime
import logging

log = logging.getLogger('vkontakte_board')

class CommentManager(models.Manager):
    pass

class TopicManager(models.Manager):
    pass

class BoardRemoteManager(VkontakteTimelineManager, ParseUsersMixin):

    def parse_response(self, response, extra_fields=None):
        if isinstance(response, dict):
            self.parse_response_users(response)
            return super(BoardRemoteManager, self).parse_response_list(response[self.response_instances_fieldname], extra_fields)
        else:
            raise VkontakteContentError('Vkontakte response should be dict')

class TopicRemoteManager(BoardRemoteManager):

    response_instances_fieldname = 'topics'

    @transaction.commit_on_success
    @fetch_all(default_count=100)
    def fetch(self, group, ids=None, extended=False, order=None, offset=0, count=100, preview=0, preview_length=90, **kwargs):
        #gid
        #ID группы, список тем которой необходимо получить.
        kwargs['gid'] = group.remote_id
        #tids
        #Список идентификаторов тем, которые необходимо получить (не более 100). По умолчанию возвращаются все темы. Если указан данный параметр, игнорируются параметры order, offset и count (возвращаются все запрошенные темы в указанном порядке).
        if ids and isinstance(ids, (list, tuple)):
            kwargs['tids'] = ','.join(map(lambda i: str(i), ids))
        #extended
        #Если указать в качестве этого параметра 1, то будет возвращена информация о пользователях, являющихся создателями тем или оставившими в них последнее сообщение. По умолчанию 0.
        kwargs['extended'] = int(extended)
        #order
        #Порядок, в котором необходимо вернуть список тем. Возможные значения:
        #1 - по убыванию даты обновления,
        #2 - по убыванию даты создания,
        #-1 - по возрастанию даты обновления,
        #-2 - по возрастанию даты создания.
        #По умолчанию темы возвращаются в порядке, установленном администратором группы. "Прилепленные" темы при любой сортировке возвращаются первыми в списке.
        if order:
            kwargs['order'] = int(order)
        #offset
        #Смещение, необходимое для выборки определенного подмножества тем.
        kwargs['offset'] = int(offset)
        #count
        #Количество тем, которое необходимо получить (но не более 100). По умолчанию 40.
        kwargs['count'] = int(count)
        #preview
        #Набор флагов, определяющий, необходимо ли вернуть вместе с информацией о темах текст первых и последних сообщений в них. Является суммой флагов:
        #1 - вернуть первое сообщение в каждой теме (поле first_comment),
        #2 - вернуть последнее сообщение в каждой теме (поле last_comment). По умолчанию 0 (не возвращать текст сообщений).
        kwargs['preview'] = int(preview)
        #preview_length
        #Количество символов, по которому нужно обрезать первое и последнее сообщение. Укажите 0, если Вы не хотите обрезать сообщение. (по умолчанию 90).
        kwargs['preview_length'] = int(preview_length)

        kwargs['extra_fields'] = {'group_id': group.pk}
        return super(TopicRemoteManager, self).fetch(**kwargs)

class CommentRemoteManager(BoardRemoteManager):

    response_instances_fieldname = 'comments'

    @transaction.commit_on_success
    @fetch_all(default_count=100)
    def fetch(self, topic, extended=False, offset=0, count=100, sort='asc', need_likes=True, before=None, after=None, **kwargs):
        if count > 100:
            raise ValueError("Attribute 'count' can not be more than 100")
        if sort not in ['asc','desc']:
            raise ValueError("Attribute 'sort' should be equal to 'asc' or 'desc'")
        if sort == 'asc' and after:
            raise ValueError("Attribute `sort` should be equal to 'desc' with defined `after` attribute")
        if before and not after:
            raise ValueError("Attribute `before` should be specified with attribute `after`")
        if before and before < after:
            raise ValueError("Attribute `before` should be later, than attribute `after`")

        #gid
        #ID группы, к обсуждениям которой относится указанная тема.
        kwargs['gid'] = topic.group.remote_id
        #tid
        #ID темы в группе
        kwargs['tid'] = topic.remote_id.split('_')[1]
        #extended
        #Если указать в качестве этого параметра 1, то будет возвращена информация о пользователях, являющихся авторами сообщений. По умолчанию 0.
        kwargs['extended'] = int(extended)
        #offset
        #Смещение, необходимое для выборки определенного подмножества сообщений.
        kwargs['offset'] = int(offset)
        # sort
        # порядок сортировки комментариев:
        # asc - хронологический
        # desc - антихронологический
        kwargs['sort'] = sort
        #count
        #Количество сообщений, которое необходимо получить (но не более 100). По умолчанию 20.
        kwargs['count'] = int(count)
        # need_likes
        # 1 - будет возвращено дополнительное поле likes. По умолчанию поле likes не возвращается.
        kwargs['need_likes'] = int(need_likes)

        # special parameters
        kwargs['after'] = after
        kwargs['before'] = before

        kwargs['extra_fields'] = {'topic_id': topic.pk}
        try:
            return super(CommentRemoteManager, self).fetch(**kwargs)
        except VkontakteError, e:
            if e.code == 100 and 'invalid tid' in e.description:
                log.error("Impossible to fetch comments for unexisted topic ID=%s" % topic.remote_id)
                return self.model.objects.none()
            else:
                raise e

class BoardAbstractModel(VkontakteModel):
    class Meta:
        abstract = True

    methods_namespace = 'board'
    slug_prefix = 'topic'

    # TODO: reduce max_length to 20, after changing Comment.remote_id generation rules
    remote_id = models.CharField(u'ID', max_length='50', help_text=u'Уникальный идентификатор', unique=True)

class Topic(BoardAbstractModel):
    class Meta:
        verbose_name = u'Дискуссия групп Вконтакте'
        verbose_name_plural = u'Дискуссии групп Вконтакте'

    remote_pk_field = 'tid'

    group = models.ForeignKey(Group, verbose_name=u'Группа', related_name='topics')

    title = models.CharField(u'Заголовок', max_length=500)
    created = models.DateTimeField(u'Дата создания', db_index=True)
    updated = models.DateTimeField(u'Дата последнего сообщения', null=True, db_index=True)

    created_by = models.ForeignKey(User, related_name='topics_created', verbose_name=u'Пользователь, создавший тему')
    updated_by = models.ForeignKey(User, related_name='topics_updated', verbose_name=u'Пользователь, оставивший последнее сообщение')

    is_closed = models.BooleanField(u'Закрыта?', help_text=u'Тема является закрытой (в ней нельзя оставлять сообщения)')
    is_fixed = models.BooleanField(u'Прикреплена?', help_text=u'Тема является прилепленной (находится в начале списка тем)')

    comments_count = models.PositiveIntegerField(u'Число сообщений в теме', default=0, db_index=True)

    first_comment = models.TextField(u'Текст первого сообщения')
    last_comment = models.TextField(u'Текст последнего сообщения')

    objects = TopicManager()
    remote = TopicRemoteManager(remote_pk=('remote_id',), methods={
        'get': 'getTopics',
    })

    @property
    def slug(self):
        return self.slug_prefix + str(self.remote_id)

    def __unicode__(self):
        return self.title

    def parse(self, response):
        self.created_by = User.objects.get_or_create(remote_id=response.pop('created_by'))[0]
        self.updated_by = User.objects.get_or_create(remote_id=response.pop('updated_by'))[0]
        if 'comments' in response:
            response['comments_count'] = response.pop('comments')

        super(Topic, self).parse(response)

        if '_' not in str(self.remote_id):
            self.remote_id = '-%s_%s' % (self.group.remote_id, self.remote_id)

    @transaction.commit_on_success
    def fetch_comments(self, *args, **kwargs):
        return Comment.remote.fetch(topic=self, *args, **kwargs)

class Comment(BoardAbstractModel):
    class Meta:
        verbose_name = u'Коммментарий дискуссии групп Вконтакте'
        verbose_name_plural = u'Коммментарии дискуссий групп Вконтакте'

    topic = models.ForeignKey(Topic, verbose_name=u'Тема', related_name='comments')
    author = models.ForeignKey(User, related_name='topics_comments', verbose_name=u'Aвтор комментария')
    date = models.DateTimeField(help_text=u'Дата создания', db_index=True)
    text = models.TextField(u'Текст сообщения')
    #attachments - присутствует только если у сообщения есть прикрепления, содержит массив объектов (фотографии, ссылки и т.п.). Более подробная информация представлена на странице Описание поля attachments

    # TODO: implement with tests
#    likes = models.PositiveIntegerField(u'Кол-во лайков', default=0)

    objects = CommentManager()
    remote = CommentRemoteManager(remote_pk=('remote_id',), methods={
        'get': 'getComments',
    })

    @property
    def slug(self):
        return self.slug_prefix + str(self.topic.remote_id) + '?post=' + self.remote_id.split('_')[2]

    def parse(self, response):
        self.author = User.objects.get_or_create(remote_id=response.pop('from_id'))[0]
        # TODO: add parsing attachments and polls
        if 'attachments' in response:
            response.pop('attachments')
        if 'poll' in response:
            response.pop('poll')

        super(Comment, self).parse(response)

        if '_' not in str(self.remote_id):
            # TODO: remove second _ from remote_id, make in 111_111 for comments
            self.remote_id = '%s_%s' % (self.topic.remote_id, self.remote_id)

    def save(self, *args, **kwargs):
        # check strings for good encoding
        # there is problems to save users with bad encoded activity strings like user ID=-2611_27475528_133349
        # TODO: move it to the level up
        try:
            self.text.encode('utf-16').decode('utf-16')
        except UnicodeDecodeError:
            self.text = ''

        return super(Comment, self).save(*args, **kwargs)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Author: Simone Marsili <simomarsili@gmail.com>
# License: BSD 3 clause
"""Functions module."""
import logging

import numpy

import ndd
from ndd.estimators import Entropy, JSDivergence
from ndd.exceptions import (CardinalityError, DataArrayError,
                            EstimatorInputError, HistogramError, NumericError)

__all__ = [
    'entropy', 'jensen_shannon_divergence', 'interaction_information',
    'coinformation', 'mutual_information', 'conditional_entropy', 'histogram',
    'from_data'
]

logger = logging.getLogger(__name__)

def entropy(pk, k=None, alpha=None, plugin=False, return_std=False):
    """
    Return a Bayesian estimate of the entropy from an array of counts.

    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from an input array of counts pk.

    Parameters
    ----------
    pk : array-like
        The number of occurrences of a set of bins.
    k : int or array-like, optional
        Total number of bins (taking into account unobserved bins);
        k >= len(pk). A float is a valid input for whole numbers (e.g. k=1.e3).
        If an array, set k = numpy.prod(k).
        Defaults to n_bins.
    alpha : float, optional
        If alpha is not None, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.
    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).
    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    Returns
    -------
    entropy : float
        Entropy estimate.
    std : float, optional
        Uncertainty in the entropy estimate
        (approximated standard deviation over the entropy posterior).
        Only if `return_std` is True.

    Raises
    ------
    NumericError
        If result is NaN

    """

    # pk is an array of counts
    estimator = Entropy(alpha, plugin).fit(pk, k)
    S, err = estimator.estimate_, estimator.err_

    if numpy.isnan(S):
        raise NumericError('NaN value')

    if return_std:
        if err is not None and numpy.isnan(err):
            err = numpy.nan
            logger.warning('nan value for entropy posterior std deviation')
        return S, err

    return S

def jensen_shannon_divergence(pk, k=None, alpha=None, plugin=False):
    """
    Return the Jensen-Shannon divergence from a matrix of counts.

    Return an estimate of the Jensen-Shannon divergence between
    n_distributions unknown discrete distributions from a
    n_distributions-by-n_bins input array of counts.
    The estimate (in nats) is computed as a combination of single Bayesian
    entropy estimates. If the total number of samples varies among the
    distributions, the function returns the divergence between the
    distributions with weights proportional to the total number of samples in
    each row (see the general definition of Jensen-Shannon divergence:
    https://en.wikipedia.org/wiki/Jensen-Shannon_divergence).

    Parameters
    ----------

    pk : array-like, shape (n_distributions, n_bins)
        Matrix of frequency counts. Each row corresponds to the number of
        occurrences of a set of bins from a different distribution.
    k : int or array-like, optional
        Total number of bins (taking into account unobserved bins); k >= n_bins
        A float is a valid input for whole numbers (e.g. k=1.e3).
        If an array, set k = numpy.prod(k).
        Defaults to n_bins.
    alpha : float, optional
        If not None, the entropy estimator uses a single Dirichlet prior with
        concentration parameter alpha (fixed alpha estimator). alpha > 0.0.
    plugin : boolean, optional
        If True, use a 'plugin' estimator for the entropy.
        If alpha is passed in combination with plugin == True, add alpha
        pseudoconts to the frequency counts in the plugin estimate.

    Returns
    -------
    float
        Jensen-Shannon divergence.

    Raises
    ------
    NumericError
        If result is NaN

    """

    estimator = JSDivergence(alpha, plugin).fit(pk, k)
    js = estimator.estimate_

    if numpy.isnan(js):
        raise NumericError('NaN value')

    return js

def _nbins(data):
    """
    The number of unique elements along axis 0. If data is p-dimensional,
    the num. of unique elements for each variable.
    """
    # reshape as a p-by-n array
    return [len(numpy.unique(v)) for v in data]

def histogram(data, axis=1, r=None):
    """Compute an histogram from a data array. Wrapper to numpy.unique.

    Parameters
    ----------
    data : array-like
        A n-by-p array of n samples from p variables.
    axis : int, optional
        The sample-indexing axis
    r : int or None, optional
        For r values in the interval [1, p],
        return a generator that yields a bin counts array for each possible
        combination of r variables.

    Returns
    -------
    counts : ndarray
        Bin counts.

    """
    from itertools import combinations

    # check data shape
    data = _check_input_data(data)
    if axis == 0:
        data = data.T
    p = data.shape[0]

    if r is not None:
        if 1 <= r <= p:
            return (ndd.histogram(d) for d in combinations(data, r=r))
        raise HistogramError('r values must be in the interval [1, p]')

    # statistics for the p-dimensional variable
    _, counts = numpy.unique(data, return_counts=True, axis=1)
    return counts

def from_data(ar, ks=None, axis=1, r=0):
    """
    Given an array of data, return an entropy estimate.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis
    r : int, optional
        If r > 0, return a generator yielding estimates for the p-choose-r
        possible combinations of length r from the p variables.

    Returns
    -------
    float
        Entropy estimate

    Raises
    ------
    CardinalityError
        If ks is array-like and len(ks) != p
        If r > 0 and is a scalar.

    """
    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]
    if r == 0:
        r = p

    # EntropyBasedEstimator objects are callable and return the fitted estimate
    estimator = Entropy()

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim:
            if len(ks) != p:
                raise CardinalityError('k should have len %s' % p)

    if r != p:
        if ks.ndim == 0:
            raise CardinalityError('For combinations, ks cant be a scalar')

        counts_combinations = histogram(ar, r=r)
        alphabet_size_combinations = (numpy.prod(x)
                                      for x in combinations(ks, r=r))
        return (
            estimator(*args)
            for args in zip(counts_combinations, alphabet_size_combinations))

    counts = histogram(ar)
    return estimator(counts, k=ks)

def interaction_information(ar, ks=None, axis=1, r=0):
    """Interaction information from p-by-n data matrix.

    If p == 2, return an estimate of the mutual information between the
    variables corresponding to the two columns.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis
    r : int, optional
        If r > 0, return a generator yielding estimates for the p-choose-r
        possible combinations of length r from the p variables.
        If r == 1, return the entropy for each variable. If r == 2 return the
        mutual information for each possible pair. If r > 2 return the
        interaction information for each possible subset of length r.
        Combinations are ordered as: list(itertools.combinations(range(p), r)).

    Returns
    -------
    float
        Interaction information estimate.

    """
    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]
    if r == 0:
        r = p

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim > 0:  # pylint: disable=comparison-with-callable
            if len(ks) != p:
                raise CardinalityError('k should have len %r (%r)' %
                                       (p, len(ks)))
        else:
            raise CardinalityError('ks cant be a scalar')

    def iinfo(X, ks):
        info = 0.0
        px = len(X)
        for ri in range(1, px + 1):
            sgn = (-1)**(px - ri)
            info -= sgn * numpy.sum(from_data(X, ks=ks, r=ri))
        return info

    if r != p:
        data_combinations = combinations(ar, r=r)
        alphabet_size_combinations = (x for x in combinations(ks, r=r))
        return (iinfo(*args)
                for args in zip(data_combinations, alphabet_size_combinations))

    return iinfo(ar, ks)

def coinformation(ar, ks=None, r=0):
    """Coinformation from p-by-n data matrix.

    If p == 2, return an estimate of the mutual information between the
    variables corresponding to the two columns.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    r : int, optional
        If r > 0, return a generator yielding estimates for the p-choose-r
        possible combinations of length r from the p variables.
        If r == 1, return the entropy for each variable. If r == 2 return the
        mutual information for each possible pair. If r > 2 return the
        interaction information for each possible subset of length r.
        Combinations are ordered as: list(itertools.combinations(range(p), r)).

    Returns
    -------
    float
        Coinformation estimate.

    """

    # change sign for odd #variables
    return (-1)**ar.shape[0] * interaction_information(ar=ar, ks=ks, r=r)

def mutual_information(ar, ks=None, axis=1):
    """Mutual information from p-by-n data matrix.

    If p > 2, return an estimate of the mutual information for each possible
    pair of variables, ordered as list(itertools.combinations(range(p), r=2)).

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis

    Returns
    -------
    float
        Coinformation estimate.

    """

    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim > 0:  # pylint: disable=comparison-with-callable
            if len(ks) != p:
                raise CardinalityError('k should have len %r (%r)' %
                                       (p, len(ks)))
        else:
            raise CardinalityError('ks cant be a scalar')

    if p > 2:
        h1 = list(from_data(ar, ks=ks, r=1))
        return (h1[i1] + h1[i2] - from_data(ar[[i1, i2]], ks=ks[[i1, i2]])
                for i1, i2 in combinations(range(p), 2))

    return numpy.sum(from_data(ar, ks=ks, r=1)) - from_data(ar, ks=ks)

def conditional_entropy(ar, c, ks=None, axis=1, r=0):
    """
    Coditional entropy estimate from data array.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    c : int or array-like
        The variables on which entropy is conditioned (as column indices).
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis
    r : int, optional
        If r > 0, return a generator yielding estimates for all possible
        combinations of r variables conditioning on the `c` variables.
        Indices are sorted as:
        list(x for x in collections.combinations(range(p), r=r+len(c))
             if set(c) <= set(x))

    Returns
    -------
    float
        Conditional entropy estimate

    Raises
    ------
    CardinalityError
        If ks is array-like and len(ks) != p
        If r > 0 and is a scalar.

    """
    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]

    try:
        c = list(c)
    except TypeError:
        c = [c]
    if not set(c) <= set(range(p)):
        return EstimatorInputError('The indices of conditioning variables'
                                   ' are not valid')

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim:
            if len(ks) != p:
                raise CardinalityError('k should have len %s' % p)

    # EntropyBasedEstimator objects are callable and return the fitted estimate
    estimator = Entropy()

    # Entropy of features on which we are conditioning
    counts = histogram(ar[c])
    hc = estimator(counts, k=ks)

    if r > 0:
        if ks.ndim == 0:
            raise CardinalityError('For combinations, ks cant be a scalar')

        r = r + len(c)
        indices = combinations(range(p), r=r)
        counts_combinations = histogram(ar, r=r)
        alphabet_size_combinations = (numpy.prod(x)
                                      for x in combinations(ks, r=r))
        return (estimator(*args) - hc for ids, *args in zip(
            indices, counts_combinations, alphabet_size_combinations)
                if set(c) <= set(ids))

    counts = histogram(ar)
    return estimator(counts, k=ks) - hc

def _check_input_data(ar):
    """Check that input arrays are non-empty 2D arrays."""

    ar = numpy.atleast_2d(ar)
    if ar.ndim > 2:
        raise DataArrayError('Input array has %s dimensions; must be 2D' %
                             ar.ndim)
    p, n = ar.shape
    if n == 0 or p == 0:
        raise DataArrayError('Empty input array')
    return ar
<commit_msg>from_data: set default r value to None<commit_after># -*- coding: utf-8 -*-
# Author: Simone Marsili <simomarsili@gmail.com>
# License: BSD 3 clause
"""Functions module."""
import logging

import numpy

import ndd
from ndd.estimators import Entropy, JSDivergence
from ndd.exceptions import (CardinalityError, DataArrayError,
                            EstimatorInputError, HistogramError, NumericError)

__all__ = [
    'entropy', 'jensen_shannon_divergence', 'interaction_information',
    'coinformation', 'mutual_information', 'conditional_entropy', 'histogram',
    'from_data'
]

logger = logging.getLogger(__name__)

def entropy(pk, k=None, alpha=None, plugin=False, return_std=False):
    """
    Return a Bayesian estimate of the entropy from an array of counts.

    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from an input array of counts pk.

    Parameters
    ----------
    pk : array-like
        The number of occurrences of a set of bins.
    k : int or array-like, optional
        Total number of bins (taking into account unobserved bins);
        k >= len(pk). A float is a valid input for whole numbers (e.g. k=1.e3).
        If an array, set k = numpy.prod(k).
        Defaults to n_bins.
    alpha : float, optional
        If alpha is not None, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.
    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).
    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    Returns
    -------
    entropy : float
        Entropy estimate.
    std : float, optional
        Uncertainty in the entropy estimate
        (approximated standard deviation over the entropy posterior).
        Only if `return_std` is True.

    Raises
    ------
    NumericError
        If result is NaN

    """

    # pk is an array of counts
    estimator = Entropy(alpha, plugin).fit(pk, k)
    S, err = estimator.estimate_, estimator.err_

    if numpy.isnan(S):
        raise NumericError('NaN value')

    if return_std:
        if err is not None and numpy.isnan(err):
            err = numpy.nan
            logger.warning('nan value for entropy posterior std deviation')
        return S, err

    return S

def jensen_shannon_divergence(pk, k=None, alpha=None, plugin=False):
    """
    Return the Jensen-Shannon divergence from a matrix of counts.

    Return an estimate of the Jensen-Shannon divergence between
    n_distributions unknown discrete distributions from a
    n_distributions-by-n_bins input array of counts.
    The estimate (in nats) is computed as a combination of single Bayesian
    entropy estimates. If the total number of samples varies among the
    distributions, the function returns the divergence between the
    distributions with weights proportional to the total number of samples in
    each row (see the general definition of Jensen-Shannon divergence:
    https://en.wikipedia.org/wiki/Jensen-Shannon_divergence).

    Parameters
    ----------

    pk : array-like, shape (n_distributions, n_bins)
        Matrix of frequency counts. Each row corresponds to the number of
        occurrences of a set of bins from a different distribution.
    k : int or array-like, optional
        Total number of bins (taking into account unobserved bins); k >= n_bins
        A float is a valid input for whole numbers (e.g. k=1.e3).
        If an array, set k = numpy.prod(k).
        Defaults to n_bins.
    alpha : float, optional
        If not None, the entropy estimator uses a single Dirichlet prior with
        concentration parameter alpha (fixed alpha estimator). alpha > 0.0.
    plugin : boolean, optional
        If True, use a 'plugin' estimator for the entropy.
        If alpha is passed in combination with plugin == True, add alpha
        pseudoconts to the frequency counts in the plugin estimate.

    Returns
    -------
    float
        Jensen-Shannon divergence.

    Raises
    ------
    NumericError
        If result is NaN

    """

    estimator = JSDivergence(alpha, plugin).fit(pk, k)
    js = estimator.estimate_

    if numpy.isnan(js):
        raise NumericError('NaN value')

    return js

def _nbins(data):
    """
    The number of unique elements along axis 0. If data is p-dimensional,
    the num. of unique elements for each variable.
    """
    # reshape as a p-by-n array
    return [len(numpy.unique(v)) for v in data]

def histogram(data, axis=1, r=None):
    """Compute an histogram from a data array. Wrapper to numpy.unique.

    Parameters
    ----------
    data : array-like
        A n-by-p array of n samples from p variables.
    axis : int, optional
        The sample-indexing axis
    r : int or None, optional
        For r values in the interval [1, p],
        return a generator yielding bin counts for the p-choose-r
        possible combinations of length r from the p variables.

    Returns
    -------
    counts : ndarray
        Bin counts.

    """
    from itertools import combinations

    # check data shape
    data = _check_input_data(data)
    if axis == 0:
        data = data.T
    p = data.shape[0]

    if r is not None:
        if r < 1 or r > p:
            raise HistogramError('r values must be in the interval [1, p]')
        return (ndd.histogram(d) for d in combinations(data, r=r))

    # statistics for the p-dimensional variable
    _, counts = numpy.unique(data, return_counts=True, axis=1)
    return counts

def from_data(ar, ks=None, axis=1, r=None):
    """
    Given an array of data, return an entropy estimate.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis
    r : int or None, optional
        For r values in the interval [1, p],
        return a generator yielding estimates for the p-choose-r
        possible combinations of length r from the p variables.

    Returns
    -------
    float
        Entropy estimate

    Raises
    ------
    CardinalityError
        If ks is array-like and len(ks) != p
        If r > 0 and is a scalar.

    """
    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]

    # EntropyBasedEstimator objects are callable and return the fitted estimate
    estimator = Entropy()

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim:
            if len(ks) != p:
                raise CardinalityError('k should have len %s' % p)

    if r is not None:
        if ks.ndim == 0:
            raise CardinalityError('For combinations, ks cant be a scalar')
        if r < 1 or r > p:
            raise HistogramError('r values must be in the interval [1, p]')

        counts_combinations = histogram(ar, r=r)
        alphabet_size_combinations = (numpy.prod(x)
                                      for x in combinations(ks, r=r))
        return (
            estimator(*args)
            for args in zip(counts_combinations, alphabet_size_combinations))

    counts = histogram(ar)
    return estimator(counts, k=ks)

def interaction_information(ar, ks=None, axis=1, r=0):
    """Interaction information from p-by-n data matrix.

    If p == 2, return an estimate of the mutual information between the
    variables corresponding to the two columns.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis
    r : int, optional
        If r > 0, return a generator yielding estimates for the p-choose-r
        possible combinations of length r from the p variables.
        If r == 1, return the entropy for each variable. If r == 2 return the
        mutual information for each possible pair. If r > 2 return the
        interaction information for each possible subset of length r.
        Combinations are ordered as: list(itertools.combinations(range(p), r)).

    Returns
    -------
    float
        Interaction information estimate.

    """
    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]
    if r == 0:
        r = p

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim > 0:  # pylint: disable=comparison-with-callable
            if len(ks) != p:
                raise CardinalityError('k should have len %r (%r)' %
                                       (p, len(ks)))
        else:
            raise CardinalityError('ks cant be a scalar')

    def iinfo(X, ks):
        info = 0.0
        px = len(X)
        for ri in range(1, px + 1):
            sgn = (-1)**(px - ri)
            info -= sgn * numpy.sum(from_data(X, ks=ks, r=ri))
        return info

    if r != p:
        data_combinations = combinations(ar, r=r)
        alphabet_size_combinations = (x for x in combinations(ks, r=r))
        return (iinfo(*args)
                for args in zip(data_combinations, alphabet_size_combinations))

    return iinfo(ar, ks)

def coinformation(ar, ks=None, r=0):
    """Coinformation from p-by-n data matrix.

    If p == 2, return an estimate of the mutual information between the
    variables corresponding to the two columns.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    r : int, optional
        If r > 0, return a generator yielding estimates for the p-choose-r
        possible combinations of length r from the p variables.
        If r == 1, return the entropy for each variable. If r == 2 return the
        mutual information for each possible pair. If r > 2 return the
        interaction information for each possible subset of length r.
        Combinations are ordered as: list(itertools.combinations(range(p), r)).

    Returns
    -------
    float
        Coinformation estimate.

    """

    # change sign for odd #variables
    return (-1)**ar.shape[0] * interaction_information(ar=ar, ks=ks, r=r)

def mutual_information(ar, ks=None, axis=1):
    """Mutual information from p-by-n data matrix.

    If p > 2, return an estimate of the mutual information for each possible
    pair of variables, ordered as list(itertools.combinations(range(p), r=2)).

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis

    Returns
    -------
    float
        Coinformation estimate.

    """

    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim > 0:  # pylint: disable=comparison-with-callable
            if len(ks) != p:
                raise CardinalityError('k should have len %r (%r)' %
                                       (p, len(ks)))
        else:
            raise CardinalityError('ks cant be a scalar')

    if p > 2:
        h1 = list(from_data(ar, ks=ks, r=1))
        return (h1[i1] + h1[i2] - from_data(ar[[i1, i2]], ks=ks[[i1, i2]])
                for i1, i2 in combinations(range(p), 2))

    return numpy.sum(from_data(ar, ks=ks, r=1)) - from_data(ar, ks=ks)

def conditional_entropy(ar, c, ks=None, axis=1, r=0):
    """
    Coditional entropy estimate from data array.

    Paramaters
    ----------
    ar : array-like
        p-by-n array of n samples from p discrete variables.
    c : int or array-like
        The variables on which entropy is conditioned (as column indices).
    ks : 1D p-dimensional array, optional
        Alphabet size for each variable.
    axis : int, optional
        The sample-indexing axis
    r : int, optional
        If r > 0, return a generator yielding estimates for all possible
        combinations of r variables conditioning on the `c` variables.
        Indices are sorted as:
        list(x for x in collections.combinations(range(p), r=r+len(c))
             if set(c) <= set(x))

    Returns
    -------
    float
        Conditional entropy estimate

    Raises
    ------
    CardinalityError
        If ks is array-like and len(ks) != p
        If r > 0 and is a scalar.

    """
    from itertools import combinations

    # check data shape
    ar = _check_input_data(ar)
    if axis == 0:
        ar = ar.T
    p = ar.shape[0]

    try:
        c = list(c)
    except TypeError:
        c = [c]
    if not set(c) <= set(range(p)):
        return EstimatorInputError('The indices of conditioning variables'
                                   ' are not valid')

    if ks is None:
        ks = numpy.array([len(numpy.unique(v)) for v in ar])
    else:
        try:
            ks = numpy.float64(ks)
        except ValueError:
            raise CardinalityError('%s: not a valid cardinality')
        if ks.ndim:
            if len(ks) != p:
                raise CardinalityError('k should have len %s' % p)

    # EntropyBasedEstimator objects are callable and return the fitted estimate
    estimator = Entropy()

    # Entropy of features on which we are conditioning
    counts = histogram(ar[c])
    hc = estimator(counts, k=ks)

    if r > 0:
        if ks.ndim == 0:
            raise CardinalityError('For combinations, ks cant be a scalar')

        r = r + len(c)
        indices = combinations(range(p), r=r)
        counts_combinations = histogram(ar, r=r)
        alphabet_size_combinations = (numpy.prod(x)
                                      for x in combinations(ks, r=r))
        return (estimator(*args) - hc for ids, *args in zip(
            indices, counts_combinations, alphabet_size_combinations)
                if set(c) <= set(ids))

    counts = histogram(ar)
    return estimator(counts, k=ks) - hc

def _check_input_data(ar):
    """Check that input arrays are non-empty 2D arrays."""

    ar = numpy.atleast_2d(ar)
    if ar.ndim > 2:
        raise DataArrayError('Input array has %s dimensions; must be 2D' %
                             ar.ndim)
    p, n = ar.shape
    if n == 0 or p == 0:
        raise DataArrayError('Empty input array')
    return ar
<|endoftext|>


<commit_before>import pydicom
import os
import numpy as np
import itertools
import glob
import argparse
import json

def normalize(x):
  return x / np.sqrt(np.dot(x,x))

def readRotAndTrans(paths):
  files = list(itertools.chain.from_iterable([glob.glob(path) for path in paths]))

  head = [(np.array([1,0,0,0]),np.array([0,0,0]))]
  if len(files) == 1 and files[0].endswith('.json'):
    with open(files[0]) as f:
      imageComments = [ x.split() for x in json.load(f)['vnavNumbers'][1:] if x is not None ]
  else:
    ds = sorted([pydicom.dcmread(x) for x in files], key=lambda dcm: dcm.AcquisitionNumber)
    imageComments = [ str.split(x.ImageComments) for x in ds[1:] if 'ImageComments' in x ]

  return list(itertools.chain.from_iterable([head, [ (np.array(list(map(float, y[1:5]))), list(map(float, y[6:9]))) for y in imageComments] ]))

def angleAxisToQuaternion(a):
  w = np.cos(a[0] / 2.0)
  axisNorm = np.sqrt(np.dot(a[1:], a[1:]))

  if 0 == axisNorm :
    return np.array([1,0,0,0])

  axisScale = np.sin(a[0] / 2.0) / np.sqrt(np.dot(a[1:], a[1:]))
  tail = a[1:] * axisScale
  q = np.ndarray(shape=(4))
  q[0] = w
  q[1:] = tail
  return q

def quaternionToAxisAngle(q) :
  a = np.ndarray(shape=(4))
  a[0] = np.arccos(q[0]) * 2.0
  a[1:] = q[1:] / np.sqrt(np.dot(q[1:], q[1:]))
  return a

def quaternionToRotationMatrix(q):
  w = q[0];
  x = q[1];
  y = q[2];
  z = q[3];

  wSq = w * w;
  xSq = x * x;
  ySq = y * y;
  zSq = z * z;

  wx2 = w*x*2.0;
  wy2 = w*y*2.0;
  wz2 = w*z*2.0;

  xy2 = x*y*2.0;
  xz2 = x*z*2.0;

  yz2 = y*z*2.0;

  rotMatrix = np.ndarray(shape=(3,3))

  rotMatrix[0,0] = wSq + xSq - ySq - zSq;
  rotMatrix[0,1] = xy2 - wz2;
  rotMatrix[0,2] = xz2 + wy2;

  rotMatrix[1,0] = xy2 + wz2;
  rotMatrix[1,1] = wSq - xSq + ySq - zSq;
  rotMatrix[1,2] = yz2 - wx2;

  rotMatrix[2,0] = xz2 - wy2;
  rotMatrix[2,1] = yz2 + wx2;
  rotMatrix[2,2] = wSq - xSq - ySq + zSq;

  return rotMatrix

def rotationMatrixToQuaternion(m):
  ## Dylan Dec 19/12
  ## This algorithm taken from http://www.euclideanspace.com/maths/geometry/rotations/conversions/matrixToQuaternion/index.htm
  ##
  ## Also tried the algorithm in
  ##		Animating rotation with quaternion curves.
  ##		Ken Shoemake
  ##		Computer Graphics 19(3):245-254,  1985
  ##		http://portal.acm.org/citation.cfm?doid=325334.325242
  ## but you'll find that it's not numerically stable without some truncation epsilon.
  ## The algorithm we're using now doesn't require us to pick some arbitrary epsilon, so
  ## I like it better.

  tr = np.trace(m);

  if (tr > 0) :
    S = np.sqrt(tr+1.0) * 2; # S=4*qw
    SInv = 1.0 / S;
    w = 0.25 * S;
    x = (m[2,1] - m[1,2]) * SInv;
    y = (m[0,2] - m[2,0]) * SInv;
    z = (m[1,0] - m[0,1]) * SInv;
  elif ((m[0,0] > m[1,1]) and (m[0,0] > m[2,2])):
    S = np.sqrt(1.0 + m[0,0] - m[1,1] - m[2,2]) * 2; # S=4*qx
    SInv = 1.0 / S;
    w = (m[2,1] - m[1,2]) * SInv;
    x = 0.25 * S;
    y = (m[0,1] + m[1,0]) * SInv;
    z = (m[0,2] + m[2,0]) * SInv
  elif (m[1,1] > m[2,2]):
    S = np.sqrt(1.0 + m[1,1] - m[0,0] - m[2,2]) * 2; # S=4*qy
    SInv = 1.0 / S;
    w = (m[0,2] - m[2,0]) * SInv;
    x = (m[0,1] + m[1,0]) * SInv;
    y = 0.25 * S;
    z = (m[1,2] + m[2,1]) * SInv;
  else:
    S = np.sqrt(1.0 + m[2,2] - m[0,0] - m[1,1]) * 2; # S=4*qz
    SInv = 1.0 / S;
    w = (m[1,0] - m[0,1]) * SInv;
    x = (m[0,2] + m[2,0]) * SInv;
    y = (m[1,2] + m[2,1]) * SInv;
    z = 0.25 * S;

  return np.array([w, x, y, z])

def motionEntryToHomogeneousTransform(e) :
  t = np.ndarray(shape=(4,4))

  t[0:3,3] = e[1]
  t[0:3,0:3] = quaternionToRotationMatrix(angleAxisToQuaternion(e[0]))
  t[3,:] = [0,0,0,1]
  return np.matrix(t)

def diffTransformToMaxMotion(t, radius):
  angleAxis = quaternionToAxisAngle(rotationMatrixToQuaternion(t[0:3, 0:3]))
  angle = angleAxis[0]
  axis = angleAxis[1:]
  trans = t[0:3,3].flatten()
  t_rotmax = radius * np.sqrt(2.0 - 2.0 * np.cos(angle))
  return np.sqrt(
    (t_rotmax * t_rotmax) +
    (2.0 * t_rotmax) *
      np.linalg.norm(
        trans - (np.dot(trans, axis) * axis)) +
    (np.linalg.norm(trans) * np.linalg.norm(trans))
    )

def diffTransformToRMSMotion(t, radius):
  rotMatMinusIdentity = t[0:3,0:3] - np.array([[1,0,0],[0,1,0],[0,0,1]])
  trans = np.ravel(t[0:3,3])

  return np.sqrt(
    0.2 * radius * radius * np.trace(np.transpose(rotMatMinusIdentity) * rotMatMinusIdentity) +
    np.dot(trans, trans)
    )

def parseMotion(input, tr, radius):
    # Transform creation and differences
    transforms = [motionEntryToHomogeneousTransform(e) for e in readRotAndTrans(input)]
    diffTransforms = [ts[1] * np.linalg.inv(ts[0]) for ts in zip(transforms[0:], transforms[1:])]

    # Motion scores
    rmsMotionScores = [diffTransformToRMSMotion(t, radius) for t in diffTransforms]
    maxMotionScores = [diffTransformToMaxMotion(t, radius) for t in diffTransforms]

    scores = {}
    scores['mean_rms'] = np.mean(rmsMotionScores) * 60.0 / tr
    scores['mean_max'] = np.mean(maxMotionScores) * 60.0 / tr
    scores['rms_scores'] = rmsMotionScores
    scores['max_scores'] = maxMotionScores

    return scores

if __name__ == '__main__':
  parser = argparse.ArgumentParser(description='Parse DICOM files from a vNav series and convert them into different motion scores.')

  parser.add_argument('--tr', required=True, type=float,
                      help='Repetition Time (TR) of the parent sequence (i.e., the MPRAGE) expressed in seconds.')
  parser.add_argument('--input', nargs='+', required=True, type=os.path.abspath,
                      help='A list of DICOM files that make up the vNav series (in chronological order), or a BIDS JSON sidecar containing extracted motion strings')
  parser.add_argument('--radius', required=True, type=float,
                      help='Assumed brain radius in millimeters for estimating rotation distance.')
  output_type = parser.add_mutually_exclusive_group(required=True)
  output_type.add_argument('--mean-rms', action='store_true', help='Print time-averaged root mean square (RMS) motion.')
  output_type.add_argument('--mean-max', action='store_true', help='Print time-averaged max motion.')
  output_type.add_argument('--rms-scores', action='store_true', help='Print root mean square (RMS) motion over time.')
  output_type.add_argument('--max-scores', action='store_true', help='Print max motion over time.')

  args = parser.parse_args()

  scores = parseMotion(args.input, args.tr, args.radius)

  # Script output to STDOUT depending on "output_type"
  if args.mean_rms:
    print(scores['mean_rms'])
  elif args.mean_max:
    print(scores['mean_max'])
  elif args.rms_scores:
    print('\n'.join(map(str, scores['rms_scores'])))
  elif args.max_scores:
    print('\n'.join(map(str, scores['max_scores'])))
<commit_msg>refactored input of already parsed DICOM headers<commit_after>import pydicom
import os
import numpy as np
import itertools
import glob
import argparse
import json

def normalize(x):
  return x / np.sqrt(np.dot(x,x))

def readRotAndTrans(paths):
  files = list(itertools.chain.from_iterable([glob.glob(path) for path in paths]))

  head = [(np.array([1,0,0,0]),np.array([0,0,0]))]
  ds = sorted([pydicom.dcmread(x) for x in files], key=lambda dcm: dcm.AcquisitionNumber)
  imageComments = [ str.split(x.ImageComments) for x in ds[1:] if 'ImageComments' in x ]

  return list(itertools.chain.from_iterable([head, [ (np.array(list(map(float, y[1:5]))), list(map(float, y[6:9]))) for y in imageComments] ]))

def angleAxisToQuaternion(a):
  w = np.cos(a[0] / 2.0)
  axisNorm = np.sqrt(np.dot(a[1:], a[1:]))

  if 0 == axisNorm :
    return np.array([1,0,0,0])

  axisScale = np.sin(a[0] / 2.0) / np.sqrt(np.dot(a[1:], a[1:]))
  tail = a[1:] * axisScale
  q = np.ndarray(shape=(4))
  q[0] = w
  q[1:] = tail
  return q

def quaternionToAxisAngle(q) :
  a = np.ndarray(shape=(4))
  a[0] = np.arccos(q[0]) * 2.0
  a[1:] = q[1:] / np.sqrt(np.dot(q[1:], q[1:]))
  return a

def quaternionToRotationMatrix(q):
  w = q[0];
  x = q[1];
  y = q[2];
  z = q[3];

  wSq = w * w;
  xSq = x * x;
  ySq = y * y;
  zSq = z * z;

  wx2 = w*x*2.0;
  wy2 = w*y*2.0;
  wz2 = w*z*2.0;

  xy2 = x*y*2.0;
  xz2 = x*z*2.0;

  yz2 = y*z*2.0;

  rotMatrix = np.ndarray(shape=(3,3))

  rotMatrix[0,0] = wSq + xSq - ySq - zSq;
  rotMatrix[0,1] = xy2 - wz2;
  rotMatrix[0,2] = xz2 + wy2;

  rotMatrix[1,0] = xy2 + wz2;
  rotMatrix[1,1] = wSq - xSq + ySq - zSq;
  rotMatrix[1,2] = yz2 - wx2;

  rotMatrix[2,0] = xz2 - wy2;
  rotMatrix[2,1] = yz2 + wx2;
  rotMatrix[2,2] = wSq - xSq - ySq + zSq;

  return rotMatrix

def rotationMatrixToQuaternion(m):
  ## Dylan Dec 19/12
  ## This algorithm taken from http://www.euclideanspace.com/maths/geometry/rotations/conversions/matrixToQuaternion/index.htm
  ##
  ## Also tried the algorithm in
  ##		Animating rotation with quaternion curves.
  ##		Ken Shoemake
  ##		Computer Graphics 19(3):245-254,  1985
  ##		http://portal.acm.org/citation.cfm?doid=325334.325242
  ## but you'll find that it's not numerically stable without some truncation epsilon.
  ## The algorithm we're using now doesn't require us to pick some arbitrary epsilon, so
  ## I like it better.

  tr = np.trace(m);

  if (tr > 0) :
    S = np.sqrt(tr+1.0) * 2; # S=4*qw
    SInv = 1.0 / S;
    w = 0.25 * S;
    x = (m[2,1] - m[1,2]) * SInv;
    y = (m[0,2] - m[2,0]) * SInv;
    z = (m[1,0] - m[0,1]) * SInv;
  elif ((m[0,0] > m[1,1]) and (m[0,0] > m[2,2])):
    S = np.sqrt(1.0 + m[0,0] - m[1,1] - m[2,2]) * 2; # S=4*qx
    SInv = 1.0 / S;
    w = (m[2,1] - m[1,2]) * SInv;
    x = 0.25 * S;
    y = (m[0,1] + m[1,0]) * SInv;
    z = (m[0,2] + m[2,0]) * SInv
  elif (m[1,1] > m[2,2]):
    S = np.sqrt(1.0 + m[1,1] - m[0,0] - m[2,2]) * 2; # S=4*qy
    SInv = 1.0 / S;
    w = (m[0,2] - m[2,0]) * SInv;
    x = (m[0,1] + m[1,0]) * SInv;
    y = 0.25 * S;
    z = (m[1,2] + m[2,1]) * SInv;
  else:
    S = np.sqrt(1.0 + m[2,2] - m[0,0] - m[1,1]) * 2; # S=4*qz
    SInv = 1.0 / S;
    w = (m[1,0] - m[0,1]) * SInv;
    x = (m[0,2] + m[2,0]) * SInv;
    y = (m[1,2] + m[2,1]) * SInv;
    z = 0.25 * S;

  return np.array([w, x, y, z])

def motionEntryToHomogeneousTransform(e) :
  t = np.ndarray(shape=(4,4))

  t[0:3,3] = e[1]
  t[0:3,0:3] = quaternionToRotationMatrix(angleAxisToQuaternion(e[0]))
  t[3,:] = [0,0,0,1]
  return np.matrix(t)

def diffTransformToMaxMotion(t, radius):
  angleAxis = quaternionToAxisAngle(rotationMatrixToQuaternion(t[0:3, 0:3]))
  angle = angleAxis[0]
  axis = angleAxis[1:]
  trans = t[0:3,3].flatten()
  t_rotmax = radius * np.sqrt(2.0 - 2.0 * np.cos(angle))
  return np.sqrt(
    (t_rotmax * t_rotmax) +
    (2.0 * t_rotmax) *
      np.linalg.norm(
        trans - (np.dot(trans, axis) * axis)) +
    (np.linalg.norm(trans) * np.linalg.norm(trans))
    )

def diffTransformToRMSMotion(t, radius):
  rotMatMinusIdentity = t[0:3,0:3] - np.array([[1,0,0],[0,1,0],[0,0,1]])
  trans = np.ravel(t[0:3,3])

  return np.sqrt(
    0.2 * radius * radius * np.trace(np.transpose(rotMatMinusIdentity) * rotMatMinusIdentity) +
    np.dot(trans, trans)
    )

def parseMotion(rotAndTrans, tr, radius):
    # Transform creation and differences
    transforms = [motionEntryToHomogeneousTransform(e) for e in rotAndTrans]
    diffTransforms = [ts[1] * np.linalg.inv(ts[0]) for ts in zip(transforms[0:], transforms[1:])]

    # Motion scores
    rmsMotionScores = [diffTransformToRMSMotion(t, radius) for t in diffTransforms]
    maxMotionScores = [diffTransformToMaxMotion(t, radius) for t in diffTransforms]

    scores = {}
    scores['mean_rms'] = np.mean(rmsMotionScores) * 60.0 / tr
    scores['mean_max'] = np.mean(maxMotionScores) * 60.0 / tr
    scores['rms_scores'] = rmsMotionScores
    scores['max_scores'] = maxMotionScores

    return scores

if __name__ == '__main__':
  parser = argparse.ArgumentParser(description='Parse DICOM files from a vNav series and convert them into different motion scores.')

  parser.add_argument('--tr', required=True, type=float,
                      help='Repetition Time (TR) of the parent sequence (i.e., the MPRAGE) expressed in seconds.')
  parser.add_argument('--input', nargs='+', required=True, type=os.path.abspath,
                      help='A list of DICOM files that make up the vNav series (in chronological order)')
  parser.add_argument('--radius', required=True, type=float,
                      help='Assumed brain radius in millimeters for estimating rotation distance.')
  output_type = parser.add_mutually_exclusive_group(required=True)
  output_type.add_argument('--mean-rms', action='store_true', help='Print time-averaged root mean square (RMS) motion.')
  output_type.add_argument('--mean-max', action='store_true', help='Print time-averaged max motion.')
  output_type.add_argument('--rms-scores', action='store_true', help='Print root mean square (RMS) motion over time.')
  output_type.add_argument('--max-scores', action='store_true', help='Print max motion over time.')

  args = parser.parse_args()

  scores = parseMotion(readRotAndTrans(args.input), args.tr, args.radius)

  # Script output to STDOUT depending on "output_type"
  if args.mean_rms:
    print(scores['mean_rms'])
  elif args.mean_max:
    print(scores['mean_max'])
  elif args.rms_scores:
    print('\n'.join(map(str, scores['rms_scores'])))
  elif args.max_scores:
    print('\n'.join(map(str, scores['max_scores'])))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2016,2017 Simone Marsili
# All rights reserved.
# License: BSD 3 clause
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)

__copyright__ = "Copyright (C) 2016,2017 Simone Marsili"
__license__ = "BSD 3 clause"
__author__ = "Simone Marsili (simomarsili@gmail.com)"
__all__ = ['entropy', 'histogram']

import numpy
import ndd
import ndd._nsb

class BaseEstimator(object):
    def __init__(self):
        self.estimate = None
        self.std = None

    def _check(self):
        # check input data
        raise NotImplementedError

    def fit(self):
        # set self.estimate, self.std
        raise NotImplementedError

    def __call__(self, *args, **kwargs):
        """Return estimate from input data. Delegate to fit."""
        self.fit(*args, **kwargs)
        return self.estimate

class Entropy(BaseEstimator):
    def __init__(self, alpha=None, plugin=False):
        super().__init__()

        # check alpha value
        if alpha:
            try:
                alpha = numpy.float64(alpha)
            except ValueError:
                raise
            if alpha <= 0:
                raise ValueError("alpha <= 0")
        self.alpha = alpha

        # set estimator
        if plugin:
            if alpha is None:
                self.estimator = self._plugin
            else:
                self.estimator = lambda counts, k: self._pseudocounts(
                    counts, k, self.alpha)
        else:
            if alpha is None:
                self.estimator = self._nsb
            else:
                self.estimator = lambda counts, k: self._ww(
                    counts, k, self.alpha)

    @staticmethod
    def _plugin(counts, k):
        return ndd._nsb.plugin(counts, k)

    @staticmethod
    def _pseudocounts(counts, k, alpha):
        return ndd._nsb.pseudo(counts, k, alpha)

    @staticmethod
    def _ww(counts, k, alpha):
        return ndd._nsb.dirichlet(counts, k, alpha)

    @staticmethod
    def _nsb(counts, k):
        return ndd._nsb.nsb(counts, k)

    def _check(self, counts, k):
        counts = self._check_counts(a=counts)
        k = self._check_k(k=k, n_bins=len(counts))
        return counts, k

    @staticmethod
    def _check_counts(a):
        try:
            a = numpy.asarray(a, dtype=numpy.int32)
        except ValueError:
            raise
        if numpy.any(a < 0):
            raise ValueError("Frequency counts can't be negative")
        return a.flatten()

    @staticmethod
    def _check_k(k, n_bins):
        """
        if k is None, set k = number of bins
        if k is an integer, just check
        ik an array set k = prod(k)
        """
        MAX_LOGK = 150 * numpy.log(2)

        if k is None:
            # set k to the number of observed bins
            k = numpy.float64(n_bins)
        else:
            try:
                k = numpy.float64(k)
            except ValueError:
                raise
            if k.ndim:
                # if k is a sequence, set k = prod(k)
                if k.ndim > 1:
                    raise ValueError('k must be a scalar or 1D array')
                logk = numpy.sum(numpy.log(x) for x in k)
                if logk > MAX_LOGK:
                    # too large a number; backoff to n_bins?
                    # TODO: log warning
                    raise ValueError('k (%r) larger than %r' %
                                     (numpy.exp(logk), numpy.exp(MAX_LOGK)))
                else:
                    k = numpy.prod(k)
            else:
                # if a scalar check size
                if numpy.log(k) > MAX_LOGK:
                    raise ValueError('k (%r) larger than %r' %
                                     (k, numpy.exp(MAX_LOGK)))
            # consistency checks
            if k < n_bins:
                raise ValueError("k (%s) is smaller than the number of bins"
                                 "(%s)" % (k, n_bins))
            if not k.is_integer():
                raise ValueError("k (%s) should be a whole number." % k)
        return k

    def fit(self, counts, k=None):
        """
        counts : array_like
            The number of occurrences of a set of bins.

        k : int, optional
            Number of bins. k >= len(counts).
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to len(counts).

        """
        counts, k = self._check(counts, k)
        if k == 1:  # single bin
            self.estimate = self.std = 0.0
        else:
            result = self.estimator(counts, k)
            if isinstance(result, tuple):
                self.estimate, self.std = result
            else:
                self.estimate = result

def entropy(counts, k=None, alpha=None, return_std=False, plugin=False):
    """
    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from an input array of counts.

    Parameters
    ----------

    counts : array_like
        The number of occurrences of a set of bins.

    k : int or array-like, optional
        Number of bins. k >= len(counts).
        A float value is a valid input for whole numbers (e.g. k=1.e3).
        If an array, set k = numpy.prod(k).
        Defaults to len(counts).

    alpha : float, optional
        If alpha is passed, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.

    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).

    Returns
    -------
    entropy : float
        Entropy estimate.

    std : float, optional
        Uncertainty in the entropy estimate
        (approximates the standard deviation over the entropy posterior).
        Only provided if `return_std` is True.

    """

    estimator = Entropy(alpha, plugin)
    estimator.fit(counts, k)

    if return_std:
        result = estimator.estimate, estimator.std
    else:
        result = estimator.estimate

    if numpy.any(numpy.isnan(numpy.squeeze(result))):
        raise FloatingPointError("NaN value")

    return result

def nbins(data):
    """
    The number of unique elements along axis 0. If data is p-dimensional,
    the num. of unique elements for each variable.
    """
    # reshape as a p-by-n array
    data = ndd.nsb._2darray(data)
    return [len(numpy.unique(v)) for v in data]

def histogram(data, axis=0, r=0):
    """Compute an histogram from data. Wrapper to numpy.unique.

    Parameters
    ----------
    data : array-like
        An array of n samples from p variables.
    axis : int, optional
        The sample-indexing axis
    r : int, optional
        If r > 0, return a generator that yields bin counts for each possible
        combination of r variables.

    Returns
    -------
    counts : ndarray
        Bin counts.

    """
    from itertools import combinations
    # reshape as a p-by-n array
    data = ndd.nsb._2darray(data, axis=axis)
    p, n = data.shape
    if r > p:
        raise ValueError('r (%r) is larger than the number of variables (%r)'
                         % (r, p))
    if r == 0:
        # statistics for the p-dimensional variable
        _, counts = numpy.unique(data, return_counts=True, axis=1)
        return counts
    else:
        return (ndd.histogram(d, axis=1) for d in combinations(data, r=r))

def _2darray(ar, axis=0):
    """
    For a 2D n-by-p data array, transpose it.
    For a generic ndarray, flatten the subarrays indexed by axis 0
    axis : int, optional
        The sample-indexing axis
    """

    ar = numpy.asanyarray(ar)

    if ar.ndim == 1:
        n = ar.shape[0]
        ar = ar.reshape(n, 1)
    elif ar.ndim == 2:
        if axis == 0:
            ar = ar.T
    elif ar.ndim > 2:
        if axis != 0:
            try:
                ar = numpy.swapaxes(ar, axis, 0)
            except ValueError:
                raise numpy.AxisError(axis, ar.ndim)
        n = ar.shape[0]
        ar = ar.reshape(n, -1)
        ar = ar.T

    return numpy.ascontiguousarray(ar)

def entropy_combinations(ar, ks, r=1):
    """
    Given a p-by-n array of data, return an entropy value for all possible
    p-choose-r combinations of r columns.

    Paramaters
    ----------

    ar : array-like
        p-by-n array of n samples from p discrete variables.

    ks : 1D p-dimensional array
        Alphabet size for each variable.

    r : int, optional
        For each possible combination of r columns, return the estimated
        entropy for the corresponding r-dimensional variable.
        See itertools.combinations(range(p), r=r).
        Defaults to 1 (a different estimate for each column/variable).

    Returns
    -------
    generator object
        Entropy estimates for all combinations.

    """
    from itertools import combinations

    p, n = ar.shape

    try:
        if len(ks) == p:
            ks = numpy.array(ks)
        else:
            raise ValueError("k should have len %s" % p)
    except TypeError:
        raise

    entropy_estimator = Entropy()
    counts_combinations = histogram(ar, axis=1, r=r)
    alphabet_size_combinations = (numpy.prod(x) for x in combinations(ks, r=r))
    for k, c in zip(counts_combinations, alphabet_size_combinations):
        yield entropy_estimator(c, k=k)
<commit_msg>rename _2darray to _as_data_array<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2016,2017 Simone Marsili
# All rights reserved.
# License: BSD 3 clause
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)

__copyright__ = "Copyright (C) 2016,2017 Simone Marsili"
__license__ = "BSD 3 clause"
__author__ = "Simone Marsili (simomarsili@gmail.com)"
__all__ = ['entropy', 'histogram']

import numpy
import ndd
import ndd._nsb

class BaseEstimator(object):
    def __init__(self):
        self.estimate = None
        self.std = None

    def _check(self):
        # check input data
        raise NotImplementedError

    def fit(self):
        # set self.estimate, self.std
        raise NotImplementedError

    def __call__(self, *args, **kwargs):
        """Return estimate from input data. Delegate to fit."""
        self.fit(*args, **kwargs)
        return self.estimate

class Entropy(BaseEstimator):
    def __init__(self, alpha=None, plugin=False):
        super().__init__()

        # check alpha value
        if alpha:
            try:
                alpha = numpy.float64(alpha)
            except ValueError:
                raise
            if alpha <= 0:
                raise ValueError("alpha <= 0")
        self.alpha = alpha

        # set estimator
        if plugin:
            if alpha is None:
                self.estimator = self._plugin
            else:
                self.estimator = lambda counts, k: self._pseudocounts(
                    counts, k, self.alpha)
        else:
            if alpha is None:
                self.estimator = self._nsb
            else:
                self.estimator = lambda counts, k: self._ww(
                    counts, k, self.alpha)

    @staticmethod
    def _plugin(counts, k):
        return ndd._nsb.plugin(counts, k)

    @staticmethod
    def _pseudocounts(counts, k, alpha):
        return ndd._nsb.pseudo(counts, k, alpha)

    @staticmethod
    def _ww(counts, k, alpha):
        return ndd._nsb.dirichlet(counts, k, alpha)

    @staticmethod
    def _nsb(counts, k):
        return ndd._nsb.nsb(counts, k)

    def _check(self, counts, k):
        counts = self._check_counts(a=counts)
        k = self._check_k(k=k, n_bins=len(counts))
        return counts, k

    @staticmethod
    def _check_counts(a):
        try:
            a = numpy.asarray(a, dtype=numpy.int32)
        except ValueError:
            raise
        if numpy.any(a < 0):
            raise ValueError("Frequency counts can't be negative")
        return a.flatten()

    @staticmethod
    def _check_k(k, n_bins):
        """
        if k is None, set k = number of bins
        if k is an integer, just check
        ik an array set k = prod(k)
        """
        MAX_LOGK = 150 * numpy.log(2)

        if k is None:
            # set k to the number of observed bins
            k = numpy.float64(n_bins)
        else:
            try:
                k = numpy.float64(k)
            except ValueError:
                raise
            if k.ndim:
                # if k is a sequence, set k = prod(k)
                if k.ndim > 1:
                    raise ValueError('k must be a scalar or 1D array')
                logk = numpy.sum(numpy.log(x) for x in k)
                if logk > MAX_LOGK:
                    # too large a number; backoff to n_bins?
                    # TODO: log warning
                    raise ValueError('k (%r) larger than %r' %
                                     (numpy.exp(logk), numpy.exp(MAX_LOGK)))
                else:
                    k = numpy.prod(k)
            else:
                # if a scalar check size
                if numpy.log(k) > MAX_LOGK:
                    raise ValueError('k (%r) larger than %r' %
                                     (k, numpy.exp(MAX_LOGK)))
            # consistency checks
            if k < n_bins:
                raise ValueError("k (%s) is smaller than the number of bins"
                                 "(%s)" % (k, n_bins))
            if not k.is_integer():
                raise ValueError("k (%s) should be a whole number." % k)
        return k

    def fit(self, counts, k=None):
        """
        counts : array_like
            The number of occurrences of a set of bins.

        k : int, optional
            Number of bins. k >= len(counts).
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to len(counts).

        """
        counts, k = self._check(counts, k)
        if k == 1:  # single bin
            self.estimate = self.std = 0.0
        else:
            result = self.estimator(counts, k)
            if isinstance(result, tuple):
                self.estimate, self.std = result
            else:
                self.estimate = result

def entropy(counts, k=None, alpha=None, return_std=False, plugin=False):
    """
    Return a Bayesian estimate of the entropy of an unknown discrete
    distribution from an input array of counts.

    Parameters
    ----------

    counts : array_like
        The number of occurrences of a set of bins.

    k : int or array-like, optional
        Number of bins. k >= len(counts).
        A float value is a valid input for whole numbers (e.g. k=1.e3).
        If an array, set k = numpy.prod(k).
        Defaults to len(counts).

    alpha : float, optional
        If alpha is passed, use a single Dirichlet prior with concentration
        parameter alpha (fixed alpha estimator). alpha > 0.0.

    return_std : boolean, optional
        If True, also return an approximated value for the standard deviation
        over the entropy posterior.

    plugin : boolean, optional
        If True, return a 'plugin' estimate of the entropy. The discrete
        distribution is estimated from the empirical frequencies over bins
        and inserted into the entropy definition (plugin estimator).
        If alpha is passed in combination with plugin=True, add
        alpha pseudocounts to each frequency count (pseudocount estimator).

    Returns
    -------
    entropy : float
        Entropy estimate.

    std : float, optional
        Uncertainty in the entropy estimate
        (approximates the standard deviation over the entropy posterior).
        Only provided if `return_std` is True.

    """

    estimator = Entropy(alpha, plugin)
    estimator.fit(counts, k)

    if return_std:
        result = estimator.estimate, estimator.std
    else:
        result = estimator.estimate

    if numpy.any(numpy.isnan(numpy.squeeze(result))):
        raise FloatingPointError("NaN value")

    return result

def nbins(data):
    """
    The number of unique elements along axis 0. If data is p-dimensional,
    the num. of unique elements for each variable.
    """
    # reshape as a p-by-n array
    data = ndd.nsb._as_data_array(data)
    return [len(numpy.unique(v)) for v in data]

def histogram(data, axis=0, r=0):
    """Compute an histogram from data. Wrapper to numpy.unique.

    Parameters
    ----------
    data : array-like
        An array of n samples from p variables.
    axis : int, optional
        The sample-indexing axis
    r : int, optional
        If r > 0, return a generator that yields bin counts for each possible
        combination of r variables.

    Returns
    -------
    counts : ndarray
        Bin counts.

    """
    from itertools import combinations
    # reshape as a p-by-n array
    data = ndd.nsb._as_data_array(data, axis=axis)
    p, n = data.shape
    if r > p:
        raise ValueError('r (%r) is larger than the number of variables (%r)'
                         % (r, p))
    if r == 0:
        # statistics for the p-dimensional variable
        _, counts = numpy.unique(data, return_counts=True, axis=1)
        return counts
    else:
        return (ndd.histogram(d, axis=1) for d in combinations(data, r=r))

def _as_data_array(ar, axis=0):
    """
    For a 2D n-by-p data array, transpose it.
    For a generic ndarray, flatten the subarrays indexed by axis 0
    axis : int, optional
        The sample-indexing axis
    """

    ar = numpy.asanyarray(ar)

    if ar.ndim == 1:
        n = ar.shape[0]
        ar = ar.reshape(n, 1)
    elif ar.ndim == 2:
        if axis == 0:
            ar = ar.T
    elif ar.ndim > 2:
        if axis != 0:
            try:
                ar = numpy.swapaxes(ar, axis, 0)
            except ValueError:
                raise numpy.AxisError(axis, ar.ndim)
        n = ar.shape[0]
        ar = ar.reshape(n, -1)
        ar = ar.T

    return numpy.ascontiguousarray(ar)

def entropy_combinations(ar, ks, r=1):
    """
    Given a p-by-n array of data, return an entropy value for all possible
    p-choose-r combinations of r columns.

    Paramaters
    ----------

    ar : array-like
        p-by-n array of n samples from p discrete variables.

    ks : 1D p-dimensional array
        Alphabet size for each variable.

    r : int, optional
        For each possible combination of r columns, return the estimated
        entropy for the corresponding r-dimensional variable.
        See itertools.combinations(range(p), r=r).
        Defaults to 1 (a different estimate for each column/variable).

    Returns
    -------
    generator object
        Entropy estimates for all combinations.

    """
    from itertools import combinations

    p, n = ar.shape

    try:
        if len(ks) == p:
            ks = numpy.array(ks)
        else:
            raise ValueError("k should have len %s" % p)
    except TypeError:
        raise

    entropy_estimator = Entropy()
    counts_combinations = histogram(ar, axis=1, r=r)
    alphabet_size_combinations = (numpy.prod(x) for x in combinations(ks, r=r))
    for k, c in zip(counts_combinations, alphabet_size_combinations):
        yield entropy_estimator(c, k=k)
<|endoftext|>