<commit_before>import json

import requests
from django.conf import settings
from django.db.models.signals import post_save, pre_delete
from django.dispatch import receiver

from karrot.conversations.models import Conversation
from karrot.groups import roles, stats
from karrot.groups.emails import prepare_user_became_editor_email
from karrot.groups.models import Group, GroupMembership, Trust
from karrot.history.models import History, HistoryTypus
from karrot.utils import frontend_urls

@receiver(post_save, sender=Group)
def group_created(sender, instance, created, **kwargs):
    """Ensure every group has a conversation."""
    if not created:
        return
    group = instance
    conversation = Conversation.objects.get_or_create_for_target(group)
    conversation.sync_users(group.members.all())

@receiver(pre_delete, sender=Group)
def group_deleted(sender, instance, **kwargs):
    """Delete the conversation when the group is deleted."""
    group = instance
    conversation = Conversation.objects.get_for_target(group)
    if conversation:
        conversation.delete()

@receiver(post_save, sender=GroupMembership)
def group_member_added(sender, instance, created, **kwargs):
    if not created:
        return
    group = instance.group
    user = instance.user
    membership = instance

    if group.is_playground():
        membership.notification_types = []
        membership.roles.append(roles.GROUP_EDITOR)
        membership.save()

    conversation = Conversation.objects.get_or_create_for_target(group)
    conversation.join(user, muted=group.is_playground())

    stats.group_joined(group)

@receiver(pre_delete, sender=GroupMembership)
def group_member_removed(sender, instance, **kwargs):
    group = instance.group
    user = instance.user
    conversation = Conversation.objects.get_for_target(group)
    if conversation:
        conversation.leave(user)

    stats.group_left(group)

@receiver(post_save, sender=Trust)
def trust_given(sender, instance, created, **kwargs):
    if not created:
        return

    membership = instance.membership
    relevant_trust = Trust.objects.filter(membership=membership)
    trust_threshold = membership.group.trust_threshold_for_newcomer()

    if relevant_trust.count() >= trust_threshold and roles.GROUP_EDITOR not in membership.roles:
        membership.add_roles([roles.GROUP_EDITOR])
        membership.save()

        History.objects.create(
            typus=HistoryTypus.MEMBER_BECAME_EDITOR,
            group=membership.group,
            users=[membership.user],
            payload={
                'threshold': trust_threshold,
            },
        )

        prepare_user_became_editor_email(user=membership.user, group=membership.group).send()

        stats.member_became_editor(membership.group)

    stats.trust_given(membership.group)

@receiver(pre_delete, sender=GroupMembership)
def remove_trust(sender, instance, **kwargs):
    membership = instance

    Trust.objects.filter(given_by=membership.user, membership__group=membership.group).delete()

@receiver(post_save, sender=Group)
def notify_chat_on_group_creation(sender, instance, created, **kwargs):
    """Send notifications to admin chat"""
    if not created:
        return
    group = instance
    webhook_url = getattr(settings, 'ADMIN_ROCKETCHAT_WEBHOOK', None)

    if webhook_url is None:
        return

    group_url = frontend_urls.group_preview_url(group)

    message_data = {
        'text': f':tada: A new group has been created on karrot.world! [Visit {group.name}]({group_url})',
    }

    response = requests.post(webhook_url, data=json.dumps(message_data), headers={'Content-Type': 'application/json'})
    response.raise_for_status()
<commit_msg>fix admin chat webhook<commit_after>import json

import requests
from django.conf import settings
from django.db.models.signals import post_save, pre_delete
from django.dispatch import receiver

from karrot.conversations.models import Conversation
from karrot.groups import roles, stats
from karrot.groups.emails import prepare_user_became_editor_email
from karrot.groups.models import Group, GroupMembership, Trust
from karrot.history.models import History, HistoryTypus
from karrot.utils import frontend_urls

@receiver(post_save, sender=Group)
def group_created(sender, instance, created, **kwargs):
    """Ensure every group has a conversation."""
    if not created:
        return
    group = instance
    conversation = Conversation.objects.get_or_create_for_target(group)
    conversation.sync_users(group.members.all())

@receiver(pre_delete, sender=Group)
def group_deleted(sender, instance, **kwargs):
    """Delete the conversation when the group is deleted."""
    group = instance
    conversation = Conversation.objects.get_for_target(group)
    if conversation:
        conversation.delete()

@receiver(post_save, sender=GroupMembership)
def group_member_added(sender, instance, created, **kwargs):
    if not created:
        return
    group = instance.group
    user = instance.user
    membership = instance

    if group.is_playground():
        membership.notification_types = []
        membership.roles.append(roles.GROUP_EDITOR)
        membership.save()

    conversation = Conversation.objects.get_or_create_for_target(group)
    conversation.join(user, muted=group.is_playground())

    stats.group_joined(group)

@receiver(pre_delete, sender=GroupMembership)
def group_member_removed(sender, instance, **kwargs):
    group = instance.group
    user = instance.user
    conversation = Conversation.objects.get_for_target(group)
    if conversation:
        conversation.leave(user)

    stats.group_left(group)

@receiver(post_save, sender=Trust)
def trust_given(sender, instance, created, **kwargs):
    if not created:
        return

    membership = instance.membership
    relevant_trust = Trust.objects.filter(membership=membership)
    trust_threshold = membership.group.trust_threshold_for_newcomer()

    if relevant_trust.count() >= trust_threshold and roles.GROUP_EDITOR not in membership.roles:
        membership.add_roles([roles.GROUP_EDITOR])
        membership.save()

        History.objects.create(
            typus=HistoryTypus.MEMBER_BECAME_EDITOR,
            group=membership.group,
            users=[membership.user],
            payload={
                'threshold': trust_threshold,
            },
        )

        prepare_user_became_editor_email(user=membership.user, group=membership.group).send()

        stats.member_became_editor(membership.group)

    stats.trust_given(membership.group)

@receiver(pre_delete, sender=GroupMembership)
def remove_trust(sender, instance, **kwargs):
    membership = instance

    Trust.objects.filter(given_by=membership.user, membership__group=membership.group).delete()

@receiver(post_save, sender=Group)
def notify_chat_on_group_creation(sender, instance, created, **kwargs):
    """Send notifications to admin chat"""
    if not created:
        return
    group = instance
    webhook_url = getattr(settings, 'ADMIN_CHAT_WEBHOOK', None)

    if webhook_url is None:
        return

    group_url = frontend_urls.group_preview_url(group)

    message_data = {
        'text': f':tada: A new group has been created on **{settings.SITE_NAME}**! [Visit {group.name}]({group_url})',
    }

    response = requests.post(webhook_url, data=json.dumps(message_data), headers={'Content-Type': 'application/json'})
    response.raise_for_status()
<|endoftext|>


<commit_before>"""
.. module:: kerfi
    :platform: Unix
    :synopsis: Kerfi package.

.. moduleauthor:: Pedro Salgado <steenzout@ymail.com>

   Copyright 2014 Pedro Emanuel de Castro Faria Salgado

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

from __future__ import absolute_import

import logging

from kerfi.version import __version__

LOGGER = logging.getLogger(__name__)

def cleanup_property(key):
    """
    Cleans the given key by removing whitespace from beginning/end of it and
    replacing any other whitespace by the underscore character.

    :param key: the key.
    :return: a cleaned up key.
    :rtype: str
    """
    return key.split(':')[0].strip().replace(' ', '_')

def load_raw(stream):
    """
    Loads the system profiler raw content into a data structure.

    :param stream: the input stream.
    :return: the system profiler data structure.
    :rtype: dict
    """
    o = {}
    namespace = []
    for line in stream.readlines():
        LOGGER.debug('line = %s', line)
        try:
            if line == '\n':
                # blank line
                continue

            elif line[0] != ' ':
                namespace = [line.split(':')[0]]

            elif line[0:6] == '      ':
                prop, value = line.split(':')
                prop = cleanup_property(prop)
                value = value.strip()
                o['%s.%s' % ('.'.join(namespace), prop)] = value

            elif line[0:4] == '    ':
                namespace = [namespace[0], cleanup_property(line)]

        except StandardError:
            LOGGER.error('Error processing line %s!', line, exc_info=True)

    return o

def load_xml(stream):
    """
    Loads the system profiler XML content into a data structure.

    :param stream: the input stream.
    :return: the system profiler data structure.
    :rtype: dict
    """
    return None
<commit_msg>fix bug when value also has colon.<commit_after>"""
.. module:: kerfi
    :platform: Unix
    :synopsis: Kerfi package.

.. moduleauthor:: Pedro Salgado <steenzout@ymail.com>

   Copyright 2014 Pedro Emanuel de Castro Faria Salgado

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

from __future__ import absolute_import

import logging

from kerfi.version import __version__

LOGGER = logging.getLogger(__name__)

def cleanup_property(key):
    """
    Cleans the given key by removing whitespace from beginning/end of it and
    replacing any other whitespace by the underscore character.

    :param key: the key.
    :return: a cleaned up key.
    :rtype: str
    """
    return key.split(':')[0].strip().replace(' ', '_')

def load_raw(stream):
    """
    Loads the system profiler raw content into a data structure.

    :param stream: the input stream.
    :return: the system profiler data structure.
    :rtype: dict
    """
    o = {}
    namespace = []
    levels = [0]
    for line in stream.readlines():
        LOGGER.debug('line = %s', line)
        try:
            if line != '\n':
                prop, value = line.split(':')
                lenprop1 = len(prop)
                prop = cleanup_property(prop)
                lenprop2 = len(prop)
                thislevel = lenprop1 - lenprop2
                value = value.strip()

                if thislevel == 0:
                    namespace = [cleanup_property(prop)]
                    levels = [0]
                else:
                    while thislevel <= levels[-1]:
                        namespace.pop()
                        levels.pop()
                    namespace.append(cleanup_property(prop))
                    levels.append(thislevel)

                if value != '':
                    o['.'.join(namespace)] = value

            LOGGER.debug('namespace = %s', namespace)
            LOGGER.debug('levels = %s', levels)

        except StandardError:
            LOGGER.error('Error processing line %s!', line, exc_info=True)

    return o

def load_xml(stream):
    """
    Loads the system profiler XML content into a data structure.

    :param stream: the input stream.
    :return: the system profiler data structure.
    :rtype: dict
    """
    return None
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from ds_binary_search_tree import TreeNode, BinarySearchTree

class AVLTreeNode(TreeNode):
    def __init__(self, *args, **kwargs):
        super(AVLTreeNode, self).__init__(*args, **kwargs)
        self.balance_factor = 0

class AVLTree(BinarySearchTree):
    TreeNode = AVLTreeNode

    def rotate_left(self, rotate_root):
        new_root = rotate_root.right_child
        rotate_root.right_child = new_root.left_child
        if not new_root.left_child:
            new_root.left_child.parent = rotate_root
        else:
            pass
        new_root.parent = rotate_root.parent

        if rotate_root.is_root():
            self.root = new_root
        else:
            if rotate_root.is_left_child():
                rotate_root.parent.left_child = new_root
            else: rotate_root.is_right_child():
                rotate_root.parent.right_child = new_root

        new_root.left_child = rotate_root
        rotate_root.parent = new_root
        rotate_root.balance_factor += 1 - min(new_root.balance_factor, 0)
        new_root.balance_factor += 1 + max(rotate_root.balance_facor, 0)

    def update_balance(self, node):
        if self.balance_factor < -1 or self.balance_factor > 1:
            self.rebalance(node)
            return None
        if not node.parent:
            if node.is_left_child():
                node.parent.balance_factor += 1
            elif node.is_right_child():
                node.parent.balance_factor -= 1
            else:
                pass

            if node.parent.balance_factor != 0:
                self.update_balance(node.parent)

    def _put(self, key, value, current_node):
        if key == current_node.key:
            self.replace_node_data(
                key, value, current_node.left_child, current_node.right_child)
        elif key < current_node.key:
            if current_node.has_left_child():
                self._put(key, value, current_node.left_child)
            else:
                current_node.left_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.left_child)
        else:
            if current_node.has_right_child():
                self._put(key, value, current_node.right_child)
            else:
                current_node.right_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.right_child)

def main():
    avlt = AVLTree()
    avlt[3] = 'red'
    avlt[4] = 'blue'
    avlt[6] = 'yellow'
    avlt[2] = 'black'

    print('len(avlt): {}'.format(len(avlt)))
    print('avlt.size: {}'.format(avlt.size))

    print(avlt[6])
    print(avlt[2])

    print('Iterate by the inorder traversal algorithm:')
    for x in avlt:
        print(x)

    print('Remove avlt[2] and then put it back')
    del avlt[2]
    for x in avlt:
        print(x)
    avlt[2] = 'black'

    print('Remove avlt[6] and then put it back')
    del avlt[6]
    for x in avlt:
        print(x)
    avlt[6] = 'yellow'

    print('Remove avlt[4] and then put it back')
    del avlt[4]
    for x in avlt:
        print(x)
    avlt[4] = 'blue'

    print('Iterate after del avlt[3]:')
    del avlt[3]
    for x in avlt:
        print(x)

if __name__ == '__main__':
    main()
<commit_msg>Add rotate_right()<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from ds_binary_search_tree import TreeNode, BinarySearchTree

class AVLTreeNode(TreeNode):
    def __init__(self, *args, **kwargs):
        super(AVLTreeNode, self).__init__(*args, **kwargs)
        self.balance_factor = 0

class AVLTree(BinarySearchTree):
    TreeNode = AVLTreeNode

    def rotate_left(self, rotate_root):
        new_root = rotate_root.right_child
        rotate_root.right_child = new_root.left_child
        if not new_root.left_child:
            new_root.left_child.parent = rotate_root
        else:
            pass
        new_root.parent = rotate_root.parent

        if rotate_root.is_root():
            self.root = new_root
        else:
            if rotate_root.is_left_child():
                rotate_root.parent.left_child = new_root
            else: rotate_root.is_right_child():
                rotate_root.parent.right_child = new_root

        new_root.left_child = rotate_root
        rotate_root.parent = new_root
        rotate_root.balance_factor += 1 - min(new_root.balance_factor, 0)
        new_root.balance_factor += 1 + max(rotate_root.balance_facor, 0)

    def rotate_right(self, rotate_root):
        pass

    def update_balance(self, node):
        if self.balance_factor < -1 or self.balance_factor > 1:
            self.rebalance(node)
            return None
        if not node.parent:
            if node.is_left_child():
                node.parent.balance_factor += 1
            elif node.is_right_child():
                node.parent.balance_factor -= 1
            else:
                pass

            if node.parent.balance_factor != 0:
                self.update_balance(node.parent)

    def _put(self, key, value, current_node):
        if key == current_node.key:
            self.replace_node_data(
                key, value, current_node.left_child, current_node.right_child)
        elif key < current_node.key:
            if current_node.has_left_child():
                self._put(key, value, current_node.left_child)
            else:
                current_node.left_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.left_child)
        else:
            if current_node.has_right_child():
                self._put(key, value, current_node.right_child)
            else:
                current_node.right_child = TreeNode(key, value, parent=current_node)
                self.update_balance(current_node.right_child)

def main():
    avlt = AVLTree()
    avlt[3] = 'red'
    avlt[4] = 'blue'
    avlt[6] = 'yellow'
    avlt[2] = 'black'

    print('len(avlt): {}'.format(len(avlt)))
    print('avlt.size: {}'.format(avlt.size))

    print(avlt[6])
    print(avlt[2])

    print('Iterate by the inorder traversal algorithm:')
    for x in avlt:
        print(x)

    print('Remove avlt[2] and then put it back')
    del avlt[2]
    for x in avlt:
        print(x)
    avlt[2] = 'black'

    print('Remove avlt[6] and then put it back')
    del avlt[6]
    for x in avlt:
        print(x)
    avlt[6] = 'yellow'

    print('Remove avlt[4] and then put it back')
    del avlt[4]
    for x in avlt:
        print(x)
    avlt[4] = 'blue'

    print('Iterate after del avlt[3]:')
    del avlt[3]
    for x in avlt:
        print(x)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import inspect
import typing

import types

from katagawa.exc import NoSuchColumnError
from katagawa.orm.schema import Column
from katagawa.orm.schema import table as md_table
from katagawa.orm import session as md_session

class TableRow(object):
    """
    Represents a single row in a table.
    :class:`.Table` objects cannot be instantiated (not without hacking at the object level), so as
    such they return TableRow objects when called.

    TableRow objects are representative of a single row in said table - the column names are the
    keys, and the value in that row are the items.

    .. code-block:: python
        class User(Table):
            id = Column(Integer, primary_key=True)

        user = User(id=1)  # user is actually a TableRow bound to the User table
    """

    def __init__(self, tbl):
        """
        :param tbl: The table object to bind this row to.
        """
        self._table = tbl

        #: If this row existed before.
        #: If this is True, this row was fetched from the DB previously.
        #: Otherwise, it is a fresh row.
        self.__existed = False

        #: The session this row is attached to.
        self._session = None  # type: md_session.Session

        #: A mapping of Column -> Previous values for this row.
        #: Used in update generation.
        self._previous_values = {}

        #: A mapping of Column -> Current value for this row.
        self._values = {}

        # BECAUSE PYTHON
        self.__setattr__ = self._setattr__

    def __repr__(self):
        gen = ("{}={}".format(col.name, self._get_column_value(col)) for col in self._table.columns)
        return "<{} {}>".format(self._table.__name__, " ".join(gen))

    def __getattr__(self, item: str):
        obb = self._resolve_item(item)
        return obb

        col = next(filter(lambda col: col.name == item, self._table.columns), None)
        if col is None:
            raise NoSuchColumnError(item)

        return self._values[col]

    def _setattr__(self, key, value):
        col = next(filter(lambda col: col.name == key, self._table.columns), None)
        if col is None:
            return super().__setattr__(key, value)

        return self.update_column(col, value)

    def _resolve_item(self, name: str):
        """
        Resolves an item on this TableRow.

        This will check:

            - Functions decorated with :func:`.row_attr`
            - Non-column :class:`.Table` members
            - Columns

        :param name: The name to resolve.
        :return: The object returned, if applicable.
        """
        # try and getattr the name from the Table object
        try:
            item = getattr(self._table, name)
        except AttributeError:
            pass
        else:
            # proxy to the table
            if hasattr(item, "__row_attr__"):
                return item(self)

            if inspect.isfunction(item):
                # bind it to ourselves, and return it
                return types.MethodType(item, self)
            return item

        # failed to load item, so load a column value instead
        col = next(filter(lambda col: col.name == item, self._table.columns), None)
        if col is None:
            raise AttributeError(name, "was not a function or attribute on the associated table, "
                                       "and was not a column")

        return self._get_column_value(col)

    def _get_column_value(self, column: 'Column'):
        """
        Gets the value from the specified column in this row.
        """
        if column.table != self._table:
            raise ValueError("Column table must match row table")

        try:
            return self._values[column]
        except KeyError:
            return column.default

    def update_column(self, column: 'Column', value: typing.Any):
        """
        Updates the value of a column in this row.
        """
        if column not in self._previous_values:
            if column in self._values:
                self._previous_values[column] = self._values[column]

        self._values[column] = value

        return self

    def to_dict(self, *, include_attrs: bool = False) -> dict:
        """
        Converts this row to a dict, indexed by Column.

        :param include_attrs: Should this include row_attrs?
        """
        # todo: include row attrs
        d = {col: self._get_column_value(col) for col in self._table.columns}
        return d

    @property
    def primary_key(self) -> typing.Union[typing.Any, typing.Iterable[typing.Any]]:
        """
        Gets the primary key for this row.

        If this table only has one primary key column, this property will be a single value.
        If this table has multiple columns in a primary key, this property will be a tuple.
        """
        pk = self._table.primary_key  # type: md_table.PrimaryKey
        result = []

        for col in pk.columns:
            val = self._get_column_value(col)
            result.append(val)

        if len(result) == 1:
            return result[0]

        return tuple(result)
<commit_msg>Remove dead code<commit_after>import inspect
import typing

import types

from katagawa.exc import NoSuchColumnError
from katagawa.orm.schema import Column
from katagawa.orm.schema import table as md_table
from katagawa.orm import session as md_session

class TableRow(object):
    """
    Represents a single row in a table.
    :class:`.Table` objects cannot be instantiated (not without hacking at the object level), so as
    such they return TableRow objects when called.

    TableRow objects are representative of a single row in said table - the column names are the
    keys, and the value in that row are the items.

    .. code-block:: python
        class User(Table):
            id = Column(Integer, primary_key=True)

        user = User(id=1)  # user is actually a TableRow bound to the User table
    """

    def __init__(self, tbl):
        """
        :param tbl: The table object to bind this row to.
        """
        self._table = tbl

        #: If this row existed before.
        #: If this is True, this row was fetched from the DB previously.
        #: Otherwise, it is a fresh row.
        self.__existed = False

        #: The session this row is attached to.
        self._session = None  # type: md_session.Session

        #: A mapping of Column -> Previous values for this row.
        #: Used in update generation.
        self._previous_values = {}

        #: A mapping of Column -> Current value for this row.
        self._values = {}

        # BECAUSE PYTHON
        self.__setattr__ = self._setattr__

    def __repr__(self):
        gen = ("{}={}".format(col.name, self._get_column_value(col)) for col in self._table.columns)
        return "<{} {}>".format(self._table.__name__, " ".join(gen))

    def __getattr__(self, item: str):
        obb = self._resolve_item(item)
        return obb

    def _setattr__(self, key, value):
        col = next(filter(lambda col: col.name == key, self._table.columns), None)
        if col is None:
            return super().__setattr__(key, value)

        return self.update_column(col, value)

    def _resolve_item(self, name: str):
        """
        Resolves an item on this TableRow.

        This will check:

            - Functions decorated with :func:`.row_attr`
            - Non-column :class:`.Table` members
            - Columns

        :param name: The name to resolve.
        :return: The object returned, if applicable.
        """
        # try and getattr the name from the Table object
        try:
            item = getattr(self._table, name)
        except AttributeError:
            pass
        else:
            # proxy to the table
            if hasattr(item, "__row_attr__"):
                return item(self)

            if inspect.isfunction(item):
                # bind it to ourselves, and return it
                return types.MethodType(item, self)
            return item

        # failed to load item, so load a column value instead
        col = next(filter(lambda col: col.name == item, self._table.columns), None)
        if col is None:
            raise AttributeError(name, "was not a function or attribute on the associated table, "
                                       "and was not a column")

        return self._get_column_value(col)

    def _get_column_value(self, column: 'Column'):
        """
        Gets the value from the specified column in this row.
        """
        if column.table != self._table:
            raise ValueError("Column table must match row table")

        try:
            return self._values[column]
        except KeyError:
            return column.default

    def update_column(self, column: 'Column', value: typing.Any):
        """
        Updates the value of a column in this row.
        """
        if column not in self._previous_values:
            if column in self._values:
                self._previous_values[column] = self._values[column]

        self._values[column] = value

        return self

    def to_dict(self, *, include_attrs: bool = False) -> dict:
        """
        Converts this row to a dict, indexed by Column.

        :param include_attrs: Should this include row_attrs?
        """
        # todo: include row attrs
        d = {col: self._get_column_value(col) for col in self._table.columns}
        return d

    @property
    def primary_key(self) -> typing.Union[typing.Any, typing.Iterable[typing.Any]]:
        """
        Gets the primary key for this row.

        If this table only has one primary key column, this property will be a single value.
        If this table has multiple columns in a primary key, this property will be a tuple.
        """
        pk = self._table.primary_key  # type: md_table.PrimaryKey
        result = []

        for col in pk.columns:
            val = self._get_column_value(col)
            result.append(val)

        if len(result) == 1:
            return result[0]

        return tuple(result)
<|endoftext|>


<commit_before>"""
Module containing the actual query
"""
import logging
import typing

from katagawa.engine.base import ResultSet
from katagawa.engine.transaction import Transaction
from katagawa.exceptions import TableConflictException
from katagawa.orm.column import _Operator, _CombinatorOperator
from katagawa.orm.table import Table, TableRow
from katagawa import session as md_sess
from katagawa.sql.dialects.common import Select, Column, From, Where, Operator

logger = logging.getLogger("Katagawa.query")

class BaseQuery(object):
    """
    A BaseQuery object is used to query the database with SELECT statements or otherwise.

    It is produced from a :meth:`.Session.query` and is used to actually query the database.
    """
    def __init__(self, session: 'md_sess.Session', table: 'Table', **kwargs):
        """
        Creates a new BaseQuery.

        :param session: The session to bind this query to.
        """
        self.session = session  # type: md_sess.Session

        #: The table being queried.
        self.from_ = table

        #: A dict of tables being joined, or similar.
        self.tables = {}

        #: A list of conditions being used.
        self.conditions = []

        # internal alias mapping
        # this is used once the query returns to get the right column from the response
        self._alias_mapping: typing.Mapping[str, Column] = {}

    # internal workhouse methods

    # query methods

    def select(self, *tables: Table):
        """
        Selects some tables to query.

        :param tables: A list of DeclarativeMeta or aliased tables to query.
        :return: Ourself.
        """
        for table in tables:
            if not isinstance(table, Table):
                raise TypeError("Table must be instance of Table")

            self.tables[table.name] = table

        return self

    def where(self, *conditions):
        """
        Adds conditions to the query.

        :param conditions: A list of field operators to query.
        :return: Ourself.
        """
        for condition in conditions:
            self.conditions.append(condition)

        return self

    # sql methods
    def get_token(self) -> typing.Tuple[Select, dict]:
        """
        Gets the Select tokens for this query.
        """
        # get the fields
        fields = []

        # add the main table's fields
        for column in self.from_.columns:
            fields.append(
                Column('"{}"."{}"'.format(self.from_.name, column.name))
            )

        # add any joined tables fields
        for tbl_name, table in self.tables.items():
            for column in table.columns:
                fields.append(Column(
                    '"{}"."{}"'.format(table.name, column.name))
                )

        s = Select(
            subtokens=[
                # expand out the fields and tables as From queries
                *fields,
                From(self.from_.name)
            ]
        )

        # update subfields with WHERE query
        params = {}
        if self.conditions:
            where = Where()
            param_count = 0
            for op in self.conditions:
                if isinstance(op, _Operator):
                    ops = [op]
                elif isinstance(op, _CombinatorOperator):
                    ops = op.operators

                final = op.get_token()

                for nop in ops:
                    o = nop.get_token()

                    # update parameterized queries
                    if isinstance(o.value, str):
                        name = "param_{}".format(param_count)
                        params[name] = o.value
                        o.value = "{{{n}}}".format(n=name)
                        param_count += 1

                where.subtokens.append(final)

            s.subtokens.append(where)

        return s, params

    # return methods
    def _convert_result(self, result: typing.Mapping[str, typing.Any]) -> TableRow:
        """
        Converts a result from the raw DB connection into a :class:`.TableRow`.

        :param result: The result to convert from.
            This should be a dictionary-like object, with keys and values methods that can be used
            to get the columns and values from the result.

        :return: A new :class:`.TableRow` representing the result.
        """
        row = TableRow(self.from_)
        for key in result.keys():
            if key in self._alias_mapping:
                col = self._alias_mapping[key]
            else:
                # try and find the column from the main table
                col = self.from_.column_mapping[key]

            # update the current values
            row._values[col.name] = result[key]

        return row

    async def all(self) -> typing.Generator[typing.Mapping, None, None]:
        """
        Gets
        """
        r = await self.session.execute(self)

        async for result in r:
            result = self._convert_result(result)
            yield result

    async def first(self):
        """
        Returns the first result that matches.
        """
        rset = await self.session.execute(self)
        next = await rset.get_next()

        return next
<commit_msg>Use the right engine params when substituting parameters.<commit_after>"""
Module containing the actual query
"""
import logging
import typing

from katagawa.engine.base import ResultSet
from katagawa.engine.transaction import Transaction
from katagawa.exceptions import TableConflictException
from katagawa.orm.column import _Operator, _CombinatorOperator
from katagawa.orm.table import Table, TableRow
from katagawa import session as md_sess
from katagawa.sql.dialects.common import Select, Column, From, Where, Operator

logger = logging.getLogger("Katagawa.query")

class BaseQuery(object):
    """
    A BaseQuery object is used to query the database with SELECT statements or otherwise.

    It is produced from a :meth:`.Session.query` and is used to actually query the database.
    """
    def __init__(self, session: 'md_sess.Session', table: 'Table', **kwargs):
        """
        Creates a new BaseQuery.

        :param session: The session to bind this query to.
        """
        self.session = session  # type: md_sess.Session

        #: The table being queried.
        self.from_ = table

        #: A dict of tables being joined, or similar.
        self.tables = {}

        #: A list of conditions being used.
        self.conditions = []

        # internal alias mapping
        # this is used once the query returns to get the right column from the response
        self._alias_mapping: typing.Mapping[str, Column] = {}

    # internal workhouse methods

    # query methods

    def select(self, *tables: Table):
        """
        Selects some tables to query.

        :param tables: A list of DeclarativeMeta or aliased tables to query.
        :return: Ourself.
        """
        for table in tables:
            if not isinstance(table, Table):
                raise TypeError("Table must be instance of Table")

            self.tables[table.name] = table

        return self

    def where(self, *conditions):
        """
        Adds conditions to the query.

        :param conditions: A list of field operators to query.
        :return: Ourself.
        """
        for condition in conditions:
            self.conditions.append(condition)

        return self

    # sql methods
    def get_token(self) -> typing.Tuple[Select, dict]:
        """
        Gets the Select tokens for this query.
        """
        # get the fields
        fields = []

        # add the main table's fields
        for column in self.from_.columns:
            fields.append(
                Column('"{}"."{}"'.format(self.from_.name, column.name))
            )

        # add any joined tables fields
        for tbl_name, table in self.tables.items():
            for column in table.columns:
                fields.append(Column(
                    '"{}"."{}"'.format(table.name, column.name))
                )

        s = Select(
            subtokens=[
                # expand out the fields and tables as From queries
                *fields,
                From(self.from_.name)
            ]
        )

        # update subfields with WHERE query
        params = {}
        if self.conditions:
            where = Where()
            param_count = 0
            for op in self.conditions:
                # extract the condition subtokens from the operators
                if isinstance(op, _Operator):
                    ops = [op]
                elif isinstance(op, _CombinatorOperator):
                    ops = op.operators

                # unfuck the final tokens to prevent sql injection
                final = op.get_token()

                for nop in ops:
                    o = nop.get_token()

                    # update parameterized queries
                    if isinstance(o.value, str):
                        name = "param_{}".format(param_count)
                        params[name] = o.value
                        o.value = self.session.engine.emit_param(name)
                        param_count += 1

                where.subtokens.append(final)

            s.subtokens.append(where)

        return s, params

    # return methods
    def _convert_result(self, result: typing.Mapping[str, typing.Any]) -> TableRow:
        """
        Converts a result from the raw DB connection into a :class:`.TableRow`.

        :param result: The result to convert from.
            This should be a dictionary-like object, with keys and values methods that can be used
            to get the columns and values from the result.

        :return: A new :class:`.TableRow` representing the result.
        """
        row = TableRow(self.from_)
        for key in result.keys():
            if key in self._alias_mapping:
                col = self._alias_mapping[key]
            else:
                # try and find the column from the main table
                col = self.from_.column_mapping[key]

            # update the current values
            row._values[col.name] = result[key]

        return row

    async def all(self) -> typing.Generator[typing.Mapping, None, None]:
        """
        Gets
        """
        r = await self.session.execute(self)

        async for result in r:
            result = self._convert_result(result)
            yield result

    async def first(self):
        """
        Returns the first result that matches.
        """
        rset = await self.session.execute(self)
        next = await rset.get_next()

        return next
<|endoftext|>


<commit_before>from django.db import DatabaseError
from django.test import TestCase

from dummy.models import Dummy

class TestDummyDecimalFields(TestCase):

    def _assertBulkCreateWorks(self, objs, batch_size=None):
        """
        Create records with bulk_create, watching for DatabaseErrors.

        If batch_size=None, Django will set batch_size=len(objs).
        """
        cls = objs[0].__class__
        try:
            cls.objects.bulk_create(objs, batch_size=batch_size)
        except DatabaseError as e:
            self.fail(
                'Bulk creation of {n} {cls} objects with '
                'batch_size={batch} raised the following '
                'exception: {exc}'.format(
                    n=len(objs), cls=cls, batch=batch_size, exc=e,
                )
            )

    def test_no_fields_set(self):
        objs = [Dummy(), Dummy()]
        self._assertBulkCreateWorks(objs)
        for n in [1, 2, 3, 4, 10, 20, 50, 100, 1000]:
            self._assertBulkCreateWorks(objs, batch_size=n)

    def test_same_field_set(self):
        objs = [Dummy(a=u'123.45'), Dummy(a=44.42)]
        for n in [1, 2, 3, 4, 10, 20, 50, 100, 1000]:
            self._assertBulkCreateWorks(objs, batch_size=n)

    def test_both_fields_set(self):
        objs = [Dummy(a=u'123.45', b=1), Dummy(a=44.42, b=u'345345345')]
        for n in [1, 2, 3, 4, 10, 20, 50, 100, 1000]:
            self._assertBulkCreateWorks(objs, batch_size=n)

    def test_different_fields_set(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs)

    def test_different_fields_set_with_batch_size_1(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs, batch_size=1)

    def test_different_fields_set_with_batch_size_2(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs, batch_size=2)

    def test_different_fields_set_with_batch_size_35(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs, batch_size=35)

    def test_one_field_set(self):
        objs = [Dummy(a=u'123.45'), Dummy()]
        self._assertBulkCreateWorks(objs)

<commit_msg>Can't convert float to Decimal in python 2.6; use unicode everywhere, instead.<commit_after>from django.db import DatabaseError
from django.test import TestCase

from dummy.models import Dummy

class TestDummyDecimalFields(TestCase):

    def _assertBulkCreateWorks(self, objs, batch_size=None):
        """
        Create records with bulk_create, watching for DatabaseErrors.

        If batch_size=None, Django will set batch_size=len(objs).
        """
        cls = objs[0].__class__
        try:
            cls.objects.bulk_create(objs, batch_size=batch_size)
        except DatabaseError as e:
            self.fail(
                'Bulk creation of {n} {cls} objects with '
                'batch_size={batch} raised the following '
                'exception: {exc}'.format(
                    n=len(objs), cls=cls, batch=batch_size, exc=e,
                )
            )

    def test_no_fields_set(self):
        objs = [Dummy(), Dummy()]
        self._assertBulkCreateWorks(objs)
        for n in [1, 2, 3, 4, 10, 20, 50, 100, 1000]:
            self._assertBulkCreateWorks(objs, batch_size=n)

    def test_same_field_set(self):
        objs = [Dummy(a=u'123.45'), Dummy(a=u'44.42')]
        for n in [1, 2, 3, 4, 10, 20, 50, 100, 1000]:
            self._assertBulkCreateWorks(objs, batch_size=n)

    def test_both_fields_set(self):
        objs = [Dummy(a=u'123.45', b=1), Dummy(a=u'44.42', b=u'345345345')]
        for n in [1, 2, 3, 4, 10, 20, 50, 100, 1000]:
            self._assertBulkCreateWorks(objs, batch_size=n)

    def test_different_fields_set(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs)

    def test_different_fields_set_with_batch_size_1(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs, batch_size=1)

    def test_different_fields_set_with_batch_size_2(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs, batch_size=2)

    def test_different_fields_set_with_batch_size_35(self):
        objs = [Dummy(a=u'123.45'), Dummy(b=u'94324.35')]
        self._assertBulkCreateWorks(objs, batch_size=35)

    def test_one_field_set(self):
        objs = [Dummy(a=u'123.45'), Dummy()]
        self._assertBulkCreateWorks(objs)

<|endoftext|>


<commit_before>import os
import glob
import yaml

from .config import get_config_value

def get_specs():
    specs_path = get_config_value('specs_path')
    get_specs_from_path(specs_path)

def get_specs_from_path(specs_path):
    specs = {}
    for key in ['bundles', 'apps', 'libs', 'services']:
        specs[key] = {}
        key_path = os.path.join(specs_path, key)
        for spec_path in glob.glob('{}/*.yml'.format(key_path)):
            spec_name = os.path.splitext(os.path.split(spec_path)[-1])[0]
            with open(spec_path, 'r') as f:
                specs[key][spec_name] = yaml.load(f.read())
    return specs

<commit_msg>return is important<commit_after>import os
import glob
import yaml

from .config import get_config_value

def get_specs():
    specs_path = get_config_value('specs_path')
    return get_specs_from_path(specs_path)

def get_specs_from_path(specs_path):
    specs = {}
    for key in ['bundles', 'apps', 'libs', 'services']:
        specs[key] = {}
        key_path = os.path.join(specs_path, key)
        for spec_path in glob.glob('{}/*.yml'.format(key_path)):
            spec_name = os.path.splitext(os.path.split(spec_path)[-1])[0]
            with open(spec_path, 'r') as f:
                specs[key][spec_name] = yaml.load(f.read())
    return specs

<|endoftext|>


<commit_before># Used in setup.py, so don't pull any additional dependencies
#
# Based on:
#   - https://github.com/python/mypy/blob/master/mypy/version.py
#   - https://github.com/python/mypy/blob/master/mypy/git.py
import os
import subprocess

_BASE_VERSION = "0.66.8"

def _generate_version(base_version):
    """Generate a version with information about the git repository"""
    pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    if not _is_git_repo(pkg_dir) or not _have_git():
        return base_version

    if _is_release(pkg_dir, base_version) and not _is_dirty(pkg_dir):
        return base_version

    return "{base_version}+{short_sha}{dirty}".format(
        base_version=base_version,
        short_sha=_git_revision(pkg_dir).decode("utf-8")[0:6],
        dirty=".mod" if _is_dirty(pkg_dir) else "",
    )

def _is_git_repo(dir_path):
    """Is the given directory version-controlled with git?"""
    return os.path.exists(os.path.join(dir_path, ".git"))

def _have_git():
    """Can we run the git executable?"""
    try:
        subprocess.check_output(["git", "--help"])
        return True
    except subprocess.CalledProcessError:
        return False
    except OSError:
        return False

def _is_release(dir_path, base_version):
    try:
        output = subprocess.check_output(
            ["git", "describe", "--tags", "--exact-match"],
            cwd=dir_path,
            stderr=subprocess.STDOUT,
        ).decode("utf-8")
        tag = output.strip()
        return tag == base_version
    except subprocess.CalledProcessError:
        return False

def _git_revision(dir_path):
    """Get the SHA-1 of the HEAD of a git repository."""
    return subprocess.check_output(
        ["git", "rev-parse", "HEAD"], cwd=dir_path
    ).strip()

def _is_dirty(dir_path):
    """Check whether a git repository has uncommitted changes."""
    try:
        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
        return False
    except subprocess.CalledProcessError:
        return True

__version__ = _generate_version(_BASE_VERSION)
<commit_msg>dvc: bump to 0.66.9<commit_after># Used in setup.py, so don't pull any additional dependencies
#
# Based on:
#   - https://github.com/python/mypy/blob/master/mypy/version.py
#   - https://github.com/python/mypy/blob/master/mypy/git.py
import os
import subprocess

_BASE_VERSION = "0.66.9"

def _generate_version(base_version):
    """Generate a version with information about the git repository"""
    pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    if not _is_git_repo(pkg_dir) or not _have_git():
        return base_version

    if _is_release(pkg_dir, base_version) and not _is_dirty(pkg_dir):
        return base_version

    return "{base_version}+{short_sha}{dirty}".format(
        base_version=base_version,
        short_sha=_git_revision(pkg_dir).decode("utf-8")[0:6],
        dirty=".mod" if _is_dirty(pkg_dir) else "",
    )

def _is_git_repo(dir_path):
    """Is the given directory version-controlled with git?"""
    return os.path.exists(os.path.join(dir_path, ".git"))

def _have_git():
    """Can we run the git executable?"""
    try:
        subprocess.check_output(["git", "--help"])
        return True
    except subprocess.CalledProcessError:
        return False
    except OSError:
        return False

def _is_release(dir_path, base_version):
    try:
        output = subprocess.check_output(
            ["git", "describe", "--tags", "--exact-match"],
            cwd=dir_path,
            stderr=subprocess.STDOUT,
        ).decode("utf-8")
        tag = output.strip()
        return tag == base_version
    except subprocess.CalledProcessError:
        return False

def _git_revision(dir_path):
    """Get the SHA-1 of the HEAD of a git repository."""
    return subprocess.check_output(
        ["git", "rev-parse", "HEAD"], cwd=dir_path
    ).strip()

def _is_dirty(dir_path):
    """Check whether a git repository has uncommitted changes."""
    try:
        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
        return False
    except subprocess.CalledProcessError:
        return True

__version__ = _generate_version(_BASE_VERSION)
<|endoftext|>


<commit_before># Used in setup.py, so don't pull any additional dependencies
#
# Based on:
#   - https://github.com/python/mypy/blob/master/mypy/version.py
#   - https://github.com/python/mypy/blob/master/mypy/git.py
import os
import subprocess

_BASE_VERSION = "2.5.1"

def _generate_version(base_version):
    """Generate a version with information about the Git repository."""
    pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    if not _is_git_repo(pkg_dir) or not _have_git():
        return base_version

    if _is_release(pkg_dir, base_version) and not _is_dirty(pkg_dir):
        return base_version

    return "{base_version}+{short_sha}{dirty}".format(
        base_version=base_version,
        short_sha=_git_revision(pkg_dir).decode("utf-8")[0:6],
        dirty=".mod" if _is_dirty(pkg_dir) else "",
    )

def _is_git_repo(dir_path):
    """Is the given directory version-controlled with Git?"""
    return os.path.exists(os.path.join(dir_path, ".git"))

def _have_git():
    """Can we run the git executable?"""
    try:
        subprocess.check_output(["git", "--help"])
        return True
    except subprocess.CalledProcessError:
        return False
    except OSError:
        return False

def _is_release(dir_path, base_version):
    try:
        output = subprocess.check_output(
            ["git", "describe", "--tags", "--exact-match"],
            cwd=dir_path,
            stderr=subprocess.STDOUT,
        ).decode("utf-8")
        tag = output.strip()
        return tag == base_version
    except subprocess.CalledProcessError:
        return False

def _git_revision(dir_path):
    """Get SHA of the HEAD of a Git repository."""
    return subprocess.check_output(
        ["git", "rev-parse", "HEAD"], cwd=dir_path
    ).strip()

def _is_dirty(dir_path):
    """Check whether a git repository has uncommitted changes."""
    try:
        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
        return False
    except subprocess.CalledProcessError:
        return True

__version__ = _generate_version(_BASE_VERSION)
<commit_msg>dvc: bump to 2.5.2<commit_after># Used in setup.py, so don't pull any additional dependencies
#
# Based on:
#   - https://github.com/python/mypy/blob/master/mypy/version.py
#   - https://github.com/python/mypy/blob/master/mypy/git.py
import os
import subprocess

_BASE_VERSION = "2.5.2"

def _generate_version(base_version):
    """Generate a version with information about the Git repository."""
    pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    if not _is_git_repo(pkg_dir) or not _have_git():
        return base_version

    if _is_release(pkg_dir, base_version) and not _is_dirty(pkg_dir):
        return base_version

    return "{base_version}+{short_sha}{dirty}".format(
        base_version=base_version,
        short_sha=_git_revision(pkg_dir).decode("utf-8")[0:6],
        dirty=".mod" if _is_dirty(pkg_dir) else "",
    )

def _is_git_repo(dir_path):
    """Is the given directory version-controlled with Git?"""
    return os.path.exists(os.path.join(dir_path, ".git"))

def _have_git():
    """Can we run the git executable?"""
    try:
        subprocess.check_output(["git", "--help"])
        return True
    except subprocess.CalledProcessError:
        return False
    except OSError:
        return False

def _is_release(dir_path, base_version):
    try:
        output = subprocess.check_output(
            ["git", "describe", "--tags", "--exact-match"],
            cwd=dir_path,
            stderr=subprocess.STDOUT,
        ).decode("utf-8")
        tag = output.strip()
        return tag == base_version
    except subprocess.CalledProcessError:
        return False

def _git_revision(dir_path):
    """Get SHA of the HEAD of a Git repository."""
    return subprocess.check_output(
        ["git", "rev-parse", "HEAD"], cwd=dir_path
    ).strip()

def _is_dirty(dir_path):
    """Check whether a git repository has uncommitted changes."""
    try:
        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
        return False
    except subprocess.CalledProcessError:
        return True

__version__ = _generate_version(_BASE_VERSION)
<|endoftext|>


<commit_before># Used in setup.py, so don't pull any additional dependencies
#
# Based on:
#   - https://github.com/python/mypy/blob/master/mypy/version.py
#   - https://github.com/python/mypy/blob/master/mypy/git.py
import os
import subprocess

_BASE_VERSION = "0.35.5"

def _generate_version(base_version):
    """Generate a version with information about the git repository"""
    pkg_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

    if not _is_git_repo(pkg_dir) or not _have_git():
        return base_version

    if _is_release(pkg_dir, base_version):
        return base_version

    return "{base_version}+{short_sha}{dirty}".format(
        base_version=base_version,
        short_sha=_git_revision(pkg_dir).decode("utf-8")[0:6],
        dirty=".mod" if _is_dirty(pkg_dir) else "",
    )

def _is_git_repo(dir_path):
    """Is the given directory version-controlled with git?"""
    return os.path.exists(os.path.join(dir_path, ".git"))

def _have_git():
    """Can we run the git executable?"""
    try:
        subprocess.check_output(["git", "--help"])
        return True
    except subprocess.CalledProcessError:
        return False
    except OSError:
        return False

def _is_release(dir_path, base_version):
    try:
        output = subprocess.check_output(
            ["git", "describe", "--tags", "--exact-match"], cwd=dir_path
        ).decode("utf-8")
        tag = output.strip()
        return tag == base_version
    except subprocess.CalledProcessError:
        return False

def _git_revision(dir_path):
    """Get the SHA-1 of the HEAD of a git repository."""
    return subprocess.check_output(
        ["git", "rev-parse", "HEAD"], cwd=dir_path
    ).strip()

def _is_dirty(dir_path):
    """Check whether a git repository has uncommitted changes."""
    try:
        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
        return False
    except subprocess.CalledProcessError:
        return True

__version__ = _generate_version(_BASE_VERSION)
<commit_msg>dvc: bump to 0.35.6<commit_after># Used in setup.py, so don't pull any additional dependencies
#
# Based on:
#   - https://github.com/python/mypy/blob/master/mypy/version.py
#   - https://github.com/python/mypy/blob/master/mypy/git.py
import os
import subprocess

_BASE_VERSION = "0.35.6"

def _generate_version(base_version):
    """Generate a version with information about the git repository"""
    pkg_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

    if not _is_git_repo(pkg_dir) or not _have_git():
        return base_version

    if _is_release(pkg_dir, base_version):
        return base_version

    return "{base_version}+{short_sha}{dirty}".format(
        base_version=base_version,
        short_sha=_git_revision(pkg_dir).decode("utf-8")[0:6],
        dirty=".mod" if _is_dirty(pkg_dir) else "",
    )

def _is_git_repo(dir_path):
    """Is the given directory version-controlled with git?"""
    return os.path.exists(os.path.join(dir_path, ".git"))

def _have_git():
    """Can we run the git executable?"""
    try:
        subprocess.check_output(["git", "--help"])
        return True
    except subprocess.CalledProcessError:
        return False
    except OSError:
        return False

def _is_release(dir_path, base_version):
    try:
        output = subprocess.check_output(
            ["git", "describe", "--tags", "--exact-match"], cwd=dir_path
        ).decode("utf-8")
        tag = output.strip()
        return tag == base_version
    except subprocess.CalledProcessError:
        return False

def _git_revision(dir_path):
    """Get the SHA-1 of the HEAD of a git repository."""
    return subprocess.check_output(
        ["git", "rev-parse", "HEAD"], cwd=dir_path
    ).strip()

def _is_dirty(dir_path):
    """Check whether a git repository has uncommitted changes."""
    try:
        subprocess.check_call(["git", "diff", "--quiet"], cwd=dir_path)
        return False
    except subprocess.CalledProcessError:
        return True

__version__ = _generate_version(_BASE_VERSION)
<|endoftext|>


<commit_before>"""Texture analysis."""

from __future__ import division
import collections
import itertools

import numpy as np
import scipy as sp
import skimage
import skimage.feature

import dwi.util

# Basic statistical features

def stats(img):
    """Get basic statistical features."""
    d = collections.OrderedDict()
    d['mean'] = np.mean(img)
    d['stddev'] = np.std(img)
    d['range'] = np.max(img) - np.min(img)
    d.update(dwi.util.fivenumd(img))
    for i in range(1, 10):
        d['decile%i' % i] = sp.stats.scoreatpercentile(img, i*10)
    d['kurtosis'] = sp.stats.kurtosis(img.ravel())
    d['skewness'] = sp.stats.skew(img.ravel())
    return d

def stats_map(img, winsize, names=None, mask=None, output=None):
    """Get basic statistical texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        d = stats(win)
        if names is None:
            names = d.keys()
        if output is None:
            output = np.zeros((len(names),) + img.shape)
        for i, name in enumerate(names):
            output[(i,) + pos] = d[name]
    return output, names

# Grey-Level Co-Occurrence Matrix (GLCM) features

PROPNAMES = 'contrast dissimilarity homogeneity energy correlation ASM'.split()

def glcm_props(img, names=PROPNAMES):
    """Get grey-level co-occurrence matrix texture properties over an image."""
    from skimage.feature import greycomatrix, greycoprops
    distances = [1]
    angles = [0, np.pi/4, np.pi/2, 3*np.pi/4]
    levels = 256
    glcm = greycomatrix(img, distances, angles, levels, symmetric=True,
            normed=True)
    keys = names
    values = [np.mean(greycoprops(glcm, p)) for p in names]
    d = collections.OrderedDict((k, v) for k, v in zip(keys, values))
    return d

def glcm_map(img, winsize, names=PROPNAMES, mask=None, output=None):
    """Grey-level co-occurrence matrix texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        d = glcm_props(win, names)
        if output is None:
            output = np.zeros((len(names),) + img.shape)
        for i, name in enumerate(names):
            output[(i,) + pos] = d[name]
    return output, names

#def coprops(windows):
#    # XXX Obsolete
#    from skimage.feature import greycomatrix, greycoprops
#    props = np.zeros((len(windows), len(PROPNAMES)))
#    for i, win in enumerate(windows):
#        #win = skimage.img_as_ubyte(win)
#        #if win.min() == 0:
#        #    props[i].fill(0)
#        #    continue
#        distances = [1]
#        angles = [0, np.pi/4, np.pi/2, 3*np.pi/4]
#        glcm = greycomatrix(win, distances, angles, 256, symmetric=True,
#                normed=True)
#        for j, propname in enumerate(PROPNAMES):
#            a = greycoprops(glcm, propname)
#            props[i,j] = np.mean(a)
#    return props
#
#def texture_pmap(img, win_step):
#    # XXX Obsolete
#    from skimage.util import view_as_windows
#    pmap = np.zeros((len(PROPNAMES)+1, img.shape[0]/win_step+1,
#        img.shape[1]/win_step+1))
#    windows = view_as_windows(img, (5,5), step=win_step)
#    #props = coprops(windows.reshape(-1,5,5))
#    #props.shape = (windows.shape[0], windows.shape[1], len(PROPNAMES))
#    for i, row in enumerate(windows):
#        for j, win in enumerate(row):
#            if win.min() > 0:
#                v = coprops([win])[0]
#            else:
#                v = 0
#            pmap[0,i,j] = np.median(win)
#            pmap[1:,i,j] = v
#    return pmap

def haralick(img):
    """Haralick texture features (14) averaged over directions."""
    import mahotas
    a = mahotas.features.texture.haralick(img, compute_14th_feature=True)
    a = np.mean(a, axis=0)
    return a, mahotas.features.texture.haralick_labels

def haralick_map(img, winsize, mask=None, output=None):
    """Haralick texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats, names = haralick(win)
        if output is None:
            output = np.zeros((len(names),) + img.shape)
        for i, v in enumerate(feats):
            output[(i,) + pos] = v
    return output, names

# Local Binary Pattern (LBP) features

def lbp_freqs(img, winsize=3, neighbours=8, radius=1, roinv=1, uniform=1):
    """Calculate local binary pattern (LBP) frequency map."""
    import lbp
    lbp_data = lbp.lbp(img, neighbours, radius, roinv, uniform)
    lbp_freq_data, n_patterns = lbp.get_freqs(lbp_data, winsize, neighbours,
            roinv, uniform)
    return lbp_data, lbp_freq_data, n_patterns

def lbp_freq_map(img, winsize=3, neighbours=8, radius=1):
    """Local binary pattern (LBP) frequency histogram map."""
    _, freqs, n = lbp_freqs(img, winsize, neighbours, radius)
    output = np.rollaxis(freqs, -1)
    names = map(str, range(n))
    return output, names

def lbpf_dist(hist1, hist2, method='chi-squared', eps=1e-6):
    """Measure the distance of two LBP frequency histograms.

    Method can be one of the following:
    intersection: histogram intersection
    log-likelihood: log-likelihood
    chi-squared: chi-squared
    """
    pairs = np.array([hist1, hist2]).T
    if method == 'intersection':
        r = sum(min(pair) for pair in pairs)
    elif method == 'log-likelihood':
        r = -sum(a*np.log(max(b, eps)) for a, b in pairs)
    elif method == 'chi-squared':
        r = sum((a-b)**2/(max(a+b, eps)) for a, b in pairs)
    else:
        raise Exception('Unknown distance measure: %s' % method)
    return r

# Gabor features

def gabor(img, sigmas=[1, 3], freqs=[0.1, 0.25, 0.4]):
    """Gabor features averaged over directions."""
    thetas = [np.pi/4*i for i in range(4)]
    shape = len(thetas), len(sigmas), len(freqs)
    feats = np.zeros(shape + (2,), dtype=np.double)
    for i, j, k in np.ndindex(shape):
        t, s, f = thetas[i], sigmas[j], freqs[k]
        kwargs = dict(frequency=f, theta=t, sigma_x=s, sigma_y=s)
        real, _ = skimage.filter.gabor_filter(img, **kwargs)
        feats[i,j,k,0] = real.mean()
        feats[i,j,k,1] = real.var()
    feats = np.mean(feats, axis=0) # Average over directions.
    d = collections.OrderedDict()
    for (i, j, k), value in np.ndenumerate(feats):
        key = sigmas[i], freqs[j], ('mean', 'var')[k]
        d[key] = value
    return d

def gabor_map(img, winsize, sigmas=[1, 3], freqs=[0.1, 0.25, 0.4], mask=None, output=None):
    """Gabor texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats = gabor(win, sigmas, freqs)
        if output is None:
            output = np.zeros((len(feats),) + img.shape)
        for i, v in enumerate(feats.values()):
            output[(i,) + pos] = v
    names = map(str, feats.keys())
    return output, names

# Histogram of Oriented Gradients (HOG)

def hog(img):
    # TODO Average over directions
    return skimage.feature.hog(img, orientations=2, pixels_per_cell=(2,2),
            cells_per_block=(2,2), normalise=True)

def hog_map(img, winsize, mask=None, output=None):
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats = hog(win)
        if output is None:
            output = np.zeros((len(feats),) + img.shape)
        for i, v in enumerate(feats):
            output[(i,) + pos] = v
    names = map(str, range(len(feats)))
    return output, names

# Image moments

def moment(img, p, q):
    """Image moment. See Tuceryan 1994: Moment Based Texture Segmentation."""
    img = np.asarray(img)
    if img.ndim != 2 or img.shape[0] != img.shape[1]:
        raise Exception('Image not square: {}'.format(img.shape))
    width = img.shape[0]
    center = width//2
    nc = lambda pos: (pos-center)/(width/2) # Normalized coordinates [-1,1]
    f = lambda m, n: img[m,n] * nc(m)**p * nc(n)**q
    a = np.fromfunction(f, img.shape, dtype=np.int)
    moment = a.sum()
    return moment

def moments(img, max_order=2):
    """Image moments of order up to p+q <= max_order."""
    r = range(max_order+1)
    tuples = (t for t in itertools.product(r, r) if sum(t) <= max_order)
    d = collections.OrderedDict(((p, q), moment(img, p, q)) for p, q in tuples)
    return d

def moment_map(img, winsize, max_order=2, mask=None, output=None):
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats = moments(win, max_order=max_order)
        if output is None:
            output = np.zeros((len(feats),) + img.shape)
        for i, v in enumerate(feats.values()):
            output[(i,) + pos] = v
    names = map(str, feats.keys())
    return output, names

# Haar transformation

def haar(img):
    """Haar"""
    import mahotas
    assert img.ndim == 2
    #assert img.shape[0] % 2 == img.shape[1] % 2 == 0
    # Prune possible odd borders.
    newshape = [x-x%2 for x in img.shape]
    img = img[:newshape[0], :newshape[1]]
    a = mahotas.haar(img)
    h, w = [x//2 for x in a.shape]
    levels = np.array([
            a[:h,:w], a[:h,w:],
            a[h:,:w], a[h:,w:],
            ])
    return levels

def haar_level_features(win):
    """Haar features of a single level."""
    d = collections.OrderedDict()
    d['aav'] = np.mean(np.abs(win))
    d['std'] = np.std(win)
    # TODO: Uses only 4 corner pixels.
    a = win[0,:] - win[-1,:]
    d['vert'] = np.mean(np.abs(a))
    a = win[:,0] - win[:,-1]
    d['horz'] = np.mean(np.abs(a))
    a = [win[0,0] - win[-1,-1], win[0,-1] - win[-1,0]]
    d['diag'] = np.mean(np.abs(a))
    return d

def haar_features(img):
    """Haar features."""
    levels = haar(img)
    d = collections.OrderedDict()
    for i, level in enumerate(levels):
        feats = haar_level_features(level)
        for k, v in feats.iteritems():
            d[(i, k)] = v
    return d

def haar_map(img, winsize, zoom=True, output=None):
    """Haar texture feature map."""
    levels = haar(img)
    names = []
    for i, level in enumerate(levels):
        for pos, win in dwi.util.sliding_window(level, winsize):
            feats = haar_level_features(win)
            if output is None:
                output = np.zeros((len(levels)*len(feats),) + level.shape)
            for j, v in enumerate(feats.values()):
                output[(i*len(feats)+j,) + pos] = v
        names += ['%i,%s' % (i, k) for k in feats.keys()]
    if zoom:
        # Zoom to original size.
        output = sp.ndimage.interpolation.zoom(output, (1,2,2))
    return output, names
<commit_msg>Haar map mask<commit_after>"""Texture analysis."""

from __future__ import division
import collections
import itertools

import numpy as np
import scipy as sp
import skimage
import skimage.feature

import dwi.util

# Basic statistical features

def stats(img):
    """Get basic statistical features."""
    d = collections.OrderedDict()
    d['mean'] = np.mean(img)
    d['stddev'] = np.std(img)
    d['range'] = np.max(img) - np.min(img)
    d.update(dwi.util.fivenumd(img))
    for i in range(1, 10):
        d['decile%i' % i] = sp.stats.scoreatpercentile(img, i*10)
    d['kurtosis'] = sp.stats.kurtosis(img.ravel())
    d['skewness'] = sp.stats.skew(img.ravel())
    return d

def stats_map(img, winsize, names=None, mask=None, output=None):
    """Get basic statistical texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        d = stats(win)
        if names is None:
            names = d.keys()
        if output is None:
            output = np.zeros((len(names),) + img.shape)
        for i, name in enumerate(names):
            output[(i,) + pos] = d[name]
    return output, names

# Grey-Level Co-Occurrence Matrix (GLCM) features

PROPNAMES = 'contrast dissimilarity homogeneity energy correlation ASM'.split()

def glcm_props(img, names=PROPNAMES):
    """Get grey-level co-occurrence matrix texture properties over an image."""
    from skimage.feature import greycomatrix, greycoprops
    distances = [1]
    angles = [0, np.pi/4, np.pi/2, 3*np.pi/4]
    levels = 256
    glcm = greycomatrix(img, distances, angles, levels, symmetric=True,
            normed=True)
    keys = names
    values = [np.mean(greycoprops(glcm, p)) for p in names]
    d = collections.OrderedDict((k, v) for k, v in zip(keys, values))
    return d

def glcm_map(img, winsize, names=PROPNAMES, mask=None, output=None):
    """Grey-level co-occurrence matrix texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        d = glcm_props(win, names)
        if output is None:
            output = np.zeros((len(names),) + img.shape)
        for i, name in enumerate(names):
            output[(i,) + pos] = d[name]
    return output, names

#def coprops(windows):
#    # XXX Obsolete
#    from skimage.feature import greycomatrix, greycoprops
#    props = np.zeros((len(windows), len(PROPNAMES)))
#    for i, win in enumerate(windows):
#        #win = skimage.img_as_ubyte(win)
#        #if win.min() == 0:
#        #    props[i].fill(0)
#        #    continue
#        distances = [1]
#        angles = [0, np.pi/4, np.pi/2, 3*np.pi/4]
#        glcm = greycomatrix(win, distances, angles, 256, symmetric=True,
#                normed=True)
#        for j, propname in enumerate(PROPNAMES):
#            a = greycoprops(glcm, propname)
#            props[i,j] = np.mean(a)
#    return props
#
#def texture_pmap(img, win_step):
#    # XXX Obsolete
#    from skimage.util import view_as_windows
#    pmap = np.zeros((len(PROPNAMES)+1, img.shape[0]/win_step+1,
#        img.shape[1]/win_step+1))
#    windows = view_as_windows(img, (5,5), step=win_step)
#    #props = coprops(windows.reshape(-1,5,5))
#    #props.shape = (windows.shape[0], windows.shape[1], len(PROPNAMES))
#    for i, row in enumerate(windows):
#        for j, win in enumerate(row):
#            if win.min() > 0:
#                v = coprops([win])[0]
#            else:
#                v = 0
#            pmap[0,i,j] = np.median(win)
#            pmap[1:,i,j] = v
#    return pmap

def haralick(img):
    """Haralick texture features (14) averaged over directions."""
    import mahotas
    a = mahotas.features.texture.haralick(img, compute_14th_feature=True)
    a = np.mean(a, axis=0)
    return a, mahotas.features.texture.haralick_labels

def haralick_map(img, winsize, mask=None, output=None):
    """Haralick texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats, names = haralick(win)
        if output is None:
            output = np.zeros((len(names),) + img.shape)
        for i, v in enumerate(feats):
            output[(i,) + pos] = v
    return output, names

# Local Binary Pattern (LBP) features

def lbp_freqs(img, winsize=3, neighbours=8, radius=1, roinv=1, uniform=1):
    """Calculate local binary pattern (LBP) frequency map."""
    import lbp
    lbp_data = lbp.lbp(img, neighbours, radius, roinv, uniform)
    lbp_freq_data, n_patterns = lbp.get_freqs(lbp_data, winsize, neighbours,
            roinv, uniform)
    return lbp_data, lbp_freq_data, n_patterns

def lbp_freq_map(img, winsize=3, neighbours=8, radius=1):
    """Local binary pattern (LBP) frequency histogram map."""
    _, freqs, n = lbp_freqs(img, winsize, neighbours, radius)
    output = np.rollaxis(freqs, -1)
    names = map(str, range(n))
    return output, names

def lbpf_dist(hist1, hist2, method='chi-squared', eps=1e-6):
    """Measure the distance of two LBP frequency histograms.

    Method can be one of the following:
    intersection: histogram intersection
    log-likelihood: log-likelihood
    chi-squared: chi-squared
    """
    pairs = np.array([hist1, hist2]).T
    if method == 'intersection':
        r = sum(min(pair) for pair in pairs)
    elif method == 'log-likelihood':
        r = -sum(a*np.log(max(b, eps)) for a, b in pairs)
    elif method == 'chi-squared':
        r = sum((a-b)**2/(max(a+b, eps)) for a, b in pairs)
    else:
        raise Exception('Unknown distance measure: %s' % method)
    return r

# Gabor features

def gabor(img, sigmas=[1, 3], freqs=[0.1, 0.25, 0.4]):
    """Gabor features averaged over directions."""
    thetas = [np.pi/4*i for i in range(4)]
    shape = len(thetas), len(sigmas), len(freqs)
    feats = np.zeros(shape + (2,), dtype=np.double)
    for i, j, k in np.ndindex(shape):
        t, s, f = thetas[i], sigmas[j], freqs[k]
        kwargs = dict(frequency=f, theta=t, sigma_x=s, sigma_y=s)
        real, _ = skimage.filter.gabor_filter(img, **kwargs)
        feats[i,j,k,0] = real.mean()
        feats[i,j,k,1] = real.var()
    feats = np.mean(feats, axis=0) # Average over directions.
    d = collections.OrderedDict()
    for (i, j, k), value in np.ndenumerate(feats):
        key = sigmas[i], freqs[j], ('mean', 'var')[k]
        d[key] = value
    return d

def gabor_map(img, winsize, sigmas=[1, 3], freqs=[0.1, 0.25, 0.4], mask=None, output=None):
    """Gabor texture feature map."""
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats = gabor(win, sigmas, freqs)
        if output is None:
            output = np.zeros((len(feats),) + img.shape)
        for i, v in enumerate(feats.values()):
            output[(i,) + pos] = v
    names = map(str, feats.keys())
    return output, names

# Histogram of Oriented Gradients (HOG)

def hog(img):
    # TODO Average over directions
    return skimage.feature.hog(img, orientations=2, pixels_per_cell=(2,2),
            cells_per_block=(2,2), normalise=True)

def hog_map(img, winsize, mask=None, output=None):
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats = hog(win)
        if output is None:
            output = np.zeros((len(feats),) + img.shape)
        for i, v in enumerate(feats):
            output[(i,) + pos] = v
    names = map(str, range(len(feats)))
    return output, names

# Image moments

def moment(img, p, q):
    """Image moment. See Tuceryan 1994: Moment Based Texture Segmentation."""
    img = np.asarray(img)
    if img.ndim != 2 or img.shape[0] != img.shape[1]:
        raise Exception('Image not square: {}'.format(img.shape))
    width = img.shape[0]
    center = width//2
    nc = lambda pos: (pos-center)/(width/2) # Normalized coordinates [-1,1]
    f = lambda m, n: img[m,n] * nc(m)**p * nc(n)**q
    a = np.fromfunction(f, img.shape, dtype=np.int)
    moment = a.sum()
    return moment

def moments(img, max_order=2):
    """Image moments of order up to p+q <= max_order."""
    r = range(max_order+1)
    tuples = (t for t in itertools.product(r, r) if sum(t) <= max_order)
    d = collections.OrderedDict(((p, q), moment(img, p, q)) for p, q in tuples)
    return d

def moment_map(img, winsize, max_order=2, mask=None, output=None):
    for pos, win in dwi.util.sliding_window(img, winsize, mask=mask):
        feats = moments(win, max_order=max_order)
        if output is None:
            output = np.zeros((len(feats),) + img.shape)
        for i, v in enumerate(feats.values()):
            output[(i,) + pos] = v
    names = map(str, feats.keys())
    return output, names

# Haar transformation

def haar(img):
    """Haar"""
    import mahotas
    assert img.ndim == 2
    #assert img.shape[0] % 2 == img.shape[1] % 2 == 0
    # Prune possible odd borders.
    newshape = [x-x%2 for x in img.shape]
    img = img[:newshape[0], :newshape[1]]
    a = mahotas.haar(img)
    h, w = [x//2 for x in a.shape]
    levels = np.array([
            a[:h,:w], a[:h,w:],
            a[h:,:w], a[h:,w:],
            ])
    return levels

def haar_level_features(win):
    """Haar features of a single level."""
    d = collections.OrderedDict()
    d['aav'] = np.mean(np.abs(win))
    d['std'] = np.std(win)
    # TODO: Uses only 4 corner pixels.
    a = win[0,:] - win[-1,:]
    d['vert'] = np.mean(np.abs(a))
    a = win[:,0] - win[:,-1]
    d['horz'] = np.mean(np.abs(a))
    a = [win[0,0] - win[-1,-1], win[0,-1] - win[-1,0]]
    d['diag'] = np.mean(np.abs(a))
    return d

def haar_features(img):
    """Haar features."""
    levels = haar(img)
    d = collections.OrderedDict()
    for i, level in enumerate(levels):
        feats = haar_level_features(level)
        for k, v in feats.iteritems():
            d[(i, k)] = v
    return d

def haar_map(img, winsize, mask=None, output=None):
    """Haar texture feature map."""
    levels = haar(img)
    levels = sp.ndimage.interpolation.zoom(levels, (1,2,2))
    names = []
    for i, level in enumerate(levels):
        for pos, win in dwi.util.sliding_window(level, winsize, mask):
            feats = haar_level_features(win)
            if output is None:
                output = np.zeros((len(levels)*len(feats),) + level.shape)
            for j, v in enumerate(feats.values()):
                output[(i*len(feats)+j,) + pos] = v
        names += ['%i,%s' % (i, k) for k in feats.keys()]
    return output, names
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Algebraic connectivity and Fiedler vectors of undirected graphs.
"""

__author__ = """ysitu <ysitu@users.noreply.github.com>"""
# Copyright (C) 2014 ysitu <ysitu@users.noreply.github.com>
# All rights reserved.
# BSD license.

import networkx as nx
from networkx.utils import not_implemented_for
from random import normalvariate

try:
    from numpy import (array, asmatrix, asarray, dot, matrix, ndarray, reshape,
                       zeros)
    from numpy.linalg import norm
    from scipy.linalg import eigh, solve
    from scipy.sparse import csc_matrix
    from scipy.sparse.linalg import eigsh
    __all__ = ['algebraic_connectivity', 'fiedler_vector']
except ImportError:
    __all__ = []

class _PCGSolver(object):
    """Jacobi-preconditioned conjugate gradient method.
    """

    def __init__(self, A, tol):
        self._A = A
        self._M = 1. / A.diagonal()
        self._tol = tol

    def solve(self, b):
        A = self._A
        M = self._M
        tol = self._tol * norm(b, 1)
        # Initialize.
        x = zeros(b.shape)
        r = b.copy()
        z = M * r
        rz = dot(r, z)
        p = z
        # Iterate.
        while True:
            Ap = A * p
            alpha = rz / dot(p, Ap)
            x += alpha * p
            r -= alpha * Ap
            if norm(r, 1) < tol:
                return x
            z = M * r
            beta = dot(r, z)
            beta, rz = beta / rz, beta
            p = z + beta * p

class _CholeskySolver(object):
    """Cholesky factorization.
    """

    def __init__(self, A):
        A = csc_matrix(A, dtype=float)
        chol = self._analyze(A)
        perm = chol.P()
        A[perm[-1], perm[-1]] = float('inf')
        chol.cholesky_inplace(A)
        self._chol = chol

    def solve(self, b):
        return self._chol(b)

    try:
        from scikits.sparse.cholmod import analyze
        _analyze = analyze
        avail = True
    except ImportError:
        avail = False

def _tracemin_fiedler(L, tol, solver=None):
    """Compute the Fiedler vector of L using the TRACEMIN-Fiedler algorithm.
    """
    n = L.shape[0]
    q = 2

    Lnorm = max(asarray(abs(L).sum(axis=0))[0, :])

    def project(X):
        """Make X orthogonal to the nullspace of L.
        """
        X = asarray(X)
        for j in range(q):
            X[:, j] -= sum(X[:, j]) / n

    def gram_schmidt(X):
        """Return an orthonormalized copy of X.
        """
        V = asarray(X.copy())
        for j in range(q):
            for k in range(j):
                V[:, j] -= dot(V[:, k], V[:, j]) * V[:, k]
            V[:, j] /= norm(V[:, j])
        return asmatrix(V)

    if solver is None:
        solver = 'pcg' if not _CholeskySolver.avail else 'chol'
    if solver == 'pcg':
        solver = _PCGSolver(L, tol / 10)
    elif solver == 'chol':
        if not _CholeskySolver.avail:
            raise nx.NetworkXError('Cholesky solver unavailable.')
        solver = _CholeskySolver(L)
    else:
        raise nx.NetworkXError('unknown solver.')

    # Randomly initialize eigenvectors.
    X = asmatrix(reshape([normalvariate(0, 1) for i in range(n * q)], (n, q),
                         order='F'))
    project(X)
    W = asmatrix(ndarray((n, q), order='F'))

    first = True
    while True:
        V = gram_schmidt(X)
        # Compute interation matrix H.
        H = V.transpose() * L * V
        sigma, Y = eigh(H, overwrite_a=True)
        # Test for convergence.
        if not first:
            r = L * X[:, 0] - sigma[0] * X[:, 0]
            if norm(r, 1) < tol * Lnorm:
                break
        else:
            first = False
        # Compute Ritz vectors.
        X = V * Y
        # Solve saddle point problem using the Schur complement.
        for j in range(q):
            asarray(W)[:, j] = solver.solve(asarray(X)[:, j])
        project(W)
        S = X.transpose() * W  # Schur complement
        N = solve(S, X.transpose() * X, overwrite_a=True, overwrite_b=True)
        X = W * N

    return sigma, asarray(X)

@not_implemented_for('directed')
def algebraic_connectivity(G, weight='weight', tol=1e-8, method='tracemin'):
    """Return the algebraic connectivity of a connected undirected graph.

    The algebraic connectivity of a connected undirected graph is the second
    smallest eigenvalue of its Laplacian matrix.

    Parameters
    ----------
    G : NetworkX graph
        An undirected graph.

    weight : string or None
        The data key used to determine the weight of each edge. If None, then
        each edge has unit weight. Default value: None.

    tol : float
        Tolerance of eigenvalue computation. Default value: 1e-8.

    method : string
        Method of eigenvalue computation. It should be either 'tracemin'
        (TraceMIN) or 'arnoldi' (Arnoldi iteration). Default value: 'tracemin'.

    Returns
    -------
    aconn : float
        Algebraic connectivity.

    Raises
    ------
    NetworkXNotImplemented :
        If G is directed.

    NetworkXError :
        If G as less than two nodes or is not connected.

    Notes
    -----
    For MultiGraph, the edge weights are summed. Zero-weighted edges are
    ignored. If there are negative-weighted edges, the algorithm not work.

    See Also
    --------
    laplacian_matrix
    """
    if len(G) < 2:
        raise nx.NetworkXError('graph has less than two nodes.')
    if not nx.is_connected(G):
        raise nx.NetworkXError('graph is not connected.')

    L = nx.laplacian_matrix(G, weight=weight)
    if any(L.diagonal() == 0):
        raise nx.NetworkXError('graph is not connected.')

    if len(G) == 2:
        return 2. * L[0, 0]

    if method == 'tracemin':
        sigma, X = _tracemin_fiedler(L, tol)
        return sigma[0]
    elif method == 'tracemin_pcg':
        sigma, X = _tracemin_fiedler(L, tol, 'pcg')
        return sigma[0]
    elif method == 'tracemin_chol':
        sigma, X = _tracemin_fiedler(L, tol, 'chol')
        return sigma[0]
    elif method == 'arnoldi':
        L = csc_matrix(L, dtype=float)
        sigma = eigsh(L, 2, which='SM', tol=tol, return_eigenvectors=False)
        return sigma[0]
    else:
        raise nx.NetworkXError(
            "method should be either 'tracemin' or 'arnoldi'.")

@not_implemented_for('directed')
def fiedler_vector(G, weight='weight', tol=1e-8, method='tracemin'):
    """Return the Fiedler vector of a connected undirected graph.

    The Fiedler vector of a connected undirected graph is the eigenvector
    corresponding to the second smallest eigenvalue of the Laplacian matrix of
    of the graph.

    Parameters
    ----------
    G : NetworkX graph
        An undirected graph.

    weight : string or None
        The data key used to determine the weight of each edge. If None, then
        each edge has unit weight. Default value: None.

    tol : float
        Tolerance of relative residual in eigenvalue computation. Default
        value: 1e-8.

    method : string
        Method of eigenvalue computation. It should be either 'tracemin'
        (TraceMIN) or 'arnoldi' (Arnoldi iteration). Default value: 'tracemin'.

    Returns
    -------
    aconn : float
        Algebraic connectivity.

    Raises
    ------
    NetworkXNotImplemented :
        If G is directed.

    NetworkXError :
        If G as less than two nodes or is not connected.

    Notes
    -----
    For MultiGraph, the edge weights are summed. Zero-weighted edges are
    ignored. If there are negative-weighted edges, the algorithm not work.

    See Also
    --------
    laplacian_matrix
    """
    if len(G) < 2:
        raise nx.NetworkXError('graph has less than two nodes.')
    if not nx.is_connected(G):
        raise nx.NetworkXError('graph is not connected.')

    L = nx.laplacian_matrix(G, weight=weight)
    if any(L.diagonal() == 0):
        raise nx.NetworkXError('graph is not connected.')

    if len(G) == 2:
        return array([1., -1.])

    if method == 'tracemin':
        sigma, X = _tracemin_fiedler(L, tol)
        return array(X[:, 0])
    elif method == 'tracemin_pcg':
        sigma, X = _tracemin_fiedler(L, tol, 'pcg')
        return array(X[:, 0])
    elif method == 'tracemin_chol':
        sigma, X = _tracemin_fiedler(L, tol, 'chol')
        return array(X[:, 0])
    elif method == 'arnoldi':
        L = csc_matrix(L, dtype=float)
        sigma, X = eigsh(L, 2, which='SM', tol=tol)
        return array(X[:, 1])
    else:
        raise nx.NetworkXError(
            "method should be either 'tracemin' or 'arnoldi'.")

# fixture for nose tests
def setup_module(module):
    from nose import SkipTest
    try:
        import numpy
        import scipy.sparse
    except ImportError:
        raise SkipTest('SciPy not available.')
<commit_msg>Add/fix comments in algebraic_connectivity<commit_after># -*- coding: utf-8 -*-
"""
Algebraic connectivity and Fiedler vectors of undirected graphs.
"""

__author__ = """ysitu <ysitu@users.noreply.github.com>"""
# Copyright (C) 2014 ysitu <ysitu@users.noreply.github.com>
# All rights reserved.
# BSD license.

import networkx as nx
from networkx.utils import not_implemented_for
from random import normalvariate

try:
    from numpy import (array, asmatrix, asarray, dot, matrix, ndarray, reshape,
                       zeros)
    from numpy.linalg import norm
    from scipy.linalg import eigh, solve
    from scipy.sparse import csc_matrix
    from scipy.sparse.linalg import eigsh
    __all__ = ['algebraic_connectivity', 'fiedler_vector']
except ImportError:
    __all__ = []

class _PCGSolver(object):
    """Jacobi-preconditioned conjugate gradient method.
    """

    def __init__(self, A, tol):
        self._A = A
        self._M = 1. / A.diagonal()
        self._tol = tol

    def solve(self, b):
        A = self._A
        M = self._M
        tol = self._tol * norm(b, 1)
        # Initialize.
        x = zeros(b.shape)
        r = b.copy()
        z = M * r
        rz = dot(r, z)
        p = z
        # Iterate.
        while True:
            Ap = A * p
            alpha = rz / dot(p, Ap)
            x += alpha * p
            r -= alpha * Ap
            if norm(r, 1) < tol:
                return x
            z = M * r
            beta = dot(r, z)
            beta, rz = beta / rz, beta
            p = z + beta * p

class _CholeskySolver(object):
    """Cholesky factorization.
    """

    def __init__(self, A):
        A = csc_matrix(A, dtype=float, copy=True)
        # Compute an unsymmetric reordering.
        chol = self._analyze(A)
        perm = chol.P()
        # Set the lower-right element after permutation to infinity.
        A[perm[-1], perm[-1]] = float('inf')
        chol.cholesky_inplace(A)
        self._chol = chol

    def solve(self, b):
        return self._chol(b)

    try:
        from scikits.sparse.cholmod import analyze
        _analyze = analyze
        avail = True
    except ImportError:
        avail = False

def _tracemin_fiedler(L, tol, solver=None):
    """Compute the Fiedler vector of L using the TRACEMIN-Fiedler algorithm.
    """
    n = L.shape[0]
    q = 2

    Lnorm = abs(L).sum(axis=0).flatten().max()

    def project(X):
        """Make X orthogonal to the nullspace of L.
        """
        X = asarray(X)
        for j in range(q):
            X[:, j] -= sum(X[:, j]) / n

    def gram_schmidt(X):
        """Return an orthonormalized copy of X.
        """
        V = asarray(X.copy())
        for j in range(q):
            for k in range(j):
                V[:, j] -= dot(V[:, k], V[:, j]) * V[:, k]
            V[:, j] /= norm(V[:, j])
        return asmatrix(V)

    if solver is None:
        solver = 'pcg' if not _CholeskySolver.avail else 'chol'
    if solver == 'pcg':
        solver = _PCGSolver(L, tol / 10)
    elif solver == 'chol':
        if not _CholeskySolver.avail:
            raise nx.NetworkXError('Cholesky solver unavailable.')
        solver = _CholeskySolver(L)
    else:
        raise nx.NetworkXError('unknown solver.')

    # Randomly initialize eigenvectors.
    X = asmatrix(reshape([normalvariate(0, 1) for i in range(n * q)], (n, q),
                         order='F'))
    project(X)
    W = asmatrix(ndarray((n, q), order='F'))

    first = True
    while True:
        V = gram_schmidt(X)
        # Compute interation matrix H.
        H = V.transpose() * L * V
        sigma, Y = eigh(H, overwrite_a=True)
        # Test for convergence.
        if not first:
            r = L * X[:, 0] - sigma[0] * X[:, 0]
            if norm(r, 1) < tol * Lnorm:
                break
        else:
            first = False
        # Compute Ritz vectors.
        X = V * Y
        # Solve saddle point problem using the Schur complement.
        for j in range(q):
            asarray(W)[:, j] = solver.solve(asarray(X)[:, j])
        project(W)
        S = X.transpose() * W  # Schur complement
        N = solve(S, X.transpose() * X, overwrite_a=True, overwrite_b=True)
        X = W * N

    return sigma, asarray(X)

@not_implemented_for('directed')
def algebraic_connectivity(G, weight='weight', tol=1e-8, method='tracemin'):
    """Return the algebraic connectivity of a connected undirected graph.

    The algebraic connectivity of a connected undirected graph is the second
    smallest eigenvalue of its Laplacian matrix.

    Parameters
    ----------
    G : NetworkX graph
        An undirected graph.

    weight : string or None
        The data key used to determine the weight of each edge. If None, then
        each edge has unit weight. Default value: None.

    tol : float
        Tolerance of eigenvalue computation. Default value: 1e-8.

    method : string
        Method of eigenvalue computation. It should be either 'tracemin'
        (TraceMIN) or 'arnoldi' (Arnoldi iteration). Default value: 'tracemin'.

    Returns
    -------
    aconn : float
        Algebraic connectivity.

    Raises
    ------
    NetworkXNotImplemented :
        If G is directed.

    NetworkXError :
        If G as less than two nodes or is not connected.

    Notes
    -----
    For MultiGraph, the edge weights are summed. Zero-weighted edges are
    ignored. If there are negative-weighted edges, the algorithm may not work.

    See Also
    --------
    laplacian_matrix
    """
    if len(G) < 2:
        raise nx.NetworkXError('graph has less than two nodes.')
    if not nx.is_connected(G):
        raise nx.NetworkXError('graph is not connected.')

    L = nx.laplacian_matrix(G, weight=weight)
    if any(L.diagonal() == 0):
        raise nx.NetworkXError('graph is not connected.')

    if len(G) == 2:
        return 2. * L[0, 0]

    if method == 'tracemin':
        sigma, X = _tracemin_fiedler(L, tol)
        return sigma[0]
    elif method == 'tracemin_pcg':
        sigma, X = _tracemin_fiedler(L, tol, 'pcg')
        return sigma[0]
    elif method == 'tracemin_chol':
        sigma, X = _tracemin_fiedler(L, tol, 'chol')
        return sigma[0]
    elif method == 'arnoldi':
        L = csc_matrix(L, dtype=float)
        sigma = eigsh(L, 2, which='SM', tol=tol, return_eigenvectors=False)
        return sigma[0]
    else:
        raise nx.NetworkXError(
            "method should be either 'tracemin' or 'arnoldi'.")

@not_implemented_for('directed')
def fiedler_vector(G, weight='weight', tol=1e-8, method='tracemin'):
    """Return the Fiedler vector of a connected undirected graph.

    The Fiedler vector of a connected undirected graph is the eigenvector
    corresponding to the second smallest eigenvalue of the Laplacian matrix of
    of the graph.

    Parameters
    ----------
    G : NetworkX graph
        An undirected graph.

    weight : string or None
        The data key used to determine the weight of each edge. If None, then
        each edge has unit weight. Default value: None.

    tol : float
        Tolerance of relative residual in eigenvalue computation. Default
        value: 1e-8.

    method : string
        Method of eigenvalue computation. It should be either 'tracemin'
        (TraceMIN) or 'arnoldi' (Arnoldi iteration). Default value: 'tracemin'.

    Returns
    -------
    aconn : float
        Algebraic connectivity.

    Raises
    ------
    NetworkXNotImplemented :
        If G is directed.

    NetworkXError :
        If G as less than two nodes or is not connected.

    Notes
    -----
    For MultiGraph, the edge weights are summed. Zero-weighted edges are
    ignored. If there are negative-weighted edges, the algorithm may not work.

    See Also
    --------
    laplacian_matrix
    """
    if len(G) < 2:
        raise nx.NetworkXError('graph has less than two nodes.')
    if not nx.is_connected(G):
        raise nx.NetworkXError('graph is not connected.')

    L = nx.laplacian_matrix(G, weight=weight)
    if any(L.diagonal() == 0):
        raise nx.NetworkXError('graph is not connected.')

    if len(G) == 2:
        return array([1., -1.])

    if method == 'tracemin':
        sigma, X = _tracemin_fiedler(L, tol)
        return array(X[:, 0])
    elif method == 'tracemin_pcg':
        sigma, X = _tracemin_fiedler(L, tol, 'pcg')
        return array(X[:, 0])
    elif method == 'tracemin_chol':
        sigma, X = _tracemin_fiedler(L, tol, 'chol')
        return array(X[:, 0])
    elif method == 'arnoldi':
        L = csc_matrix(L, dtype=float)
        sigma, X = eigsh(L, 2, which='SM', tol=tol)
        return array(X[:, 1])
    else:
        raise nx.NetworkXError(
            "method should be either 'tracemin' or 'arnoldi'.")

# fixture for nose tests
def setup_module(module):
    from nose import SkipTest
    try:
        import numpy
        import scipy.sparse
    except ImportError:
        raise SkipTest('SciPy not available.')
<|endoftext|>