<commit_before>"""Operations governing the creation and destruction of user facing alerts."""
import binascii
import boto3
import datetime
import json
import logging
import os
import requests

from boto3.dynamodb.conditions import Attr
from faker import Faker

fake = Faker()
logger = logging.getLogger(__name__)

class Feedback(object):
    """Send user data back to MozDef or other subscribers via an SNS Topic"""
    def __init__(self, alert_dict, alert_action):
        self.alert_dict = alert_dict
        self.alert_action = alert_action
        self.ssm = None
        self.sns = None
        self.sns_topic_arn = self.get_sns_arn()

    def connect_sns(self):
        if self.sns is None:
            self.sns = boto3.client('sns', region_name='us-west-2')

    def connect_ssm(self):
        if self.ssm is None:
            self.ssm = boto3.client('ssm', region_name='us-west-2')

    def get_sns_arn(self):
        self.connect_ssm()

        response = self.ssm.get_parameter(
            Name='sso-dashboard-alerts-sns',
            WithDecryption=False
        )

        return response.get('Parameter').get('Value')

    def _construct_alert(self):
        message = {
            'category': 'user_feedback',
            'details': {
                'action': self.alert_action,  # (escalate|acknowledge|false-positive)
                'alert_information': self.alert_dict
            }
        }

        return message

    def send(self):
        """Send the event to the SNS Topic."""
        self.connect_sns()
        message = self._construct_alert()
        response = self.sns.publish(
            TopicArn=self.sns_topic_arn,
            Message=json.dumps(message),
            Subject='sso-dashboard-user-feedback'
        )
        return response

class Alert(object):
    """Primary object containing alert functions."""
    def __init__(self):
        self.alert_table_name = 'sso-dashboard-alert'
        self.dynamodb = None

    def has_actions(self, alert_dict):
        """Let's the view know if it should render actions for the alert."""

        # Whitelist the firefox out of date alert.  It should not get buttons.
        if self.alert_dict.get('alert_code') is not '63f675d8896f4fb2b3caa204c8c2761e':
            return True
        else:
            return False

    def has_escalation(self, alert_dict):
        """Let's the view know if it should render actions for the alert."""

        # Whitelist the firefox out of date alert.  It should not get buttons.
        if self.alert_dict.get('alert_code') is not '63f675d8896f4fb2b3caa204c8c2761e':
            return True
        else:
            return False

    def connect_dynamodb(self):
        if self.dynamodb is None:
            dynamodb = boto3.resource('dynamodb', region_name='us-west-2')
            table = dynamodb.Table(self.alert_table_name)
            self.dynamodb = table

    def find_or_create_by(self, alert_dict, user_id):
        """
        :param alert_dict: takes a dictionary of alert information
        :param user_id: the session info user_id
        :return: dynamodb response or none
        """

        # Search for the user alerts
        current_alerts = self.find(user_id)

        # If the alert is duplicate false do not create another instance of it.
        for alert in current_alerts:
            try:
                if alert.get('alert_code') == alert_dict.get('alert_code') and alert_dict.get('duplicate') is False:
                    return None
                else:
                    continue
            except AttributeError as e:
                logger.error('Bad data in alerts table for user: {}, exception was {}'.format(user_id, e))

        # Else create another alert.
        return self.create(alert_dict)

    def create(self, alert_dict):
        """
        Create an alert.
        :param alert_json: takes a dictionary of alert information
        :return: dynamodb response
        """
        self.connect_dynamodb()

        alert_dict['alert_id'] = self._create_alert_id()
        response = self.dynamodb.put_item(
            Item=alert_dict
        )

        return response

    def destroy(self, alert_id, user_id):
        """
        Delete an alert.
        :param alert_id: Primary key of the alert to destroy.
        :return: status_code
        """
        self.connect_dynamodb()

        response = self.dynamodb.delete_item(
            Key={
                'alert_id': alert_id,
                'user_id': user_id
            }
        )

        return response

    def update(self, alert_id, alert_dict):
        """
        Update an alert record for a user.
        :param alert_id: Primary key of the alert to update.
        :param alert_dict: Complete information for replacement of the alert.
        :return:
        """
        self.connect_dynamodb()

        alert_dict['alert_id'] = alert_id
        response = self.dynamodb.put_item(
            Item=alert_dict
        )

        return response

    def find(self, user_id):
        """
        Search dynamodb for all non-expired alerts for a given user.
        :param user_id: The auth0 id of the user.
        :return: List of alerts
        """
        self.connect_dynamodb()

        response = self.dynamodb.scan(
            FilterExpression=Attr('user_id').eq(user_id)
        )

        inactive_alerts = []
        visible_alerts = []
        ranked_alerts = []
        escalations = []

        for alert in response.get('Items'):
            if alert.get('state', '') == 'acknowledge':
                inactive_alerts.append(alert)
            elif alert.get('helpfulness', '') != '':
                ranked_alerts.append(alert)
                visible_alerts.append(alert)
            elif alert.get('state', '') == 'escalate':
                escalations.append(alert)
                visible_alerts.append(alert)
            else:
                visible_alerts.append(alert)

        return {
            'visible_alerts': visible_alerts,
            'ranked_alerts': ranked_alerts,
            'escalations': escalations,
            'inactive_alerts': inactive_alerts
        }

    def find_by_id(self, alert_id):
        """
        Search dynamodb for all non-expired alerts for a given user.
        :param alert_id: The guid of the alert.
        :return: List of alerts
        """
        self.connect_dynamodb()

        response = self.dynamodb.scan(
            FilterExpression=Attr('alert_id').eq(alert_id)
        )

        return response.get('Items')[0]

    def _create_alert_id(self):
        """

        :return: random alertid
        """
        return str(binascii.b2a_hex(os.urandom(15)))

class Rules(object):
    def __init__(self, userinfo, request):
        """

        :param userinfo: Flask session info about the user so that Rules can make decisions about the user.
        :param browser_header: The browser header as seen by Flask.
        """
        self.userinfo = userinfo
        self.request = request
        self.alert = Alert()

    def run(self):
        self.alert_firefox_out_of_date()

    def alert_firefox_out_of_date(self):
        if self._firefox_out_of_date():
            alert_dict = {
                'alert_code': '63f675d8896f4fb2b3caa204c8c2761e',
                'user_id': self.userinfo.get('user_id'),
                'risk': 'medium',
                'summary': 'Your version of Firefox is older than the current stable release.',
                'description': 'Running the latest version of your browser is key to keeping your '
                               'computer secure and your private data private. Older browsers may '
                               'have known security vulnerabilities that attackers can exploit to '
                               'steal your data or load malware, which can put you and Mozilla at risk. ',
                'date': str(datetime.date.today()),
                'url': 'https://www.mozilla.org/firefox/',
                'url_title': 'Download',
                'duplicate': False
            }
            self.alert.find_or_create_by(alert_dict=alert_dict, user_id=self.userinfo.get('user_id'))

    def _firefox_info(self):
        release_json = requests.get('https://product-details.mozilla.org/1.0/firefox_versions.json')
        return release_json.json()

    def _user_firefox_version(self):
        agent = self.request.headers.get('User-Agent')
        if agent.find('Firefox') != -1:
            version = agent.split('Firefox/')[1]
        else:
            version = None
        return version

    def _version_to_dictionary(self, version_number):
        version_number_list = version_number.split('.')
        version_dict = {
            'major_version': version_number_list[0],
            'minor_version': version_number_list[1]
        }

        if len(version_number_list) == 3:
            version_dict['dot_version'] = version_number_list[2]
        else:
            version_dict['dot_version'] = None

        return version_dict

    def _firefox_out_of_date(self):
        if self._user_firefox_version() is not None:
            u_version = self._version_to_dictionary(self._user_firefox_version())
            f_version = self._version_to_dictionary(self._firefox_info().get('LATEST_FIREFOX_VERSION'))

            if u_version.get('major_version') < f_version.get('major_version'):
                return True
            elif u_version.get('major_version') == f_version.get('major_version'):
                if u_version.get('minor_version') < f_version.get('minor_version'):
                    return True
                elif u_version.get('minor_version') == f_version.get('minor_version') \
                        and u_version.get('dot_version') is not None:
                    if u_version.get('dot_version') < f_version.get('dot_version'):
                        return True
                else:
                    return False
            else:
                return False

class FakeAlert(object):
    """Class only fires in development mode.  Adds alerts to a given user for testing only."""
    def __init__(self, user_id):
        self.user_id = user_id
        self.alert = Alert()

    def create_fake_alerts(self):
        self._create_fake_browser_alert()
        self._create_fake_geolocation_alert()

    def _create_fake_browser_alert(self):
        alert_dict = {
            'alert_code': '63f675d8896f4fb2b3caa204c8c2761e',
            'user_id': self.user_id,
            'risk': 'medium',
            'summary': 'Your version of Firefox is older than the current stable release.',
            'description': 'Running the latest version of your browser is key to keeping your '
                           'computer secure and your private data private. Older browsers may '
                           'have known security vulnerabilities that attackers can exploit to '
                           'steal your data or load malware, which can put you and Mozilla at risk. ',
            'date': str(fake.date(pattern="%Y-%m-%d", end_datetime=None)),
            'url': 'https://www.mozilla.org/firefox/',
            'url_title': 'Download',
            'duplicate': False
        }
        self.alert.find_or_create_by(alert_dict=alert_dict, user_id=self.user_id)

    def _create_fake_geolocation_alert(self):
        alert_dict = {
            'alert_code': '416c65727447656f6d6f64656c',
            'user_id': self.user_id,
            'risk': 'high',
            'summary': 'Did you recently login from {}, {} (5.6.7.8)?'.format(
                fake.state(),
                fake.country()
            ),
            'description': 'This alert is created based on geo ip information about the last login of a user.',
            'date': str(fake.date(pattern="%Y-%m-%d", end_datetime=None)),
            'url': 'https://www.mozilla.org',
            'url_title': 'Get Help',
            'duplicate': False
        }
        self.alert.find_or_create_by(alert_dict=alert_dict, user_id=self.user_id)
<commit_msg>alert ids are always strings<commit_after>"""Operations governing the creation and destruction of user facing alerts."""
import binascii
import boto3
import datetime
import json
import logging
import os
import requests

from boto3.dynamodb.conditions import Attr
from faker import Faker

fake = Faker()
logger = logging.getLogger(__name__)

class Feedback(object):
    """Send user data back to MozDef or other subscribers via an SNS Topic"""
    def __init__(self, alert_dict, alert_action):
        self.alert_dict = alert_dict
        self.alert_action = alert_action
        self.ssm = None
        self.sns = None
        self.sns_topic_arn = self.get_sns_arn()

    def connect_sns(self):
        if self.sns is None:
            self.sns = boto3.client('sns', region_name='us-west-2')

    def connect_ssm(self):
        if self.ssm is None:
            self.ssm = boto3.client('ssm', region_name='us-west-2')

    def get_sns_arn(self):
        self.connect_ssm()

        response = self.ssm.get_parameter(
            Name='sso-dashboard-alerts-sns',
            WithDecryption=False
        )

        return response.get('Parameter').get('Value')

    def _construct_alert(self):
        message = {
            'category': 'user_feedback',
            'details': {
                'action': self.alert_action,  # (escalate|acknowledge|false-positive)
                'alert_information': self.alert_dict
            }
        }

        return message

    def send(self):
        """Send the event to the SNS Topic."""
        self.connect_sns()
        message = self._construct_alert()
        response = self.sns.publish(
            TopicArn=self.sns_topic_arn,
            Message=json.dumps(message),
            Subject='sso-dashboard-user-feedback'
        )
        return response

class Alert(object):
    """Primary object containing alert functions."""
    def __init__(self):
        self.alert_table_name = 'sso-dashboard-alert'
        self.dynamodb = None

    def has_actions(self, alert_dict):
        """Let's the view know if it should render actions for the alert."""

        # Whitelist the firefox out of date alert.  It should not get buttons.
        if self.alert_dict.get('alert_code') is not '63f675d8896f4fb2b3caa204c8c2761e':
            return True
        else:
            return False

    def has_escalation(self, alert_dict):
        """Let's the view know if it should render actions for the alert."""

        # Whitelist the firefox out of date alert.  It should not get buttons.
        if self.alert_dict.get('alert_code') is not '63f675d8896f4fb2b3caa204c8c2761e':
            return True
        else:
            return False

    def connect_dynamodb(self):
        if self.dynamodb is None:
            dynamodb = boto3.resource('dynamodb', region_name='us-west-2')
            table = dynamodb.Table(self.alert_table_name)
            self.dynamodb = table

    def find_or_create_by(self, alert_dict, user_id):
        """
        :param alert_dict: takes a dictionary of alert information
        :param user_id: the session info user_id
        :return: dynamodb response or none
        """

        # Search for the user alerts
        current_alerts = self.find(user_id)

        # If the alert is duplicate false do not create another instance of it.
        for alert in current_alerts:
            try:
                if alert.get('alert_code') == alert_dict.get('alert_code') and alert_dict.get('duplicate') is False:
                    return None
                else:
                    continue
            except AttributeError as e:
                logger.error('Bad data in alerts table for user: {}, exception was {}'.format(user_id, e))

        # Else create another alert.
        return self.create(alert_dict)

    def create(self, alert_dict):
        """
        Create an alert.
        :param alert_json: takes a dictionary of alert information
        :return: dynamodb response
        """
        self.connect_dynamodb()

        alert_dict['alert_id'] = self._create_alert_id()
        response = self.dynamodb.put_item(
            Item=alert_dict
        )

        return response

    def destroy(self, alert_id, user_id):
        """
        Delete an alert.
        :param alert_id: Primary key of the alert to destroy.
        :return: status_code
        """
        self.connect_dynamodb()

        response = self.dynamodb.delete_item(
            Key={
                'alert_id': alert_id,
                'user_id': user_id
            }
        )

        return response

    def update(self, alert_id, alert_dict):
        """
        Update an alert record for a user.
        :param alert_id: Primary key of the alert to update.
        :param alert_dict: Complete information for replacement of the alert.
        :return:
        """
        self.connect_dynamodb()

        alert_dict['alert_id'] = alert_id
        response = self.dynamodb.put_item(
            Item=alert_dict
        )

        return response

    def find(self, user_id):
        """
        Search dynamodb for all non-expired alerts for a given user.
        :param user_id: The auth0 id of the user.
        :return: List of alerts
        """
        self.connect_dynamodb()

        response = self.dynamodb.scan(
            FilterExpression=Attr('user_id').eq(user_id)
        )

        inactive_alerts = []
        visible_alerts = []
        ranked_alerts = []
        escalations = []

        for alert in response.get('Items'):
            if alert.get('state', '') == 'acknowledge':
                inactive_alerts.append(alert)
            elif alert.get('helpfulness', '') != '':
                ranked_alerts.append(alert)
                visible_alerts.append(alert)
            elif alert.get('state', '') == 'escalate':
                escalations.append(alert)
                visible_alerts.append(alert)
            else:
                visible_alerts.append(alert)

        return {
            'visible_alerts': visible_alerts,
            'ranked_alerts': ranked_alerts,
            'escalations': escalations,
            'inactive_alerts': inactive_alerts
        }

    def find_by_id(self, alert_id):
        """
        Search dynamodb for all non-expired alerts for a given user.
        :param alert_id: The guid of the alert.
        :return: List of alerts
        """
        self.connect_dynamodb()

        response = self.dynamodb.scan(
            FilterExpression=Attr('alert_id').eq(alert_id)
        )

        return response.get('Items')[0]

    def _create_alert_id(self):
        """

        :return: random alertid
        """
        return binascii.b2a_hex(os.urandom(15)).decode()

class Rules(object):
    def __init__(self, userinfo, request):
        """

        :param userinfo: Flask session info about the user so that Rules can make decisions about the user.
        :param browser_header: The browser header as seen by Flask.
        """
        self.userinfo = userinfo
        self.request = request
        self.alert = Alert()

    def run(self):
        self.alert_firefox_out_of_date()

    def alert_firefox_out_of_date(self):
        if self._firefox_out_of_date():
            alert_dict = {
                'alert_code': '63f675d8896f4fb2b3caa204c8c2761e',
                'user_id': self.userinfo.get('user_id'),
                'risk': 'medium',
                'summary': 'Your version of Firefox is older than the current stable release.',
                'description': 'Running the latest version of your browser is key to keeping your '
                               'computer secure and your private data private. Older browsers may '
                               'have known security vulnerabilities that attackers can exploit to '
                               'steal your data or load malware, which can put you and Mozilla at risk. ',
                'date': str(datetime.date.today()),
                'url': 'https://www.mozilla.org/firefox/',
                'url_title': 'Download',
                'duplicate': False
            }
            self.alert.find_or_create_by(alert_dict=alert_dict, user_id=self.userinfo.get('user_id'))

    def _firefox_info(self):
        release_json = requests.get('https://product-details.mozilla.org/1.0/firefox_versions.json')
        return release_json.json()

    def _user_firefox_version(self):
        agent = self.request.headers.get('User-Agent')
        if agent.find('Firefox') != -1:
            version = agent.split('Firefox/')[1]
        else:
            version = None
        return version

    def _version_to_dictionary(self, version_number):
        version_number_list = version_number.split('.')
        version_dict = {
            'major_version': version_number_list[0],
            'minor_version': version_number_list[1]
        }

        if len(version_number_list) == 3:
            version_dict['dot_version'] = version_number_list[2]
        else:
            version_dict['dot_version'] = None

        return version_dict

    def _firefox_out_of_date(self):
        if self._user_firefox_version() is not None:
            u_version = self._version_to_dictionary(self._user_firefox_version())
            f_version = self._version_to_dictionary(self._firefox_info().get('LATEST_FIREFOX_VERSION'))

            if u_version.get('major_version') < f_version.get('major_version'):
                return True
            elif u_version.get('major_version') == f_version.get('major_version'):
                if u_version.get('minor_version') < f_version.get('minor_version'):
                    return True
                elif u_version.get('minor_version') == f_version.get('minor_version') \
                        and u_version.get('dot_version') is not None:
                    if u_version.get('dot_version') < f_version.get('dot_version'):
                        return True
                else:
                    return False
            else:
                return False

class FakeAlert(object):
    """Class only fires in development mode.  Adds alerts to a given user for testing only."""
    def __init__(self, user_id):
        self.user_id = user_id
        self.alert = Alert()

    def create_fake_alerts(self):
        self._create_fake_browser_alert()
        self._create_fake_geolocation_alert()

    def _create_fake_browser_alert(self):
        alert_dict = {
            'alert_code': '63f675d8896f4fb2b3caa204c8c2761e',
            'user_id': self.user_id,
            'risk': 'medium',
            'summary': 'Your version of Firefox is older than the current stable release.',
            'description': 'Running the latest version of your browser is key to keeping your '
                           'computer secure and your private data private. Older browsers may '
                           'have known security vulnerabilities that attackers can exploit to '
                           'steal your data or load malware, which can put you and Mozilla at risk. ',
            'date': str(fake.date(pattern="%Y-%m-%d", end_datetime=None)),
            'url': 'https://www.mozilla.org/firefox/',
            'url_title': 'Download',
            'duplicate': False
        }
        self.alert.find_or_create_by(alert_dict=alert_dict, user_id=self.user_id)

    def _create_fake_geolocation_alert(self):
        alert_dict = {
            'alert_code': '416c65727447656f6d6f64656c',
            'user_id': self.user_id,
            'risk': 'high',
            'summary': 'Did you recently login from {}, {} (5.6.7.8)?'.format(
                fake.state(),
                fake.country()
            ),
            'description': 'This alert is created based on geo ip information about the last login of a user.',
            'date': str(fake.date(pattern="%Y-%m-%d", end_datetime=None)),
            'url': 'https://www.mozilla.org',
            'url_title': 'Get Help',
            'duplicate': False
        }
        self.alert.find_or_create_by(alert_dict=alert_dict, user_id=self.user_id)
<|endoftext|>


<commit_before>'''An implementation of the Raft consensus algorithm'''
from kon_tiki.persist import LogEntry
from kon_tiki.fundamentals import median
from twisted.internet import reactor, defer
from twisted.spread import pb
import random

RERUN_RPC = object()

class Server(object):
    '''A Raft participant.

    `peers`: identities for the server's peers

    `persister`: an object that implements the Persist protocol and
    can save and restore to stable storage

    `applyCommand`: callable invoked with the command to apply
     '''

    def __init__(self, cycle, identity, peers, persister, applyCommand,
                 electionTimeoutRange, commitIndex=0, lastApplied=0):
        self.cycle = cycle
        self.identity = identity
        self.peers = peers
        self.persister = persister
        self.applyCommand = applyCommand
        self.electionTimeoutRange = electionTimeoutRange
        self.commitIndex = commitIndex
        self.lastApplied = lastApplied
        self.pending = set()
        self.applyCommitted()

    @classmethod
    def fromServer(cls, electionTimeoutRange, cycle, server):
        server.cancel_all()
        return cls(electionTimeoutRange=electionTimeoutRange,
                   cycle=server.cycle,
                   identity=server.identity,
                   peers=server.peers,
                   persister=server.persister,
                   commitIndex=server.commitIndex,
                   lastApplied=server.lastApplied)

    def track(self, deferred):
        def remove(result):
            self.pending.remove(deferred)
            return result

        deferred.addCallbacks(remove, remove)
        return deferred

    def cancel_all(self):
        for deferred in self.pending:
            deferred.cancel()

    def applyCommitted(self):
        if self.lastApplied < self.commitIndex:
            for entry in self.persister.logSlice(self.lastApplied,
                                                 self.commitIndex + 1):
                self.applyCommand(entry.command)
            self.lastApplied = self.commitIndex

    def willBecomeFollower(self, term):
        if term > self.persister.currentTerm:
            self.persister.currentTerm = term
            self.cycle.changeState(Follower)
            return True
        return False

    def candidateIdOK(self, candidateId):
        return (self.persister.votedFor is None
                or self.persister.votedFor == candidateId)

    def candidateLogUpToDate(self, lastLogIndex, lastLogTerm):
        # Section 5.4.1
        if self.persister.currentTerm == lastLogTerm:
            return self.persister.lastIndex <= lastLogIndex
        else:
            return self.persister.lastIndexNewerThanTerm(lastLogTerm)

    def remote_appendEntries(self,
                             term, leaderId, prevLogIndex,
                             prevLogTerm, entries, leaderCommit):
        # RPC
        if self.willBecomeFollower(term):
            return RERUN_RPC
        return self.persister.currentTerm, False

    def remote_requestVote(self, term, candidateId, lastLogIndex, lastLogTerm):
        # RPC
        if term < self.persister.currentTerm:
            voteGranted = False
        else:
            voteGranted = (self.candidateIdOK(candidateId)
                           and self.candidateLogUpToDate(lastLogIndex,
                                                         lastLogTerm))
            self.persister.votedFor = candidateId
            self.willBecomeFollower(term)
        return self.persister.currentTerm, voteGranted

class StartsElection(Server):
    becomeCandidateDeferred = None

    def __init__(self, *args, **kwargs):
        super(StartsElection, self).__init__(*args, **kwargs)
        self.resetElectionTimeout()

    def resetElectionTimeout(self):
        if (self.becomeCandidateDeferred is not None
            and self.becomeCandidateDeferred.active()):
            self.becomeCandidateDeferred.cancel()

        self.electionTimeout = random.uniform(*self.electionTimeoutRange)
        d = self.track(reactor.callLater(self.electionTimeout,
                                         self.cycle.changeState,
                                         Candidate))
        self.becomeCandidateDeferred = d

    def remote_appendEntries(self, *args, **kwargs):
        self.resetElectionTimeout()
        return super(StartsElection, self).appendEntries(*args, **kwargs)

class Follower(Server):
    '''A Raft follower.'''

    leaderId = None

    def remote_appendEntries(self,
                             term, leaderId, prevLogIndex,
                             prevLogTerm, entries, leaderCommit):
        # RPC
        # 1 & 2
        if (term < self.persister.currentTerm
            or not self.persister.indexMatchesTerm(prevLogIndex,
                                                   prevLogTerm)):
            success = False
        else:
            # 3
            new = self.persister.matchLogToEntries(matchAfter=prevLogIndex,
                                                   entries=entries)
            # 4
            self.persister.appendNewEntries(new)

            # 5
            if leaderCommit > self.commitIndex:
                self.commitIndex = min(leaderCommit, self.persister.lastIndex)

            self.applyCommitted()

            self.leaderId = leaderId
            success = True
            self.resetElectionTimeout()

        return self.persister.currentTerm, success

    def remote_command(self, command):
        d = self.track(self.peers[self.leaderId].callRemote('command',
                                                            command))
        return d

class Candidate(StartsElection):

    def __init__(self, *args, **kwargs):
        super(Candidate, self).__init__(*args, **kwargs)
        self.conductElection()

    def prepareForElection(self):
        self.persister.currentTerm += 1
        self.persister.votedFor = self.identity
        self.votes = 0

    def willBecomeLeader(self, votesSoFar):
        if votesSoFar >= len(self.peers) / 2 + 1:
            self.cycle.changeState(Leader)
            return True
        return False

    def completeRequestVote(self, result):
        term, voteGranted = result
        if not self.willBecomeFollower(term) and voteGranted:
            self.willBecomeLeader(self.votes)

    def sendRequestVote(self, perspective):
        term = self.persister.currentTerm
        lastLogIndex = self.persister.lastLogIndex
        lastLogTerm = self.logSlice(lastLogIndex - 1)
        if lastLogTerm:
            (lastLogTerm,) = lastLogTerm
        else:
            lastLogTerm = 0
        d = self.track(perspective.callRemote('requestVote',
                                              term,
                                              self.identity,
                                              lastLogIndex,
                                              lastLogTerm))
        d.addCallback(self.completeRequestVote)
        return d

    def broadcastRequestVote(self):
        for perspective in self.peers.values():
            self.sendRequestVote(perspective)

    def conductElection(self):
        self.prepareForElection()
        self.broadcastRequestVote()

    def remote_appendEntries(self, *args, **kwargs):
        return self.persister.currentTerm, False

class Leader(Server):
    '''A Raft leader.'''

    def __init__(self, *args, **kwargs):
        super(Server, self).__init__(*args, **kwargs)
        self.heartbeatInterval = min(self.electionTimeoutRange[0] - 50, 50)
        self.postElection()
        d = self.track(reactor.loopingCall(self.heartbeatInterval,
                                           self.broadcastAppendEntries))
        self.heartbeatLoopingCall = d

    def postElection(self):
        lastLogIndex = self.persister.lastLogIndex
        self.nextIndex = dict.fromkeys(self.peers, lastLogIndex + 1)
        self.matchIndex = dict.fromkeys(self.peers, 0)

    def updateCommitIndex(self):
        newCommitIndex = median(self.matchIndex.values())
        if newCommitIndex > self.commitIndex:
            self.commitIndex = newCommitIndex
            return True
        return False

    def completeAppendEntries(self, result, identity, lastLogIndex):
        term, success = result
        if self.persister.currentTerm < term:
            self.willBecomeFollower()
        elif not success:
            self.nextIndex[identity] -= 1
            # retry
        else:
            self.nextIndex[identity] = lastLogIndex + 1
            self.matchIndex[identity] = lastLogIndex
            if self.updateCommitIndex():
                self.applyCommitted()

    def sendAppendEntries(self, identity, perspective):
        prevLogIndex = self.nextIndex[identity] - 1
        allEntries = self.persister.logSlice(start=prevLogIndex, end=None)
        prevLogTerm, entries = allEntries[0], allEntries[1:]
        lastLogIndex = self.persister.lastLogIndex

        d = self.track(perspective.callRemote('appendEntries',
                                              term=self.persister.currentTerm,
                                              candidateId=self.identity,
                                              prevLogIndex=prevLogIndex,
                                              prevLogTerm=prevLogTerm,
                                              entries=entries))

        d.addCallback(self.completeAppendEntries,
                      identity=identity,
                      lastLogIndex=lastLogIndex)
        return d

    def broadcastAppendEntries(self):
        for identity, perspective in self.peers:
            self.sendAppendEntries(identity, perspective)

    def remote_command(self, command):
        self.persister.appendEntries([LogEntry(term=self.persister.currentTerm,
                                               command=command)])
        self.broadcastAppendEntries()
        return True

class ServerCycle(pb.Root):

    def __init__(self, identity, peers, persister, applyCommand,
                 electionTimeoutRange=(.150, .350)):
        self.identity = identity
        self.peers = peers
        self.persister = persister
        self.applyCommand = applyCommand
        self.electionTimeoutRange = electionTimeoutRange
        self.state = Follower(electionTimeoutRange=electionTimeoutRange,
                              cycle=self,
                              identity=identity,
                              peers=peers,
                              persister=persister,
                              applyCommand=applyCommand)

    def changeState(self, newState):
        self.state = newState.fromServer(self.electionTimeoutRange,
                                         cycle=self,
                                         server=self.state)

    def rerun(self, methodName, *args, **kwargs):
        result = RERUN_RPC
        while result is RERUN_RPC:
            method = getattr(self.state, methodName)
            result = method(*args, **kwargs)
        return result

    def remote_appendEntries(self,
                             term, leaderId, prevLogIndex,
                             prevLogTerm, entries, leaderCommit):
        return self.rereun('remote_appendEntries',
                           term, leaderId,
                           prevLogIndex,
                           prevLogTerm,
                           entries,
                           leaderCommit)

    def remote_requestVote(self,
                           term, candidateId, lastLogIndex, lastLogTerm):
        return self.rerun('remote_requestVote',
                          term, candidateId,
                          lastLogIndex, lastLogIndex)

    def remote_command(self, command):
        return self.rerun('remote_command', command)
<commit_msg>Add deferreds to self.pending when asked to track them<commit_after>'''An implementation of the Raft consensus algorithm'''
from kon_tiki.persist import LogEntry
from kon_tiki.fundamentals import median
from twisted.internet import reactor, defer
from twisted.spread import pb
import random

RERUN_RPC = object()

class Server(object):
    '''A Raft participant.

    `peers`: identities for the server's peers

    `persister`: an object that implements the Persist protocol and
    can save and restore to stable storage

    `applyCommand`: callable invoked with the command to apply
     '''

    def __init__(self, cycle, identity, peers, persister, applyCommand,
                 electionTimeoutRange, commitIndex=0, lastApplied=0):
        self.cycle = cycle
        self.identity = identity
        self.peers = peers
        self.persister = persister
        self.applyCommand = applyCommand
        self.electionTimeoutRange = electionTimeoutRange
        self.commitIndex = commitIndex
        self.lastApplied = lastApplied
        self.pending = set()
        self.applyCommitted()

    @classmethod
    def fromServer(cls, electionTimeoutRange, cycle, server):
        server.cancel_all()
        return cls(electionTimeoutRange=electionTimeoutRange,
                   cycle=server.cycle,
                   identity=server.identity,
                   peers=server.peers,
                   persister=server.persister,
                   commitIndex=server.commitIndex,
                   lastApplied=server.lastApplied)

    def track(self, deferred):
        def remove(result):
            self.pending.remove(deferred)
            return result

        deferred.addCallbacks(remove, remove)
        self.pending.add(deferred)
        return deferred

    def cancel_all(self):
        for deferred in self.pending:
            deferred.cancel()

    def applyCommitted(self):
        if self.lastApplied < self.commitIndex:
            for entry in self.persister.logSlice(self.lastApplied,
                                                 self.commitIndex + 1):
                self.applyCommand(entry.command)
            self.lastApplied = self.commitIndex

    def willBecomeFollower(self, term):
        if term > self.persister.currentTerm:
            self.persister.currentTerm = term
            self.cycle.changeState(Follower)
            return True
        return False

    def candidateIdOK(self, candidateId):
        return (self.persister.votedFor is None
                or self.persister.votedFor == candidateId)

    def candidateLogUpToDate(self, lastLogIndex, lastLogTerm):
        # Section 5.4.1
        if self.persister.currentTerm == lastLogTerm:
            return self.persister.lastIndex <= lastLogIndex
        else:
            return self.persister.lastIndexNewerThanTerm(lastLogTerm)

    def remote_appendEntries(self,
                             term, leaderId, prevLogIndex,
                             prevLogTerm, entries, leaderCommit):
        # RPC
        if self.willBecomeFollower(term):
            return RERUN_RPC
        return self.persister.currentTerm, False

    def remote_requestVote(self, term, candidateId, lastLogIndex, lastLogTerm):
        # RPC
        if term < self.persister.currentTerm:
            voteGranted = False
        else:
            voteGranted = (self.candidateIdOK(candidateId)
                           and self.candidateLogUpToDate(lastLogIndex,
                                                         lastLogTerm))
            self.persister.votedFor = candidateId
            self.willBecomeFollower(term)
        return self.persister.currentTerm, voteGranted

class StartsElection(Server):
    becomeCandidateDeferred = None

    def __init__(self, *args, **kwargs):
        super(StartsElection, self).__init__(*args, **kwargs)
        self.resetElectionTimeout()

    def resetElectionTimeout(self):
        if (self.becomeCandidateDeferred is not None
            and self.becomeCandidateDeferred.active()):
            self.becomeCandidateDeferred.cancel()

        self.electionTimeout = random.uniform(*self.electionTimeoutRange)
        d = self.track(reactor.callLater(self.electionTimeout,
                                         self.cycle.changeState,
                                         Candidate))
        self.becomeCandidateDeferred = d

    def remote_appendEntries(self, *args, **kwargs):
        self.resetElectionTimeout()
        return super(StartsElection, self).appendEntries(*args, **kwargs)

class Follower(Server):
    '''A Raft follower.'''

    leaderId = None

    def remote_appendEntries(self,
                             term, leaderId, prevLogIndex,
                             prevLogTerm, entries, leaderCommit):
        # RPC
        # 1 & 2
        if (term < self.persister.currentTerm
            or not self.persister.indexMatchesTerm(prevLogIndex,
                                                   prevLogTerm)):
            success = False
        else:
            # 3
            new = self.persister.matchLogToEntries(matchAfter=prevLogIndex,
                                                   entries=entries)
            # 4
            self.persister.appendNewEntries(new)

            # 5
            if leaderCommit > self.commitIndex:
                self.commitIndex = min(leaderCommit, self.persister.lastIndex)

            self.applyCommitted()

            self.leaderId = leaderId
            success = True
            self.resetElectionTimeout()

        return self.persister.currentTerm, success

    def remote_command(self, command):
        d = self.track(self.peers[self.leaderId].callRemote('command',
                                                            command))
        return d

class Candidate(StartsElection):

    def __init__(self, *args, **kwargs):
        super(Candidate, self).__init__(*args, **kwargs)
        self.conductElection()

    def prepareForElection(self):
        self.persister.currentTerm += 1
        self.persister.votedFor = self.identity
        self.votes = 0

    def willBecomeLeader(self, votesSoFar):
        if votesSoFar >= len(self.peers) / 2 + 1:
            self.cycle.changeState(Leader)
            return True
        return False

    def completeRequestVote(self, result):
        term, voteGranted = result
        if not self.willBecomeFollower(term) and voteGranted:
            self.willBecomeLeader(self.votes)

    def sendRequestVote(self, perspective):
        term = self.persister.currentTerm
        lastLogIndex = self.persister.lastLogIndex
        lastLogTerm = self.logSlice(lastLogIndex - 1)
        if lastLogTerm:
            (lastLogTerm,) = lastLogTerm
        else:
            lastLogTerm = 0
        d = self.track(perspective.callRemote('requestVote',
                                              term,
                                              self.identity,
                                              lastLogIndex,
                                              lastLogTerm))
        d.addCallback(self.completeRequestVote)
        return d

    def broadcastRequestVote(self):
        for perspective in self.peers.values():
            self.sendRequestVote(perspective)

    def conductElection(self):
        self.prepareForElection()
        self.broadcastRequestVote()

    def remote_appendEntries(self, *args, **kwargs):
        return self.persister.currentTerm, False

class Leader(Server):
    '''A Raft leader.'''

    def __init__(self, *args, **kwargs):
        super(Server, self).__init__(*args, **kwargs)
        self.heartbeatInterval = min(self.electionTimeoutRange[0] - 50, 50)
        self.postElection()
        d = self.track(reactor.loopingCall(self.heartbeatInterval,
                                           self.broadcastAppendEntries))
        self.heartbeatLoopingCall = d

    def postElection(self):
        lastLogIndex = self.persister.lastLogIndex
        self.nextIndex = dict.fromkeys(self.peers, lastLogIndex + 1)
        self.matchIndex = dict.fromkeys(self.peers, 0)

    def updateCommitIndex(self):
        newCommitIndex = median(self.matchIndex.values())
        if newCommitIndex > self.commitIndex:
            self.commitIndex = newCommitIndex
            return True
        return False

    def completeAppendEntries(self, result, identity, lastLogIndex):
        term, success = result
        if self.persister.currentTerm < term:
            self.willBecomeFollower()
        elif not success:
            self.nextIndex[identity] -= 1
            # retry
        else:
            self.nextIndex[identity] = lastLogIndex + 1
            self.matchIndex[identity] = lastLogIndex
            if self.updateCommitIndex():
                self.applyCommitted()

    def sendAppendEntries(self, identity, perspective):
        prevLogIndex = self.nextIndex[identity] - 1
        allEntries = self.persister.logSlice(start=prevLogIndex, end=None)
        prevLogTerm, entries = allEntries[0], allEntries[1:]
        lastLogIndex = self.persister.lastLogIndex

        d = self.track(perspective.callRemote('appendEntries',
                                              term=self.persister.currentTerm,
                                              candidateId=self.identity,
                                              prevLogIndex=prevLogIndex,
                                              prevLogTerm=prevLogTerm,
                                              entries=entries))

        d.addCallback(self.completeAppendEntries,
                      identity=identity,
                      lastLogIndex=lastLogIndex)
        return d

    def broadcastAppendEntries(self):
        for identity, perspective in self.peers:
            self.sendAppendEntries(identity, perspective)

    def remote_command(self, command):
        self.persister.appendEntries([LogEntry(term=self.persister.currentTerm,
                                               command=command)])
        self.broadcastAppendEntries()
        return True

class ServerCycle(pb.Root):

    def __init__(self, identity, peers, persister, applyCommand,
                 electionTimeoutRange=(.150, .350)):
        self.identity = identity
        self.peers = peers
        self.persister = persister
        self.applyCommand = applyCommand
        self.electionTimeoutRange = electionTimeoutRange
        self.state = Follower(electionTimeoutRange=electionTimeoutRange,
                              cycle=self,
                              identity=identity,
                              peers=peers,
                              persister=persister,
                              applyCommand=applyCommand)

    def changeState(self, newState):
        self.state = newState.fromServer(self.electionTimeoutRange,
                                         cycle=self,
                                         server=self.state)

    def rerun(self, methodName, *args, **kwargs):
        result = RERUN_RPC
        while result is RERUN_RPC:
            method = getattr(self.state, methodName)
            result = method(*args, **kwargs)
        return result

    def remote_appendEntries(self,
                             term, leaderId, prevLogIndex,
                             prevLogTerm, entries, leaderCommit):
        return self.rereun('remote_appendEntries',
                           term, leaderId,
                           prevLogIndex,
                           prevLogTerm,
                           entries,
                           leaderCommit)

    def remote_requestVote(self,
                           term, candidateId, lastLogIndex, lastLogTerm):
        return self.rerun('remote_requestVote',
                          term, candidateId,
                          lastLogIndex, lastLogIndex)

    def remote_command(self, command):
        return self.rerun('remote_command', command)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import warnings

import numpy as np
import pandas as pd

from .core import DataFrame, Series, Index, aca, map_partitions, no_default
from .shuffle import shuffle
from .utils import make_meta, insert_meta_param_description, raise_on_meta_error
from ..utils import derived_from, M, funcname

def _maybe_slice(grouped, columns):
    """
    Slice columns if grouped is pd.DataFrameGroupBy
    """
    if isinstance(grouped, pd.core.groupby.DataFrameGroupBy):
        if columns is not None:
            columns = columns if isinstance(columns, str) else list(columns)
            return grouped[columns]
    return grouped

def _groupby_slice_apply(df, grouper, key, func):
    g = df.groupby(grouper)
    if key:
        g = g[key]
    return g.apply(func)

def _groupby_get_group(df, by_key, get_key, columns):
    # SeriesGroupBy may pass df which includes group key
    grouped = df.groupby(by_key)

    if get_key in grouped.groups:
        if isinstance(df, pd.DataFrame):
            grouped = grouped[columns]
        return grouped.get_group(get_key)

    else:
        # to create empty DataFrame/Series, which has the same
        # dtype as the original
        if isinstance(df, pd.DataFrame):
            # may be SeriesGroupBy
            df = df[columns]
        return df.iloc[0:0]

###############################################################
# Aggregation
###############################################################

def _groupby_aggregate(df, aggfunc=None, levels=None):
    return aggfunc(df.groupby(level=levels))

def _apply_chunk(df, index, func, columns):
    if isinstance(df, pd.Series):
        return func(df.groupby(index))
    else:
        columns = columns if isinstance(columns, str) else list(columns)
        return func(df.groupby(index)[columns])

def _var_chunk(df, index):
    if isinstance(df, pd.Series):
        df = df.to_frame()
    x = df.groupby(index).sum()
    x2 = (df**2).rename(columns=lambda c: c + '-x2')
    cols = [c + '-x2' for c in x.columns]
    x2 = pd.concat([df, x2], axis=1).groupby(index)[cols].sum()
    n = (df.groupby(index).count()
           .rename(columns=lambda c: c + '-count'))

    result = pd.concat([x, x2, n], axis=1)
    return result

def _var_agg(g, ddof):
    g = g.groupby(level=0).sum()
    nc = len(g.columns)
    x = g[g.columns[:nc // 3]]
    x2 = g[g.columns[nc // 3:2 * nc // 3]].rename(columns=lambda c: c[:-3])
    n = g[g.columns[-nc // 3:]].rename(columns=lambda c: c[:-6])

    result = x2 - x ** 2 / n
    div = (n - ddof)
    div[div < 0] = 0
    result /= div
    result[(n - ddof) == 0] = np.nan
    assert isinstance(result, pd.DataFrame)
    return result

###############################################################
# nunique
###############################################################

def _nunique_df_chunk(df, index):
    # we call set_index here to force a possibly duplicate index
    # for our reduce step
    grouped = (df.groupby(index).apply(pd.DataFrame.drop_duplicates))
    grouped.index = grouped.index.get_level_values(level=0)
    return grouped

def _nunique_df_aggregate(df, name):
    return df.groupby(level=0)[name].nunique()

def _nunique_series_chunk(df, index):
    assert isinstance(df, pd.Series)
    if isinstance(index, np.ndarray):
        assert len(index) == len(df)
        index = pd.Series(index, index=df.index)
    grouped = pd.concat([df, index], axis=1).drop_duplicates()
    return grouped

def _nunique_series_aggregate(df):
    return df.groupby(df.columns[1])[df.columns[0]].nunique()

class _GroupBy(object):
    """ Superclass for DataFrameGroupBy and SeriesGroupBy

    Parameters
    ----------

    obj: DataFrame or Series
        DataFrame or Series to be grouped
    index: str, list or Series
        The key for grouping
    kwargs: dict
        Other keywords passed to groupby
    """
    def __init__(self, df, index=None, slice=None, **kwargs):
        assert isinstance(df, (DataFrame, Series))
        self.obj = df

        # grouping key passed via groupby method
        if (isinstance(index, (DataFrame, Series, Index)) and
           isinstance(df, DataFrame)):

            if (isinstance(index, Series) and index.name in df.columns and
               index._name == df[index.name]._name):
                index = index.name
            elif (isinstance(index, DataFrame) and
                  set(index.columns).issubset(df.columns) and
                  index._name == df[index.columns]._name):
                index = list(index.columns)

        self.index = index

        # slicing key applied to _GroupBy instance
        self._slice = slice

        self.kwargs = kwargs

        if isinstance(index, Series) and df.divisions != index.divisions:
            msg = ("The Series and index of the groupby"
                   " must have the same divisions.")
            raise NotImplementedError(msg)

        if self._is_grouped_by_sliced_column(self.obj, index):
            # check whether given Series is taken from given df and unchanged.
            # If any operations are performed, _name will be changed to
            # e.g. "elemwise-xxxx"

            # if group key (index) is a Series sliced from DataFrame,
            # emulation must be performed as the same.
            # otherwise, group key is regarded as a separate column
            self._meta = self.obj._meta.groupby(self.obj._meta[index.name])

        elif isinstance(self.index, Series):
            self._meta = self.obj._meta.groupby(self.index._meta)
        else:
            self._meta = self.obj._meta.groupby(self.index)

    def _is_grouped_by_sliced_column(self, df, index):
        """
        Return whether index is a Series sliced from df
        """
        if isinstance(df, Series):
            return False
        if (isinstance(index, Series) and index._name in df.columns and
                index._name == df[index.name]._name):
            return True
        if (isinstance(index, DataFrame) and
                set(index.columns).issubset(df.columns) and
                index._name == df[index.columns]._name):
            index = list(index.columns)
            return True
        return False

    @property
    def _meta_nonempty(self):
        """
        Return a pd.DataFrameGroupBy / pd.SeriesGroupBy which contains sample data.
        """
        sample = self.obj._meta_nonempty
        if isinstance(self.index, Series):
            if self._is_grouped_by_sliced_column(self.obj, self.index):
                grouped = sample.groupby(sample[self.index.name])
            else:
                grouped = sample.groupby(self.index._meta_nonempty)
        else:
            grouped = sample.groupby(self.index)
        return _maybe_slice(grouped, self._slice)

    def _aca_agg(self, token, func, aggfunc=None):
        if aggfunc is None:
            aggfunc = func

        meta = func(self._meta)
        columns = meta.name if isinstance(meta, pd.Series) else meta.columns

        token = self._token_prefix + token

        if isinstance(self.index, (tuple, list)) and len(self.index) > 1:
            levels = list(range(len(self.index)))
        else:
            levels = 0

        return aca([self.obj, self.index, func, columns],
                   chunk=_apply_chunk, aggregate=_groupby_aggregate,
                   meta=meta, token=token,
                   aggregate_kwargs=dict(aggfunc=aggfunc, levels=levels))

    @derived_from(pd.core.groupby.GroupBy)
    def sum(self):
        return self._aca_agg(token='sum', func=M.sum)

    @derived_from(pd.core.groupby.GroupBy)
    def min(self):
        return self._aca_agg(token='min', func=M.min)

    @derived_from(pd.core.groupby.GroupBy)
    def max(self):
        return self._aca_agg(token='max', func=M.max)

    @derived_from(pd.core.groupby.GroupBy)
    def count(self):
        return self._aca_agg(token='count', func=M.count,
                             aggfunc=M.sum)

    @derived_from(pd.core.groupby.GroupBy)
    def mean(self):
        return self.sum() / self.count()

    @derived_from(pd.core.groupby.GroupBy)
    def var(self, ddof=1):
        from functools import partial
        meta = self.obj._meta
        if isinstance(meta, pd.Series):
            meta = meta.to_frame()
        meta = meta.groupby(self.index).var(ddof=1)
        result = aca([self.obj, self.index], chunk=_var_chunk,
                     aggregate=partial(_var_agg, ddof=ddof), meta=meta,
                     token=self._token_prefix + 'var')

        if isinstance(self.obj, Series):
            result = result[result.columns[0]]
        if self._slice:
            result = result[self._slice]

        return result

    @derived_from(pd.core.groupby.GroupBy)
    def std(self, ddof=1):
        v = self.var(ddof)
        result = map_partitions(np.sqrt, v, meta=v)
        return result

    @derived_from(pd.core.groupby.GroupBy)
    def get_group(self, key):
        token = self._token_prefix + 'get_group'

        meta = self._meta.obj
        if isinstance(meta, pd.DataFrame) and self._slice is not None:
            meta = meta[self._slice]
        columns = meta.columns if isinstance(meta, pd.DataFrame) else meta.name

        return map_partitions(_groupby_get_group, self.obj, self.index, key,
                              columns, meta=meta, token=token)

    @insert_meta_param_description(pad=12)
    def apply(self, func, meta=no_default, columns=no_default):
        """ Parallel version of pandas GroupBy.apply

        This mimics the pandas version except for the following:

        1.  The user should provide output metadata.
        2.  If the grouper does not align with the index then this causes a full
            shuffle.  The order of rows within each group may not be preserved.

        Parameters
        ----------
        func: function
            Function to apply
        $META
        columns: list, scalar or None
            Deprecated, use `meta` instead. If list is given, the result is a
            DataFrame which columns is specified list. Otherwise, the result is
            a Series which name is given scalar or None (no name). If name
            keyword is not given, dask tries to infer the result type using its
            beginning of data. This inference may take some time and lead to
            unexpected result

        Returns
        -------
        applied : Series or DataFrame depending on columns keyword
        """
        if columns is not no_default:
            warnings.warn("`columns` is deprecated, please use `meta` instead")
            if meta is no_default and isinstance(columns, (pd.DataFrame, pd.Series)):
                meta = columns
        if meta is no_default:
            msg = ("`meta` is not specified, inferred from partial data. "
                   "Please provide `meta` if the result is unexpected.\n"
                   "  Before: .apply(func)\n"
                   "  After:  .apply(func, meta={'x': 'f8', 'y': 'f8'}) for dataframe result\n"
                   "  or:     .apply(func, meta=('x', 'f8'))            for series result")
            warnings.warn(msg)

            with raise_on_meta_error("groupby.apply({0})".format(funcname(func))):
                meta = self._meta_nonempty.apply(func)
        else:
            meta = make_meta(meta)

        df = self.obj
        if isinstance(self.index, DataFrame):  # add index columns to dataframe
            df2 = df.assign(**{'_index_' + c: self.index[c]
                                for c in self.index.columns})
            index = self.index
        elif isinstance(self.index, Series):
            df2 = df.assign(_index=self.index)
            index = self.index
        else:
            df2 = df
            index = df[self.index]

        df3 = shuffle(df2, index, **self.kwargs)  # shuffle dataframe and index

        if isinstance(self.index, DataFrame):  # extract index from dataframe
            cols = ['_index_' + c for c in self.index.columns]
            index2 = df3[cols]
            df4 = df3.drop(cols, axis=1, dtype=meta.columns.dtype if
                           isinstance(meta, pd.DataFrame) else None)
        elif isinstance(self.index, Series):
            index2 = df3['_index']
            index2.name = self.index.name
            df4 = df3.drop('_index', axis=1, dtype=meta.columns.dtype if
                           isinstance(meta, DataFrame) else None)
        else:
            df4 = df3
            index2 = self.index

        # Perform embarrassingly parallel groupby-apply
        df5 = map_partitions(_groupby_slice_apply, df4, index2,
                             self._slice, func, meta=meta)

        return df5

class DataFrameGroupBy(_GroupBy):

    _token_prefix = 'dataframe-groupby-'

    def __init__(self, df, index=None, slice=None, **kwargs):

        if not kwargs.get('as_index', True):
            msg = ("The keyword argument `as_index=False` is not supported in "
                   "dask.dataframe.groupby")
            raise NotImplementedError(msg)

        super(DataFrameGroupBy, self).__init__(df, index=index,
                                               slice=slice, **kwargs)

    def __getitem__(self, key):
        if isinstance(key, list):
            g = DataFrameGroupBy(self.obj, index=self.index,
                                 slice=key, **self.kwargs)
        else:
            g = SeriesGroupBy(self.obj, index=self.index,
                              slice=key, **self.kwargs)

        # error is raised from pandas
        g._meta = g._meta[key]
        return g

    def __dir__(self):
        return sorted(set(dir(type(self)) + list(self.__dict__) +
                      list(filter(pd.compat.isidentifier, self.obj.columns))))

    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError as e:
            raise AttributeError(e)

class SeriesGroupBy(_GroupBy):

    _token_prefix = 'series-groupby-'

    def __init__(self, df, index, slice=None, **kwargs):

        # raise pandas-compat error message
        if isinstance(df, Series):
            # When obj is Series, index must be Series
            if not isinstance(index, Series):
                if isinstance(index, list):
                    if len(index) == 0:
                        raise ValueError("No group keys passed!")
                    msg = "Grouper for '{0}' not 1-dimensional"
                    raise ValueError(msg.format(index[0]))
                # raise error from pandas
                df._meta.groupby(index)
        super(SeriesGroupBy, self).__init__(df, index=index,
                                            slice=slice, **kwargs)

    def nunique(self):
        name = self._meta.obj.name
        meta = pd.Series([], dtype='int64',
                         index=pd.Index([], dtype=self._meta.obj.dtype),
                         name=name)

        if isinstance(self.obj, DataFrame):
            return aca([self.obj, self.index],
                       chunk=_nunique_df_chunk,
                       aggregate=_nunique_df_aggregate,
                       meta=meta, token='series-groupby-nunique',
                       aggregate_kwargs={'name': name})
        else:
            return aca([self.obj, self.index],
                       chunk=_nunique_series_chunk,
                       aggregate=_nunique_series_aggregate,
                       meta=meta, token='series-groupby-nunique')
<commit_msg>Fix segfault in groupby-var<commit_after>from __future__ import absolute_import, division, print_function

import warnings

import numpy as np
import pandas as pd

from .core import DataFrame, Series, Index, aca, map_partitions, no_default
from .shuffle import shuffle
from .utils import make_meta, insert_meta_param_description, raise_on_meta_error
from ..utils import derived_from, M, funcname

def _maybe_slice(grouped, columns):
    """
    Slice columns if grouped is pd.DataFrameGroupBy
    """
    if isinstance(grouped, pd.core.groupby.DataFrameGroupBy):
        if columns is not None:
            columns = columns if isinstance(columns, str) else list(columns)
            return grouped[columns]
    return grouped

def _groupby_slice_apply(df, grouper, key, func):
    g = df.groupby(grouper)
    if key:
        g = g[key]
    return g.apply(func)

def _groupby_get_group(df, by_key, get_key, columns):
    # SeriesGroupBy may pass df which includes group key
    grouped = df.groupby(by_key)

    if get_key in grouped.groups:
        if isinstance(df, pd.DataFrame):
            grouped = grouped[columns]
        return grouped.get_group(get_key)

    else:
        # to create empty DataFrame/Series, which has the same
        # dtype as the original
        if isinstance(df, pd.DataFrame):
            # may be SeriesGroupBy
            df = df[columns]
        return df.iloc[0:0]

###############################################################
# Aggregation
###############################################################

def _groupby_aggregate(df, aggfunc=None, levels=None):
    return aggfunc(df.groupby(level=levels))

def _apply_chunk(df, index, func, columns):
    if isinstance(df, pd.Series):
        return func(df.groupby(index))
    else:
        columns = columns if isinstance(columns, str) else list(columns)
        return func(df.groupby(index)[columns])

def _var_chunk(df, index):
    if isinstance(df, pd.Series):
        df = df.to_frame()
    x = df.groupby(index).sum()
    x2 = (df**2).rename(columns=lambda c: c + '-x2')
    cols = [c + '-x2' for c in x.columns]
    x2 = pd.concat([df, x2], axis=1).groupby(index)[cols].sum()
    n = (df.groupby(index).count()
           .rename(columns=lambda c: c + '-count'))

    result = pd.concat([x, x2, n], axis=1)
    return result

def _var_agg(g, ddof):
    g = g.groupby(level=0).sum()
    nc = len(g.columns)
    x = g[g.columns[:nc // 3]]
    x2 = g[g.columns[nc // 3:2 * nc // 3]].rename(columns=lambda c: c[:-3])
    n = g[g.columns[-nc // 3:]].rename(columns=lambda c: c[:-6])

    result = x2 - x ** 2 / n
    div = (n - ddof)
    div[div < 0] = 0
    result /= div
    result[(n - ddof) == 0] = np.nan
    assert isinstance(result, pd.DataFrame)
    return result

###############################################################
# nunique
###############################################################

def _nunique_df_chunk(df, index):
    # we call set_index here to force a possibly duplicate index
    # for our reduce step
    grouped = (df.groupby(index).apply(pd.DataFrame.drop_duplicates))
    grouped.index = grouped.index.get_level_values(level=0)
    return grouped

def _nunique_df_aggregate(df, name):
    return df.groupby(level=0)[name].nunique()

def _nunique_series_chunk(df, index):
    assert isinstance(df, pd.Series)
    if isinstance(index, np.ndarray):
        assert len(index) == len(df)
        index = pd.Series(index, index=df.index)
    grouped = pd.concat([df, index], axis=1).drop_duplicates()
    return grouped

def _nunique_series_aggregate(df):
    return df.groupby(df.columns[1])[df.columns[0]].nunique()

class _GroupBy(object):
    """ Superclass for DataFrameGroupBy and SeriesGroupBy

    Parameters
    ----------

    obj: DataFrame or Series
        DataFrame or Series to be grouped
    index: str, list or Series
        The key for grouping
    kwargs: dict
        Other keywords passed to groupby
    """
    def __init__(self, df, index=None, slice=None, **kwargs):
        assert isinstance(df, (DataFrame, Series))
        self.obj = df

        # grouping key passed via groupby method
        if (isinstance(index, (DataFrame, Series, Index)) and
           isinstance(df, DataFrame)):

            if (isinstance(index, Series) and index.name in df.columns and
               index._name == df[index.name]._name):
                index = index.name
            elif (isinstance(index, DataFrame) and
                  set(index.columns).issubset(df.columns) and
                  index._name == df[index.columns]._name):
                index = list(index.columns)

        self.index = index

        # slicing key applied to _GroupBy instance
        self._slice = slice

        self.kwargs = kwargs

        if isinstance(index, Series) and df.divisions != index.divisions:
            msg = ("The Series and index of the groupby"
                   " must have the same divisions.")
            raise NotImplementedError(msg)

        if self._is_grouped_by_sliced_column(self.obj, index):
            # check whether given Series is taken from given df and unchanged.
            # If any operations are performed, _name will be changed to
            # e.g. "elemwise-xxxx"

            # if group key (index) is a Series sliced from DataFrame,
            # emulation must be performed as the same.
            # otherwise, group key is regarded as a separate column
            self._meta = self.obj._meta.groupby(self.obj._meta[index.name])

        elif isinstance(self.index, Series):
            self._meta = self.obj._meta.groupby(self.index._meta)
        else:
            self._meta = self.obj._meta.groupby(self.index)

    def _is_grouped_by_sliced_column(self, df, index):
        """
        Return whether index is a Series sliced from df
        """
        if isinstance(df, Series):
            return False
        if (isinstance(index, Series) and index._name in df.columns and
                index._name == df[index.name]._name):
            return True
        if (isinstance(index, DataFrame) and
                set(index.columns).issubset(df.columns) and
                index._name == df[index.columns]._name):
            index = list(index.columns)
            return True
        return False

    @property
    def _meta_nonempty(self):
        """
        Return a pd.DataFrameGroupBy / pd.SeriesGroupBy which contains sample data.
        """
        sample = self.obj._meta_nonempty
        if isinstance(self.index, Series):
            if self._is_grouped_by_sliced_column(self.obj, self.index):
                grouped = sample.groupby(sample[self.index.name])
            else:
                grouped = sample.groupby(self.index._meta_nonempty)
        else:
            grouped = sample.groupby(self.index)
        return _maybe_slice(grouped, self._slice)

    def _aca_agg(self, token, func, aggfunc=None):
        if aggfunc is None:
            aggfunc = func

        meta = func(self._meta)
        columns = meta.name if isinstance(meta, pd.Series) else meta.columns

        token = self._token_prefix + token

        if isinstance(self.index, (tuple, list)) and len(self.index) > 1:
            levels = list(range(len(self.index)))
        else:
            levels = 0

        return aca([self.obj, self.index, func, columns],
                   chunk=_apply_chunk, aggregate=_groupby_aggregate,
                   meta=meta, token=token,
                   aggregate_kwargs=dict(aggfunc=aggfunc, levels=levels))

    @derived_from(pd.core.groupby.GroupBy)
    def sum(self):
        return self._aca_agg(token='sum', func=M.sum)

    @derived_from(pd.core.groupby.GroupBy)
    def min(self):
        return self._aca_agg(token='min', func=M.min)

    @derived_from(pd.core.groupby.GroupBy)
    def max(self):
        return self._aca_agg(token='max', func=M.max)

    @derived_from(pd.core.groupby.GroupBy)
    def count(self):
        return self._aca_agg(token='count', func=M.count,
                             aggfunc=M.sum)

    @derived_from(pd.core.groupby.GroupBy)
    def mean(self):
        return self.sum() / self.count()

    @derived_from(pd.core.groupby.GroupBy)
    def var(self, ddof=1):
        result = aca([self.obj, self.index], chunk=_var_chunk,
                     aggregate=_var_agg,
                     token=self._token_prefix + 'var',
                     aggregate_kwargs={'ddof': ddof})

        if isinstance(self.obj, Series):
            result = result[result.columns[0]]
        if self._slice:
            result = result[self._slice]

        return result

    @derived_from(pd.core.groupby.GroupBy)
    def std(self, ddof=1):
        v = self.var(ddof)
        result = map_partitions(np.sqrt, v, meta=v)
        return result

    @derived_from(pd.core.groupby.GroupBy)
    def get_group(self, key):
        token = self._token_prefix + 'get_group'

        meta = self._meta.obj
        if isinstance(meta, pd.DataFrame) and self._slice is not None:
            meta = meta[self._slice]
        columns = meta.columns if isinstance(meta, pd.DataFrame) else meta.name

        return map_partitions(_groupby_get_group, self.obj, self.index, key,
                              columns, meta=meta, token=token)

    @insert_meta_param_description(pad=12)
    def apply(self, func, meta=no_default, columns=no_default):
        """ Parallel version of pandas GroupBy.apply

        This mimics the pandas version except for the following:

        1.  The user should provide output metadata.
        2.  If the grouper does not align with the index then this causes a full
            shuffle.  The order of rows within each group may not be preserved.

        Parameters
        ----------
        func: function
            Function to apply
        $META
        columns: list, scalar or None
            Deprecated, use `meta` instead. If list is given, the result is a
            DataFrame which columns is specified list. Otherwise, the result is
            a Series which name is given scalar or None (no name). If name
            keyword is not given, dask tries to infer the result type using its
            beginning of data. This inference may take some time and lead to
            unexpected result

        Returns
        -------
        applied : Series or DataFrame depending on columns keyword
        """
        if columns is not no_default:
            warnings.warn("`columns` is deprecated, please use `meta` instead")
            if meta is no_default and isinstance(columns, (pd.DataFrame, pd.Series)):
                meta = columns
        if meta is no_default:
            msg = ("`meta` is not specified, inferred from partial data. "
                   "Please provide `meta` if the result is unexpected.\n"
                   "  Before: .apply(func)\n"
                   "  After:  .apply(func, meta={'x': 'f8', 'y': 'f8'}) for dataframe result\n"
                   "  or:     .apply(func, meta=('x', 'f8'))            for series result")
            warnings.warn(msg)

            with raise_on_meta_error("groupby.apply({0})".format(funcname(func))):
                meta = self._meta_nonempty.apply(func)
        else:
            meta = make_meta(meta)

        df = self.obj
        if isinstance(self.index, DataFrame):  # add index columns to dataframe
            df2 = df.assign(**{'_index_' + c: self.index[c]
                                for c in self.index.columns})
            index = self.index
        elif isinstance(self.index, Series):
            df2 = df.assign(_index=self.index)
            index = self.index
        else:
            df2 = df
            index = df[self.index]

        df3 = shuffle(df2, index, **self.kwargs)  # shuffle dataframe and index

        if isinstance(self.index, DataFrame):  # extract index from dataframe
            cols = ['_index_' + c for c in self.index.columns]
            index2 = df3[cols]
            df4 = df3.drop(cols, axis=1, dtype=meta.columns.dtype if
                           isinstance(meta, pd.DataFrame) else None)
        elif isinstance(self.index, Series):
            index2 = df3['_index']
            index2.name = self.index.name
            df4 = df3.drop('_index', axis=1, dtype=meta.columns.dtype if
                           isinstance(meta, DataFrame) else None)
        else:
            df4 = df3
            index2 = self.index

        # Perform embarrassingly parallel groupby-apply
        df5 = map_partitions(_groupby_slice_apply, df4, index2,
                             self._slice, func, meta=meta)

        return df5

class DataFrameGroupBy(_GroupBy):

    _token_prefix = 'dataframe-groupby-'

    def __init__(self, df, index=None, slice=None, **kwargs):

        if not kwargs.get('as_index', True):
            msg = ("The keyword argument `as_index=False` is not supported in "
                   "dask.dataframe.groupby")
            raise NotImplementedError(msg)

        super(DataFrameGroupBy, self).__init__(df, index=index,
                                               slice=slice, **kwargs)

    def __getitem__(self, key):
        if isinstance(key, list):
            g = DataFrameGroupBy(self.obj, index=self.index,
                                 slice=key, **self.kwargs)
        else:
            g = SeriesGroupBy(self.obj, index=self.index,
                              slice=key, **self.kwargs)

        # error is raised from pandas
        g._meta = g._meta[key]
        return g

    def __dir__(self):
        return sorted(set(dir(type(self)) + list(self.__dict__) +
                      list(filter(pd.compat.isidentifier, self.obj.columns))))

    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError as e:
            raise AttributeError(e)

class SeriesGroupBy(_GroupBy):

    _token_prefix = 'series-groupby-'

    def __init__(self, df, index, slice=None, **kwargs):

        # raise pandas-compat error message
        if isinstance(df, Series):
            # When obj is Series, index must be Series
            if not isinstance(index, Series):
                if isinstance(index, list):
                    if len(index) == 0:
                        raise ValueError("No group keys passed!")
                    msg = "Grouper for '{0}' not 1-dimensional"
                    raise ValueError(msg.format(index[0]))
                # raise error from pandas
                df._meta.groupby(index)
        super(SeriesGroupBy, self).__init__(df, index=index,
                                            slice=slice, **kwargs)

    def nunique(self):
        name = self._meta.obj.name
        meta = pd.Series([], dtype='int64',
                         index=pd.Index([], dtype=self._meta.obj.dtype),
                         name=name)

        if isinstance(self.obj, DataFrame):
            return aca([self.obj, self.index],
                       chunk=_nunique_df_chunk,
                       aggregate=_nunique_df_aggregate,
                       meta=meta, token='series-groupby-nunique',
                       aggregate_kwargs={'name': name})
        else:
            return aca([self.obj, self.index],
                       chunk=_nunique_series_chunk,
                       aggregate=_nunique_series_aggregate,
                       meta=meta, token='series-groupby-nunique')
<|endoftext|>


<commit_before>from qgl2.qgl2 import concur, qbit, qgl2decl, sequence, pulse, qgl2main
from qgl2.qgl1 import QubitFactory, X90, Y90

@qgl2decl
def func_c(q: qbit):

	for i in range(0,3):
 		func_b(q)

@qgl2decl
def func_b(q: qbit):

	for i in range(0,3):
 		func_a(q)

@qgl2decl
def func_a(q: qbit) -> pulse:

	for i in range(0,3):
 		X90(q)

@qgl2main
def main():

    q = Qubit("1")
    func_c(q)

<commit_msg>updated to use QubitFactory<commit_after>from qgl2.qgl2 import concur, qbit, qgl2decl, sequence, pulse, qgl2main
from qgl2.qgl1 import QubitFactory, X90, Y90

@qgl2decl
def func_c(q: qbit):

	for i in range(0,3):
 		func_b(q)

@qgl2decl
def func_b(q: qbit):

	for i in range(0,3):
 		func_a(q)

@qgl2decl
def func_a(q: qbit) -> pulse:

	for i in range(0,3):
 		X90(q)

@qgl2main
def main():

    q = QubitFactory("1")
    func_c(q)

<|endoftext|>