<commit_before>"""
Wrapper around the JIRA API to allow us to categorize issues by
project/component/label etc and report on:

- Work in Progress
- Work completed - including Cycle Time
- Work history
- Cumulative Flow
- Throughput
- Rate at which types of work are created

Also abstracts away from batch searching and other implementation
details we don't want to present to the user.
"""

import jira.client
import sys

from datetime import date, datetime

import pandas as pd
import numpy as np
import math

from index import fill_date_index_blanks, week_start_date
from history import time_in_states, cycle_time, arrivals, history_from_jira_changelog
from bucket import bucket_labels

from collections import Counter

class MissingState(Exception):

    def __init__(self, expr, msg):
        self.expr = expr
        self.msg = msg

    def __str__(self):

        return self.msg

class MissingConfigItem(Exception):

    def __init__(self, expr, msg):
        self.expr = expr
        self.msg = msg

    def __str__(self):

        return self.msg

class JiraWrapper(object):
    """
    Wrapper around our JIRA instance
    """

    def __init__(self, config):

        authentication = None

        try:
            authentication = config['authentication']
        except KeyError as e:
            raise MissingConfigItem(e, "Missing Config Item:{0}".format(e))

        self.jira = None

        if 'username' in authentication and 'password' in authentication:
            self.jira = jira.client.JIRA({'server': config['server']},
                                         basic_auth=(authentication['username'],
                                                     authentication['password']))
        elif ('access_token' in authentication and
              'access_token_secret' in authentication and
              'consumer_key' in authentication and
              'key_cert'):

            try:
                with open(authentication['key_cert'], 'r') as key_cert_file:
                    key_cert_data = key_cert_file.read()
            except IOError:
                raise MissingConfigItem('key_cert', "key_cert not found:{0}". format(authentication['key_cert']))

            self.jira = jira.client.JIRA({'server': config['server']},
                                         oauth={'access_token': authentication['access_token'],
                                                'access_token_secret': authentication['access_token_secret'],
                                                'consumer_key': authentication['consumer_key'],
                                                'key_cert': key_cert_data})
        else:
            raise MissingConfigItem('authentication', "Authentication misconfigured")

        self.categories = None
        self.cycles = None
        self.types = None
        self.states = []
        self.until_date = None

        if 'until_date' in config:
            self.until_date = datetime.strptime(config['until_date'], '%Y-%m-%d').date()

        if 'throughput_dow' in config:
            self.throughput_dow = config['throughput_dow']
        else:
            self.throughput_dow = 4

        try:
            self.categories = config['categories']
            self.cycles = config['cycles']
            self.types = config['types']
            self.counts_towards_throughput = config['counts_towards_throughput']
        except KeyError as e:
            raise MissingConfigItem(e.message, "Missing Config Item:{0}".format(e.message))

        # Optional

        try:
            self.states = config['states']
            self.states.append(None)
        except KeyError:
            pass

        self.all_issues = None

    def issue(self, key):
        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        matches = [issue for issue in self.all_issues if issue.key == key]
        return matches[0]

    def issues(self, fields=None):
        """
        All issues
        """
        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        issue_rows = self._issues_as_rows(self.all_issues)
        df = pd.DataFrame(issue_rows)

        if fields is None:
            return df
        else:
            return df.filter(fields)

    def history(self, from_date=None, until_date=None, types=None):

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        history = {}

        for issue in self.all_issues:

            if types is None:
                history[issue.key] = issue.history
            else:
                for type_grouping in types:
                    if issue.fields.issuetype.name in self.types[type_grouping]:
                        history[issue.key] = issue.history

        if history is not None:
            df = pd.DataFrame(history)
            return df

        return None

    def cfd(self, from_date=None, until_date=None, types=None):
        """
        Cumulative Flow Diagram
        """

        cfd = self.history(from_date, until_date, types)

        days = {}

        for day in cfd.index:
            tickets = []
            for ticket in cfd.ix[day]:
                tickets.append(ticket)

            def state_order(state):

                try:
                    return self.states.index(state)
                except ValueError:
                    if type(state) == float:
                        if math.isnan(state):
                            return -1
                    if type(state) == np.float64:
                        if math.isnan(state):
                            return -1

                    raise MissingState(state, "Missing state:{0}".format(state))

            days[day] = sorted(tickets, key=state_order)

        return pd.DataFrame(days)

    def demand(self,
               from_date,
               to_date,
               types=None):
        """
        Return the number of issues created each week - i.e. the demand on the system
        """

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        issue_rows = self._issues_as_rows(self.all_issues, types)

        df = pd.DataFrame(issue_rows)
        table = pd.tools.pivot.pivot_table(df, rows=['week_created'], cols=['swimlane'], values='count', aggfunc=np.count_nonzero)

        reindexed = table.reindex(index=fill_date_index_blanks(table.index), fill_value=np.int64(0))
        reindexed.index.name = "week"
        return reindexed

    def throughput(self,
                   from_date,
                   to_date,
                   cumulative=True,
                   category=None,
                   types=None):
        """
        Return throughput for all issues in a state where they are considered
        to count towards throughput.

        Might want to add some additional conditions other than state but state
        allows us the most options as to where to place the 'finishing line'
        """

        issue_history = self.history(from_date, to_date)

        issue_rows = []

        for issue_key in issue_history:

            issue = self.issue(issue_key)

            if category is not None:

                if category != issue.category:
                    continue

            swimlane = issue.category

            # Are we grouping by work type?
            f = issue.fields

            if types is not None:
                for type_grouping in types:
                    if f.issuetype.name in self.types[type_grouping]:
                        swimlane = swimlane + '-' + type_grouping
                    else:
                        continue
                        # print "Not counting " + f.issuetype.name
                if swimlane == issue.category:
                    continue

            for day, state in issue_history[issue_key].iteritems():
                if day.weekday() == self.throughput_dow:

                    if state in self.counts_towards_throughput:

                        issue_row = {'swimlane': swimlane,
                                     'id':       issue_key,
                                     'week':     day,
                                     'count':    1}
                        issue_rows.append(issue_row)

        df = pd.DataFrame(issue_rows)

        if len(df.index) > 0:

            table = pd.tools.pivot.pivot_table(df, rows=['week'], cols=['swimlane'], values='count', aggfunc=np.count_nonzero)

            if cumulative:
                return table

            else:

                def foo_func(x):
                    for i in range(x.size-1, 0, -1):
                        x[i] = x[i] - x[i-1]
                    return x

                table = table.apply(foo_func)

                return table

        else:

            return None

    def arrival_rate(self,
                     from_date,
                     to_date):
        """
        So that we can get an idea of the flow of work that has not been completed and so does not have a resolution date
        and so does not count towards throughput, what is the rate at which that work arrived at states further up the
        value chain?
        """

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        arrivals_count = {}

        for issue in self.all_issues:
            try:
                arrivals_count = arrivals(issue.changelog.histories, arrivals_count)
            except AttributeError as e:
                print e

        df = pd.DataFrame.from_dict(arrivals_count, orient='index')
        df.index = pd.to_datetime(df.index)
        wf = df.resample('W-MON', how='sum')

        return wf

    def cycle_time_histogram(self,
                             cycle,
                             types=None,
                             buckets=None):
        """
        Time taken for work to complete one or more 'cycles' - i.e. transitions from a start state to an end state
        """

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        rows = self._issues_as_rows(self.all_issues)

        cycle_time_data = {}

        for row in rows:

            if types is not None:

                include = False

                for type_grouping in types:

                    if row['type'] in self.types[type_grouping]:
                        include = True
                        key = "{0}-{1}".format(type_grouping, cycle)
                        break

                if not include:
                    continue

            else:
                key = cycle

            try:
                if row[cycle] is not None:
                    if key not in cycle_time_data:
                        cycle_time_data[key] = [row[cycle]]
                    else:
                        cycle_time_data[key].append(row[cycle])
            except KeyError:
                continue

        histogram = None

        for cycle in cycle_time_data:
            if buckets is not None:

                try:
                    li = buckets.index('max')
                    buckets[li] = max(cycle_time_data[cycle])

                except ValueError:
                    pass

                labels = bucket_labels(buckets)
                count, division = np.histogram(cycle_time_data[cycle], bins=buckets)
            else:

                count, division = np.histogram(cycle_time_data[cycle])
                labels = bucket_labels(division)

            cycle_histogram = pd.DataFrame(count, index=labels, columns=[cycle])
            cycle_histogram.index.name = 'bucket'

            if histogram is None:
                histogram = cycle_histogram.copy(deep=True)
            else:
                old_histogram = histogram.copy(deep=True)
                histogram = old_histogram.join(cycle_histogram, how='outer')

        return histogram

    def totals(self):
        """
        What are current totals of work in our various states
        """

        # We can get this by doing a count of the last day of the CFD

        cfd = self.cfd()

        return None

###############################################################################
# Internal methods
###############################################################################

    def _issues_from_jira(self, filter=None):
        """
        Get the actual issues from Jira itself via the Jira REST API
        """

        batch_size = 100
        issues = []

        for category in self.categories:

            n = 0
            while 1:

                jql = self.categories[category]
                if filter is not None:
                    jql = jql + filter

                issue_batch = self.jira.search_issues(jql,
                                                      startAt=n,
                                                      maxResults=batch_size,
                                                      expand='changelog')

                if issue_batch is None:
                    #TODO: Fix mocking so we can get rid of this.
                    # 'expand' seems to have some magic meaning in Mockito...
                    issue_batch = self.jira.search_issues(jql,
                                                          startAt=n,
                                                          maxResults=batch_size)

                for issue in issue_batch:

                    issue.category = category

                    created_date = datetime.strptime(issue.fields.created[:10], '%Y-%m-%d')
                    if issue.changelog is not None:
                        issue.history = history_from_jira_changelog(issue.changelog, created_date, self.until_date)

                    try:
                        for cycle in self.cycles:
                            reopened_state = None
                            after_state = None
                            start_state = None
                            exit_state = None
                            end_state = None
                            include_states = None
                            exclude_states = None

                            if 'ignore' in self.cycles[cycle]:
                                reopened_state = self.cycles[cycle]['ignore']

                            if 'after' in self.cycles[cycle]:
                                after_state = self.cycles[cycle]['after']

                            if 'start' in self.cycles[cycle]:
                                start_state = self.cycles[cycle]['start']

                            if 'exit' in self.cycles[cycle]:
                                exit_state = self.cycles[cycle]['exit']

                            if 'end' in self.cycles[cycle]:
                                end_state = self.cycles[cycle]['end']

                            if 'include' in self.cycles[cycle]:
                                include_states = self.cycles[cycle]['include']

                            if 'exclude' in self.cycles[cycle]:
                                exclude_states = self.cycles[cycle]['exclude']

                                setattr(issue,
                                        cycle,
                                        cycle_time(issue.history,
                                                   start_state=start_state,
                                                   after_state=after_state,
                                                   exit_state=exit_state,
                                                   end_state=end_state,
                                                   include_states=include_states,
                                                   exclude_states=exclude_states,
                                                   reopened_state=reopened_state))

                    except AttributeError:

                        pass

                    issues.append(issue)

                if len(issue_batch) < batch_size:
                    break
                n += batch_size
                sys.stdout.write('.')
                sys.stdout.flush()

        return issues

    def _issues_as_rows(self, issues, types=None):

        """
        Get issues into a state where we can stick them into a Pandas dataframe
        """

        # TODO: Decide if this should be a class / helper method?
        # TODO: See if has any overlap with throughput function

        issue_rows = []

        for issue in issues:
            f = issue.fields

            resolution_date_str = f.resolutiondate

            if resolution_date_str is not None:
                resolution_date = datetime.strptime(resolution_date_str[:10],
                                                    '%Y-%m-%d')

                week = week_start_date(resolution_date.isocalendar()[0],
                                       resolution_date.isocalendar()[1]).strftime('%Y-%m-%d')

            else:

                week = None

            date_created = datetime.strptime(f.created[:10], '%Y-%m-%d')
            week_created = week_start_date(date_created.isocalendar()[0],
                                           date_created.isocalendar()[1]).strftime('%Y-%m-%d')

            if issue.changelog is not None:
                tis = time_in_states(issue.changelog.histories,
                                     datetime.strptime(f.created[:10],
                                                       '%Y-%m-%d'),
                                     date.today())

                since = tis[-1]['days']

            else:

                since = None

            include = True

            # TODO: This looks like a bit of a hack.
            # Can we do without iterating over loop, seeing as we
            # only ever want one swimlane/category combination?

            swimlane = issue.category

            if types is not None and self.types is not None:
                include = False
                for type_grouping in types:
                    if f.issuetype.name in self.types[type_grouping]:
                        swimlane = swimlane + '-' + type_grouping
                        include = True

            if include:

                try:
                    story_points = f.customfield_10002
                except AttributeError:
                    story_points = None

                try:
                    epic_link = f.customfield_10200
                except AttributeError:
                    epic_link = None

                # TODO: Fields need to come out of config too.

                issue_row = {'swimlane':     swimlane,
                             'type':         f.issuetype.name,
                             'id':           issue.key,
                             'name':         f.summary,
                             'status':       f.status.name,
                             'project':      f.project.name,
                             'components':   None,
                             'week':         week,
                             'since':        since,
                             'created':      f.created,
                             'week_created': week_created,
                             'story_points': story_points,
                             'epic_link':    epic_link,
                             'count':        1}

                for cycle in self.cycles:
                    try:
                        issue_row[cycle] = getattr(issue, cycle)
                    except AttributeError:
                        pass

                components = []

                for component in f.components:
                    components.append(component.name)

                issue_row['components'] = ",".join(components)

                issue_rows.append(issue_row)

        return issue_rows
<commit_msg>Fixed broken tests<commit_after>"""
Wrapper around the JIRA API to allow us to categorize issues by
project/component/label etc and report on:

- Work in Progress
- Work completed - including Cycle Time
- Work history
- Cumulative Flow
- Throughput
- Rate at which types of work are created

Also abstracts away from batch searching and other implementation
details we don't want to present to the user.
"""

import jira.client
import sys

from datetime import date, datetime

import pandas as pd
import numpy as np
import math

from index import fill_date_index_blanks, week_start_date
from history import time_in_states, cycle_time, arrivals, history_from_jira_changelog
from bucket import bucket_labels

from collections import Counter

class MissingState(Exception):

    def __init__(self, expr, msg):
        self.expr = expr
        self.msg = msg

    def __str__(self):

        return self.msg

class MissingConfigItem(Exception):

    def __init__(self, expr, msg):
        self.expr = expr
        self.msg = msg

    def __str__(self):

        return self.msg

class JiraWrapper(object):
    """
    Wrapper around our JIRA instance
    """

    def __init__(self, config):

        authentication = None

        try:
            authentication = config['authentication']
        except KeyError as e:
            raise MissingConfigItem(e, "Missing Config Item:{0}".format(e))

        self.jira = None

        if 'username' in authentication and 'password' in authentication:
            self.jira = jira.client.JIRA({'server': config['server']},
                                         basic_auth=(authentication['username'],
                                                     authentication['password']))
        elif ('access_token' in authentication and
              'access_token_secret' in authentication and
              'consumer_key' in authentication and
              'key_cert'):

            try:
                with open(authentication['key_cert'], 'r') as key_cert_file:
                    key_cert_data = key_cert_file.read()
            except IOError:
                raise MissingConfigItem('key_cert', "key_cert not found:{0}". format(authentication['key_cert']))

            self.jira = jira.client.JIRA({'server': config['server']},
                                         oauth={'access_token': authentication['access_token'],
                                                'access_token_secret': authentication['access_token_secret'],
                                                'consumer_key': authentication['consumer_key'],
                                                'key_cert': key_cert_data})
        else:
            raise MissingConfigItem('authentication', "Authentication misconfigured")

        self.categories = None
        self.cycles = None
        self.types = None
        self.states = []
        self.until_date = None

        if 'until_date' in config:
            self.until_date = datetime.strptime(config['until_date'], '%Y-%m-%d').date()

        if 'throughput_dow' in config:
            self.throughput_dow = config['throughput_dow']
        else:
            self.throughput_dow = 4

        try:
            self.categories = config['categories']
            self.cycles = config['cycles']
            self.types = config['types']
            self.counts_towards_throughput = config['counts_towards_throughput']
        except KeyError as e:
            raise MissingConfigItem(e.message, "Missing Config Item:{0}".format(e.message))

        # Optional

        try:
            self.states = config['states']
            self.states.append(None)
        except KeyError:
            pass

        self.all_issues = None

    def issue(self, key):
        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        matches = [issue for issue in self.all_issues if issue.key == key]
        return matches[0]

    def issues(self, fields=None):
        """
        All issues
        """
        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        issue_rows = self._issues_as_rows(self.all_issues)
        df = pd.DataFrame(issue_rows)

        if fields is None:
            return df
        else:
            return df.filter(fields)

    def history(self, from_date=None, until_date=None, types=None):

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        history = {}

        for issue in self.all_issues:

            if types is None:
                history[issue.key] = issue.history
            else:
                for type_grouping in types:
                    if issue.fields.issuetype.name in self.types[type_grouping]:
                        history[issue.key] = issue.history

        if history is not None:
            df = pd.DataFrame(history)
            return df

        return None

    def cfd(self, from_date=None, until_date=None, types=None):
        """
        Cumulative Flow Diagram
        """

        cfd = self.history(from_date, until_date, types)

        days = {}

        for day in cfd.index:
            tickets = []
            for ticket in cfd.ix[day]:
                tickets.append(ticket)

            def state_order(state):

                try:
                    return self.states.index(state)
                except ValueError:
                    if type(state) == float:
                        if math.isnan(state):
                            return -1
                    if type(state) == np.float64:
                        if math.isnan(state):
                            return -1

                    raise MissingState(state, "Missing state:{0}".format(state))

            days[day] = sorted(tickets, key=state_order)

        return pd.DataFrame(days)

    def demand(self,
               from_date,
               to_date,
               types=None):
        """
        Return the number of issues created each week - i.e. the demand on the system
        """

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        issue_rows = self._issues_as_rows(self.all_issues, types)

        df = pd.DataFrame(issue_rows)
        table = pd.tools.pivot.pivot_table(df, rows=['week_created'], cols=['swimlane'], values='count', aggfunc=np.count_nonzero)

        reindexed = table.reindex(index=fill_date_index_blanks(table.index), fill_value=np.int64(0))
        reindexed.index.name = "week"
        return reindexed

    def throughput(self,
                   from_date,
                   to_date,
                   cumulative=True,
                   category=None,
                   types=None):
        """
        Return throughput for all issues in a state where they are considered
        to count towards throughput.

        Might want to add some additional conditions other than state but state
        allows us the most options as to where to place the 'finishing line'
        """

        issue_history = self.history(from_date, to_date)

        issue_rows = []

        for issue_key in issue_history:

            issue = self.issue(issue_key)

            if category is not None:

                if category != issue.category:
                    continue

            swimlane = issue.category

            # Are we grouping by work type?
            f = issue.fields

            if types is not None:
                for type_grouping in types:
                    if f.issuetype.name in self.types[type_grouping]:
                        swimlane = swimlane + '-' + type_grouping
                    else:
                        continue
                        # print "Not counting " + f.issuetype.name
                if swimlane == issue.category:
                    continue

            for day, state in issue_history[issue_key].iteritems():
                if day.weekday() == self.throughput_dow:

                    if state in self.counts_towards_throughput:

                        issue_row = {'swimlane': swimlane,
                                     'id':       issue_key,
                                     'week':     day,
                                     'count':    1}
                        issue_rows.append(issue_row)

        df = pd.DataFrame(issue_rows)

        if len(df.index) > 0:

            table = pd.tools.pivot.pivot_table(df, rows=['week'], cols=['swimlane'], values='count', aggfunc=np.count_nonzero)

            if cumulative:
                return table

            else:

                def foo_func(x):
                    for i in range(x.size-1, 0, -1):
                        x[i] = x[i] - x[i-1]
                    return x

                table = table.apply(foo_func)

                return table

        else:

            return None

    def arrival_rate(self,
                     from_date,
                     to_date):
        """
        So that we can get an idea of the flow of work that has not been completed and so does not have a resolution date
        and so does not count towards throughput, what is the rate at which that work arrived at states further up the
        value chain?
        """

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        arrivals_count = {}

        for issue in self.all_issues:
            try:
                arrivals_count = arrivals(issue.changelog.histories, arrivals_count)
            except AttributeError as e:
                print e

        df = pd.DataFrame.from_dict(arrivals_count, orient='index')
        df.index = pd.to_datetime(df.index)
        wf = df.resample('W-MON', how='sum')

        return wf

    def cycle_time_histogram(self,
                             cycle,
                             types=None,
                             buckets=None):
        """
        Time taken for work to complete one or more 'cycles' - i.e. transitions from a start state to an end state
        """

        if self.all_issues is None:
            self.all_issues = self._issues_from_jira()

        rows = self._issues_as_rows(self.all_issues)

        cycle_time_data = {}

        for row in rows:

            if types is not None:

                include = False

                for type_grouping in types:

                    if row['type'] in self.types[type_grouping]:
                        include = True
                        key = "{0}-{1}".format(type_grouping, cycle)
                        break

                if not include:
                    continue

            else:
                key = cycle

            try:
                if row[cycle] is not None:
                    if key not in cycle_time_data:
                        cycle_time_data[key] = [row[cycle]]
                    else:
                        cycle_time_data[key].append(row[cycle])
            except KeyError:
                continue

        histogram = None

        for cycle in cycle_time_data:
            if buckets is not None:

                try:
                    li = buckets.index('max')
                    buckets[li] = max(cycle_time_data[cycle])

                except ValueError:
                    pass

                labels = bucket_labels(buckets)
                count, division = np.histogram(cycle_time_data[cycle], bins=buckets)
            else:

                count, division = np.histogram(cycle_time_data[cycle])
                labels = bucket_labels(division)

            cycle_histogram = pd.DataFrame(count, index=labels, columns=[cycle])
            cycle_histogram.index.name = 'bucket'

            if histogram is None:
                histogram = cycle_histogram.copy(deep=True)
            else:
                old_histogram = histogram.copy(deep=True)
                histogram = old_histogram.join(cycle_histogram, how='outer')

        return histogram

    def totals(self):
        """
        What are current totals of work in our various states
        """

        # We can get this by doing a count of the last day of the CFD

        cfd = self.cfd()

        return None

###############################################################################
# Internal methods
###############################################################################

    def _issues_from_jira(self, filter=None):
        """
        Get the actual issues from Jira itself via the Jira REST API
        """

        batch_size = 100
        issues = []

        for category in self.categories:

            n = 0
            while 1:

                jql = self.categories[category]
                if filter is not None:
                    jql = jql + filter

                issue_batch = self.jira.search_issues(jql,
                                                      startAt=n,
                                                      maxResults=batch_size,
                                                      expand='changelog')

                if issue_batch is None:
                    #TODO: Fix mocking so we can get rid of this.
                    # 'expand' seems to have some magic meaning in Mockito...
                    issue_batch = self.jira.search_issues(jql,
                                                          startAt=n,
                                                          maxResults=batch_size)

                for issue in issue_batch:

                    issue.category = category

                    created_date = datetime.strptime(issue.fields.created[:10], '%Y-%m-%d')
                    if issue.changelog is not None:
                        issue.history = history_from_jira_changelog(issue.changelog, created_date, self.until_date)

                    try:
                        for cycle in self.cycles:
                            reopened_state = None
                            after_state = None
                            start_state = None
                            exit_state = None
                            end_state = None
                            include_states = None
                            exclude_states = None

                            if 'ignore' in self.cycles[cycle]:
                                reopened_state = self.cycles[cycle]['ignore']

                            if 'after' in self.cycles[cycle]:
                                after_state = self.cycles[cycle]['after']

                            if 'start' in self.cycles[cycle]:
                                start_state = self.cycles[cycle]['start']

                            if 'exit' in self.cycles[cycle]:
                                exit_state = self.cycles[cycle]['exit']

                            if 'include' in self.cycles[cycle]:
                                include_states = self.cycles[cycle]['include']

                            if 'exclude' in self.cycles[cycle]:
                                exclude_states = self.cycles[cycle]['exclude']

                            if 'end' in self.cycles[cycle]:
                                end_state = self.cycles[cycle]['end']

                                setattr(issue,
                                        cycle,
                                        cycle_time(issue.history,
                                                   start_state=start_state,
                                                   after_state=after_state,
                                                   include_states=include_states,
                                                   exclude_states=exclude_states,
                                                   end_state=self.cycles[cycle]['end'],
                                                   reopened_state=reopened_state))

                            else:

                                setattr(issue,
                                        cycle,
                                        cycle_time(issue.history,
                                                   start_state=start_state,
                                                   after_state=after_state,
                                                   include_states=include_states,
                                                   exclude_states=exclude_states,
                                                   exit_state=exit_state,
                                                   reopened_state=reopened_state))

                    except AttributeError:

                        pass

                    issues.append(issue)

                if len(issue_batch) < batch_size:
                    break
                n += batch_size
                sys.stdout.write('.')
                sys.stdout.flush()

        return issues

    def _issues_as_rows(self, issues, types=None):

        """
        Get issues into a state where we can stick them into a Pandas dataframe
        """

        # TODO: Decide if this should be a class / helper method?
        # TODO: See if has any overlap with throughput function

        issue_rows = []

        for issue in issues:
            f = issue.fields

            resolution_date_str = f.resolutiondate

            if resolution_date_str is not None:
                resolution_date = datetime.strptime(resolution_date_str[:10],
                                                    '%Y-%m-%d')

                week = week_start_date(resolution_date.isocalendar()[0],
                                       resolution_date.isocalendar()[1]).strftime('%Y-%m-%d')

            else:

                week = None

            date_created = datetime.strptime(f.created[:10], '%Y-%m-%d')
            week_created = week_start_date(date_created.isocalendar()[0],
                                           date_created.isocalendar()[1]).strftime('%Y-%m-%d')

            if issue.changelog is not None:
                tis = time_in_states(issue.changelog.histories,
                                     datetime.strptime(f.created[:10],
                                                       '%Y-%m-%d'),
                                     date.today())

                since = tis[-1]['days']

            else:

                since = None

            include = True

            # TODO: This looks like a bit of a hack.
            # Can we do without iterating over loop, seeing as we
            # only ever want one swimlane/category combination?

            swimlane = issue.category

            if types is not None and self.types is not None:
                include = False
                for type_grouping in types:
                    if f.issuetype.name in self.types[type_grouping]:
                        swimlane = swimlane + '-' + type_grouping
                        include = True

            if include:

                try:
                    story_points = f.customfield_10002
                except AttributeError:
                    story_points = None

                try:
                    epic_link = f.customfield_10200
                except AttributeError:
                    epic_link = None

                # TODO: Fields need to come out of config too.

                issue_row = {'swimlane':     swimlane,
                             'type':         f.issuetype.name,
                             'id':           issue.key,
                             'name':         f.summary,
                             'status':       f.status.name,
                             'project':      f.project.name,
                             'components':   None,
                             'week':         week,
                             'since':        since,
                             'created':      f.created,
                             'week_created': week_created,
                             'story_points': story_points,
                             'epic_link':    epic_link,
                             'count':        1}

                for cycle in self.cycles:
                    try:
                        issue_row[cycle] = getattr(issue, cycle)
                    except AttributeError:
                        pass

                components = []

                for component in f.components:
                    components.append(component.name)

                issue_row['components'] = ",".join(components)

                issue_rows.append(issue_row)

        return issue_rows
<|endoftext|>


<commit_before># encoding=utf-8
## SOLVED 25/12/13
## 840

# If p is the perimeter of a right angle triangle with integral length sides,
# {a,b,c}, there are exactly three solutions for p = 120.

# {20,48,52}, {24,45,51}, {30,40,50}

# For which value of p ≤ 1000, is the number of solutions maximised?

from math import sqrt

def euler ():
    triangle_counts = [0 for i in range (1, 1000)]
    for a in range (1, 1000):
        for b in range (1, 1000):
            c = sqrt (a * a + b * b)
            if int (c) == c and a + b + c < 1000:
                triangle_counts [int (a + b + c)] += 1
    return triangle_counts.index (max (triangle_counts))
<commit_msg>using qualified imports in problem 39<commit_after># encoding=utf-8
## SOLVED 25/12/13
## 840

# If p is the perimeter of a right angle triangle with integral length sides,
# {a,b,c}, there are exactly three solutions for p = 120.

# {20,48,52}, {24,45,51}, {30,40,50}

# For which value of p ≤ 1000, is the number of solutions maximised?

import math

def euler ():
    triangle_counts = [0 for i in range (1, 1000)]
    for a in range (1, 1000):
        for b in range (1, 1000):
            c = math.sqrt (a * a + b * b)
            if int (c) == c and a + b + c < 1000:
                triangle_counts [int (a + b + c)] += 1
    return triangle_counts.index (max (triangle_counts))
<|endoftext|>


<commit_before># Config of faster rcnn
import sys, os
sys.path.append('..')
from ue4cv import *

# RCNN config
rcnn_path = '/home/qiuwch/workspace/py-faster-rcnn'
sys.path.append(os.path.join(rcnn_path, 'tools'))
import demo as D # Use demo.py provided in faster-rcnn
import numpy as np
import matplotlib.pyplot as plt
net = None

HOST, PORT = "localhost", 9000

def init_caffe(): # TODO: parse args into here
    global net
    prototxt = os.path.join(D.cfg.MODELS_DIR, D.NETS['vgg16'][0],
    'faster_rcnn_alt_opt', 'faster_rcnn_test.pt')
    caffemodel = os.path.join(D.cfg.DATA_DIR, 'faster_rcnn_models'
    , D.NETS['vgg16'][1])
    gpu_id = 0
    D.caffe.set_mode_gpu()
    D.caffe.set_device(gpu_id)
    D.cfg.GPU_ID = gpu_id
    D.cfg.TEST.HAS_RPN = True
    net = D.caffe.Net(prototxt, caffemodel, D.caffe.TEST)

    # Warmup on a dummy image
    im = 128 * np.ones((300, 500, 3), dtype = np.uint8)
    for i in xrange(2):
        _, _ = D.im_detect(net, im)

def plot_image(image, boxes, scores):
    ax.cla() # Clear axis
    ax.imshow(image, aspect='equal')

    CONF_THRESH = 0.8
    NMS_THRESH = 0.3
    for cls_ind, cls in enumerate(D.CLASSES[1:]):
        cls_ind += 1 # Skip background
        cls_boxes = boxes[:, 4*cls_ind:4*(cls_ind+1)]
        cls_scores = scores[:, cls_ind]
        dets = np.hstack((cls_boxes, cls_scores[:,np.newaxis])).astype(np.float32)
        keep = D.nms(dets, NMS_THRESH)
        dets = dets[keep, :]
        plot_bb(image, cls, dets, thresh=CONF_THRESH)

    fig.canvas.draw()

def plot_bb(im, class_name, dets, thresh=0.5):
    inds = np.where(dets[:, -1] >= thresh)[0] #
    if len(inds) == 0:
        return

    for i in inds:
        bbox = dets[i, :4]
        score = dets[i, -1]

        patch = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0]
        , bbox[3] - bbox[1], fill=False, edgecolor='red', linewidth=3.5)
        ax.add_patch(patch)
        text = '{:s} {:.3f}'.format(class_name, score)
        ax.text(bbox[0], bbox[1] - 2, text, bbox=dict(facecolor='blue', alpha=0.5)
        , fontsize=14, color='white')

def process_image(filename):
    print 'Process image: %s' % filename

    if not os.path.isfile(filename):
        print 'Image file %s not exist' % filename
        return

    image = D.cv2.imread(filename)
    timer = D.Timer()
    timer.tic()
    scores, boxes = D.im_detect(net, image)
    timer.toc()
    print ('Detection took {:.3f}s for '
    '{:d} object proposals').format(timer.total_time, boxes.shape[0])

    show_img = image[:,:, (2,1,0)] # Reorder to RGB
    plot_image(show_img, boxes, scores)

def message_handler(message):
    # This is a different thread
    filename = message
    if not net:
        init_caffe() # Caffe needs to be started in this thread, otherwise GIL will make it very slow

    print repr(message)
    lines = message.split('\n')
    if lines:
        process_image(lines[0])

if __name__ == '__main__':
    image = np.zeros((300, 300))

    client = Client((HOST, PORT), message_handler)

    # Initialize the matplotlib
    fig, ax = plt.subplots()
    ax.imshow(image)
    plt.axis('off')
    plt.tight_layout()
    plt.show()

    # client.wait() # Wait forever
<commit_msg>Update faster-rcnn integration.<commit_after># Config of faster rcnn
import sys, os
sys.path.append('..')
from ue4cv import *

# RCNN config
rcnn_path = '/home/qiuwch/workspace/py-faster-rcnn'
sys.path.append(os.path.join(rcnn_path, 'tools'))
import demo as D # Use demo.py provided in faster-rcnn
import numpy as np
import matplotlib.pyplot as plt
net = None

HOST, PORT = "localhost", 9000

def init_caffe(): # TODO: parse args into here
    global net
    prototxt = os.path.join(D.cfg.MODELS_DIR, D.NETS['vgg16'][0],
    'faster_rcnn_alt_opt', 'faster_rcnn_test.pt')
    caffemodel = os.path.join(D.cfg.DATA_DIR, 'faster_rcnn_models'
    , D.NETS['vgg16'][1])
    gpu_id = 0
    D.caffe.set_mode_gpu()
    D.caffe.set_device(gpu_id)
    D.cfg.GPU_ID = gpu_id
    D.cfg.TEST.HAS_RPN = True
    net = D.caffe.Net(prototxt, caffemodel, D.caffe.TEST)

    # Warmup on a dummy image
    im = 128 * np.ones((300, 500, 3), dtype = np.uint8)
    for i in xrange(2):
        _, _ = D.im_detect(net, im)

def plot_image(image, boxes, scores):
    ax.cla() # Clear axis
    ax.imshow(image, aspect='equal')

    CONF_THRESH = 0.8
    NMS_THRESH = 0.3
    for cls_ind, cls in enumerate(D.CLASSES[1:]):
        cls_ind += 1 # Skip background
        cls_boxes = boxes[:, 4*cls_ind:4*(cls_ind+1)]
        cls_scores = scores[:, cls_ind]
        dets = np.hstack((cls_boxes, cls_scores[:,np.newaxis])).astype(np.float32)
        keep = D.nms(dets, NMS_THRESH)
        dets = dets[keep, :]
        plot_bb(image, cls, dets, thresh=CONF_THRESH)

    fig.canvas.draw()

def plot_bb(im, class_name, dets, thresh=0.5):
    inds = np.where(dets[:, -1] >= thresh)[0] #
    if len(inds) == 0:
        return

    for i in inds:
        bbox = dets[i, :4]
        score = dets[i, -1]

        patch = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0]
        , bbox[3] - bbox[1], fill=False, edgecolor='red', linewidth=3.5)
        ax.add_patch(patch)
        text = '{:s} {:.3f}'.format(class_name, score)
        ax.text(bbox[0], bbox[1] - 2, text, bbox=dict(facecolor='blue', alpha=0.5)
        , fontsize=14, color='white')

def process_image(filename):
    print 'Process image: %s' % filename

    if not os.path.isfile(filename):
        print 'Image file %s not exist' % filename
        return

    image = D.cv2.imread(filename)
    timer = D.Timer()
    timer.tic()
    scores, boxes = D.im_detect(net, image)
    timer.toc()
    print ('Detection took {:.3f}s for '
    '{:d} object proposals').format(timer.total_time, boxes.shape[0])

    show_img = image[:,:, (2,1,0)] # Reorder to RGB
    plot_image(show_img, boxes, scores)

# def message_handler(message):
#     # This is a different thread
#     filename = message
#     if not net:
#         init_caffe() # Caffe needs to be started in this thread, otherwise GIL will make it very slow
#
#     print repr(message)
#     lines = message.split('\n')
#     if lines:
#         process_image(lines[0])

def onclick(event):
    image = ue4cv.client.request('vget /camera/0/lit')
    process_image(image)
    pass

if __name__ == '__main__':
    image = np.zeros((300, 300))

    # client = Client((HOST, PORT), message_handler)
    ue4cv.client.connect()

    # Initialize the matplotlib
    fig, ax = plt.subplots()
    fig.canvas.mpl_connect('button_press_event', onclick)

    ax.imshow(image)
    plt.axis('off')
    plt.tight_layout()
    plt.show()

    # client.wait() # Wait forever
<|endoftext|>


<commit_before>"""Utilities for handling DWI images."""

from time import time
import os
import numpy as np
import scipy as sp

import asciifile
import fit
import util

def load(filename, nrois=1, varname='ROIdata'):
    """Load images from a file."""
    root, ext = os.path.splitext(filename)
    if ext == '.mat':
        return load_matlab(filename, varname)
    else:
        return load_ascii(filename, nrois)

def load_matlab(filename, varname='ROIdata'):
    """Load images from a MATLAB file."""
    mat = sp.io.loadmat(filename, struct_as_record=False)
    r = []
    for window in mat[varname][0]:
        win = window[0,0]
        sis = win.SIs.T
        bset = win.bset[0]
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = '-' # Not implemented.
        dwi.name = '-' # Not implemented.
        dwi.number = int(win.number[0,0])
        try:
            dwi.subwindow = map(int, win.subwindow[0])
        except:
            dwi.subwindow = util.fabricate_subwindow(len(sis))
        r.append(dwi)
    return r

def load_ascii(filename, nrois=1):
    """Load images from an ASCII file."""
    af = asciifile.AsciiFile(filename)
    a = af.a.reshape(nrois,-1,af.a.shape[-1])
    r = []
    for i in range(nrois):
        sis = a[i]
        bset = af.bset()
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = af.roislice()
        dwi.name = af.name()
        dwi.number = af.number + i
        dwi.subwindow = af.subwindow()
        r.append(dwi)
    return r

class DWImage(object):
    """DWI image, stored single-dimensionally."""

    def __init__(self, sis, bset):
        """Create a new DWI image.

        Parameters
        ----------
        sis : array_like, shape = (n_voxels, n_bvalues)
            Voxels representing signal intensities.
        bset : sequence
            Different b-values.
        """
        self.sis = np.array(sis, dtype=float)
        self.bset = np.array(sorted(set(bset)), dtype=float)
        self.execution_time = -1
        if self.sis.shape != (len(self.sis), len(self.bset)):
            raise Exception('Image size does not match with b-values.')

    def subwindow_shape(self):
        return tuple((b-a for a, b in util.chunks(self.subwindow, 2)))

    def size(self):
        """Return number of voxels."""
        return len(self.sis)

    def __repr__(self):
        return '%s:%i' % (self.filename, self.number)

    def __str__(self):
        d = dict(fn=self.filename, n=self.number,
                nb=len(self.bset), b=list(self.bset),
                s=self.size(), w=self.subwindow,
                ws=self.subwindow_shape())
        s = 'File: {fn}\n'\
                'Number: {n}\n'\
                'B-values: {nb}: {b}\n'\
                'Window: {s}, {w}, {ws}'.format(**d)
        return s

    def fit_elem(self, model, elem, bvalues=[], mean=False):
        """Curve fitting for an image element."""
        if not bvalues:
            bvalues = range(len(self.bset))
        xdata = self.bset[bvalues]
        if mean:
            ydata = self.sis.mean(axis=0)[bvalues]
        else:
            ydata = self.sis[elem][bvalues]
        return model.fit_mi(xdata, ydata)

    def fit_whole(self, model, bvalues=[], log=None, mean=False):
        """Curve fitting for the whole image."""
        start = time()
        size = 1 if mean else self.size()
        pmap = np.zeros((size, len(model.params) + 1))
        cnt_errors = 0
        cnt_warnings = 0
        if log:
            log('Fitting %i elements to %s...\n' % (size, model))
        for i in range(size):
            if log:
                log('\r%i...' % i)
            params, err = self.fit_elem(model, i, bvalues, mean)
            pmap[i, -1] = err
            if np.isfinite(err):
                pmap[i, :-1] = params
            else:
                pmap[i, :-1].fill(np.nan)
                cnt_errors += 1
        if cnt_errors:
            map(util.impute, pmap.T)
        self.execution_time = time() - start
        if log:
            log('\nFinished with %i errors, %i warnings.\n'\
                    % (cnt_errors, cnt_warnings))
        return pmap
<commit_msg>Moved functions around a bit<commit_after>"""Utilities for handling DWI images."""

from time import time
import os
import numpy as np
import scipy as sp

import asciifile
import fit
import util

def load(filename, nrois=1, varname='ROIdata'):
    """Load images from a file."""
    root, ext = os.path.splitext(filename)
    if ext == '.mat':
        return load_matlab(filename, varname)
    else:
        return load_ascii(filename, nrois)

def load_matlab(filename, varname='ROIdata'):
    """Load images from a MATLAB file."""
    mat = sp.io.loadmat(filename, struct_as_record=False)
    r = []
    for window in mat[varname][0]:
        win = window[0,0]
        sis = win.SIs.T
        bset = win.bset[0]
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = '-' # Not implemented.
        dwi.name = '-' # Not implemented.
        dwi.number = int(win.number[0,0])
        try:
            dwi.subwindow = map(int, win.subwindow[0])
        except:
            dwi.subwindow = util.fabricate_subwindow(len(sis))
        r.append(dwi)
    return r

def load_ascii(filename, nrois=1):
    """Load images from an ASCII file."""
    af = asciifile.AsciiFile(filename)
    a = af.a.reshape(nrois,-1,af.a.shape[-1])
    r = []
    for i in range(nrois):
        sis = a[i]
        bset = af.bset()
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = af.roislice()
        dwi.name = af.name()
        dwi.number = af.number + i
        dwi.subwindow = af.subwindow()
        r.append(dwi)
    return r

class DWImage(object):
    """DWI image, stored single-dimensionally."""

    def __init__(self, sis, bset):
        """Create a new DWI image.

        Parameters
        ----------
        sis : array_like, shape = (n_voxels, n_bvalues)
            Voxels representing signal intensities.
        bset : sequence
            Different b-values.
        """
        self.sis = np.array(sis, dtype=float)
        self.bset = np.array(sorted(set(bset)), dtype=float)
        self.execution_time = -1
        if self.sis.shape != (len(self.sis), len(self.bset)):
            raise Exception('Image size does not match with b-values.')

    def __repr__(self):
        return '%s:%i' % (self.filename, self.number)

    def __str__(self):
        d = dict(fn=self.filename, n=self.number,
                nb=len(self.bset), b=list(self.bset),
                s=self.size(), w=self.subwindow,
                ws=self.subwindow_shape())
        s = 'File: {fn}\n'\
                'Number: {n}\n'\
                'B-values: {nb}: {b}\n'\
                'Window: {s}, {w}, {ws}'.format(**d)
        return s

    def subwindow_shape(self):
        return tuple((b-a for a, b in util.chunks(self.subwindow, 2)))

    def size(self):
        """Return number of voxels."""
        return len(self.sis)

    def fit_elem(self, model, elem, bvalues=[], mean=False):
        """Curve fitting for an image element."""
        if not bvalues:
            bvalues = range(len(self.bset))
        xdata = self.bset[bvalues]
        if mean:
            ydata = self.sis.mean(axis=0)[bvalues]
        else:
            ydata = self.sis[elem][bvalues]
        return model.fit_mi(xdata, ydata)

    def fit_whole(self, model, bvalues=[], log=None, mean=False):
        """Curve fitting for the whole image."""
        start = time()
        size = 1 if mean else self.size()
        pmap = np.zeros((size, len(model.params) + 1))
        cnt_errors = 0
        cnt_warnings = 0
        if log:
            log('Fitting %i elements to %s...\n' % (size, model))
        for i in range(size):
            if log:
                log('\r%i...' % i)
            params, err = self.fit_elem(model, i, bvalues, mean)
            pmap[i, -1] = err
            if np.isfinite(err):
                pmap[i, :-1] = params
            else:
                pmap[i, :-1].fill(np.nan)
                cnt_errors += 1
        if cnt_errors:
            map(util.impute, pmap.T)
        self.execution_time = time() - start
        if log:
            log('\nFinished with %i errors, %i warnings.\n'\
                    % (cnt_errors, cnt_warnings))
        return pmap
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
This is a script for testing pull requests for NetworkX. It merges the pull
request with current master, installs and tests on all available versions of
Python, and posts the results to Gist if any tests fail.

This script is heavily based on IPython's test_pr.py and friends. See:

http://github.com/ipython/ipython/tree/master/tools

Usage:
    python test_pr.py 742
"""
from __future__ import print_function

import errno
from glob import glob
import io
import json
import os
import pickle
import re
import requests
import shutil
import time
from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
import sys

import gh_api
from gh_api import Obj

basedir = os.path.join(os.path.expanduser("~"), ".nx_pr_tests")
repodir = os.path.join(basedir, "networkx")
nx_repository = 'git://github.com/networkx/networkx.git'
nx_http_repository = 'http://github.com/networkx/networkx.git'
gh_project="networkx/networkx"

# TODO Add PyPy support
supported_pythons = ['python2.6', 'python2.7', 'python3.2']

# Report missing libraries during tests and number of skipped
# and passed tests.
missing_libs_re = re.compile('SKIP: (\w+) not available')
def get_missing_libraries(log):
    libs = set()
    for line in log.split('\n'):
        m = missing_libs_re.search(line)
        if m:
            libs.add(m.group(1).lower())
    if libs:
        return ", ".join(libs)

skipped_re = re.compile('SKIP=(\d+)')
def get_skipped(log):
    m = skipped_re.search(log)
    if m:
        return m.group(1)

number_tests_re = re.compile('Ran (\d+) tests in')
def get_number_tests(log):
    m = number_tests_re.search(log)
    if m:
        return m.group(1)

class TestRun(object):
    def __init__(self, pr_num):
        self.unavailable_pythons = []
        self.venvs = []
        self.pr_num = pr_num

        self.pr = gh_api.get_pull_request(gh_project, pr_num)

        self.setup()

        self.results = []

    def available_python_versions(self):
        """Get the executable names of available versions of Python on the system.
        """
        for py in supported_pythons:
            try:
                check_call([py, '-c', 'import nose'], stdout=PIPE)
                yield py
            except (OSError, CalledProcessError):
                self.unavailable_pythons.append(py)

    def setup(self):
        """Prepare the repository and virtualenvs."""
        try:
            os.mkdir(basedir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        os.chdir(basedir)

        # Delete virtualenvs and recreate
        for venv in glob('venv-*'):
            shutil.rmtree(venv)
        for py in self.available_python_versions():
            check_call(['virtualenv', '-p', py,
                            '--system-site-packages', 'venv-%s' % py])

            self.venvs.append((py, 'venv-%s' % py))

        # Check out and update the repository
        if not os.path.exists('networkx'):
            try :
                check_call(['git', 'clone', nx_repository])
            except CalledProcessError:
                check_call(['git', 'clone', nx_http_repository])
        os.chdir(repodir)
        check_call(['git', 'checkout', 'master'])
        try :
            check_call(['git', 'pull', 'origin', 'master'])
        except CalledProcessError:
            check_call(['git', 'pull', nx_http_repository, 'master'])
        self.master_sha = check_output(['git', 'log', '-1',
                                        '--format=%h']).decode('ascii').strip()
        os.chdir(basedir)

    def get_branch(self):
        repo = self.pr['head']['repo']['clone_url']
        branch = self.pr['head']['ref']
        owner = self.pr['head']['repo']['owner']['login']
        mergeable = self.pr['mergeable']

        os.chdir(repodir)
        if mergeable:
            merged_branch = "%s-%s" % (owner, branch)
            # Delete the branch first
            call(['git', 'branch', '-D', merged_branch])
            check_call(['git', 'checkout', '-b', merged_branch])
            check_call(['git', 'pull', '--no-ff', '--no-commit', repo, branch])
            check_call(['git', 'commit', '-m', "merge %s/%s" % (repo, branch)])
        else:
            # Fetch the branch without merging it.
            check_call(['git', 'fetch', repo, branch])
            check_call(['git', 'checkout', 'FETCH_HEAD'])
        os.chdir(basedir)

    def markdown_format(self):
        def format_result(result):
            s = "* %s: " % result.py
            if result.passed:
                s += "%s OK (SKIP=%s) Ran %s tests" % \
                                        (ok, result.skipped, result.num_tests)
            else:
                s += "%s Failed, log at %s" % (fail, result.log_url)
            if result.missing_libraries:
                s += " (libraries not available: " + result.missing_libraries + ")"
            return s
        pr_num = self.pr_num
        branch = self.pr['head']['ref']
        branch_url = self.pr['head']['repo']['html_url'] + '/tree/' + branch
        owner = self.pr['head']['repo']['owner']['login']
        mergeable = self.pr['mergeable']
        master_sha = self.master_sha
        branch_sha = self.pr['head']['sha'][:7]
        ok = ':eight_spoked_asterisk:'
        fail = ':red_circle:'

        header = "**NetworkX: Test results for pull request #%s " % pr_num
        header += "([%s '%s' branch](%s))**" % (owner, branch, branch_url)
        if mergeable:
            mrg = "%s This pull request can be merged cleanly " % ok
            mrg += "(commit %s into NetworkX master %s)" % (branch_sha, master_sha)
        else:
            mrg = "%s This pull request **cannot** be merged cleanly " % fail
            mrg += "(commit %s into NetworkX master %s)" % (branch_sha, master_sha)
            mrg += ". Please rebase from upstream/master"
        lines = [header,
                 mrg,
                 "Platform: " + sys.platform,
                 ""] + \
                [format_result(r) for r in self.results]
        if self.unavailable_pythons:
            lines += ["",
                        "Not available for testing: "  \
                            + ", ".join(self.unavailable_pythons)]
        return "\n".join(lines)

    def post_results_comment(self):
        body = self.markdown_format()
        gh_api.post_issue_comment(gh_project, self.pr_num, body)

    def print_results(self):
        pr_num = self.pr_num
        branch = self.pr['head']['ref']
        branch_url = self.pr['head']['repo']['html_url'] + '/tree/' + branch
        owner = self.pr['head']['repo']['owner']['login']
        mergeable = self.pr['mergeable']
        master_sha = self.master_sha
        branch_sha = self.pr['head']['sha'][:7]

        print("\n")
        print("**NetworkX: Test results for pull request %s " % pr_num,
              "(%s '%s' branch at %s)**" % (owner, branch, branch_url))
        if mergeable:
            print("This pull request can be merged cleanly",
                  "(commit %s into NetworkX master %s)" % (branch_sha, master_sha))
        else:
            print("This pull request **cannot** be merged cleanly",
                  "(commit %s into NetworkX master %s)." % (branch_sha, master_sha),
                  "Please rebase from upstream/master")
        print("Platform:", sys.platform)
        for result in self.results:
            if result.passed:
                print(result.py, ":", "OK (SKIP=%s) Ran %s tests" % \
                                        (result.skipped, result.num_tests))
            else:
                print(result.py, ":", "Failed")
                print("    Test log:", result.get('log_url') or result.log_file)
            if result.missing_libraries:
                print("    Libraries not available:", result.missing_libraries)
        if self.unavailable_pythons:
            print("Not available for testing:",
                    ", ".join(self.unavailable_pythons))

    def dump_results(self):
        with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
            pickle.dump(self, f)

    @staticmethod
    def load_results():
        with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
            return pickle.load(f)

    def save_logs(self):
        for result in self.results:
            if not result.passed:
                result_locn = os.path.abspath(os.path.join('venv-%s' % result.py,
                                            self.pr['head']['sha'][:7]+".log"))
                with io.open(result_locn, 'w', encoding='utf-8') as f:
                    f.write(result.log)

                result.log_file = result_locn

    def post_logs(self):
        for result in self.results:
            if not result.passed:
                result.log_url = gh_api.post_gist(result.log,
                                                description='NetworkX test log',
                                                filename="results.log", auth=True)

    def run(self):
        for py, venv in self.venvs:
            tic = time.time()
            passed, log = run_tests(venv)
            elapsed = int(time.time() - tic)
            print("Ran tests with %s in %is" % (py, elapsed))
            missing_libraries = get_missing_libraries(log)
            skipped = get_skipped(log)
            num_tests = get_number_tests(log)

            self.results.append(Obj(py=py,
                                    passed=passed,
                                    log=log,
                                    missing_libraries=missing_libraries,
                                    skipped=skipped,
                                    num_tests=num_tests
                                  )
                               )

def run_tests(venv):
    version = venv.split('-')[1]
    py = os.path.join(basedir, venv, 'bin', 'python')
    os.chdir(repodir)
    # cleanup build-dir
    if os.path.exists('build'):
        shutil.rmtree('build')
    #tic = time.time()
    print ("\nInstalling NetworkX with %s" % py)
    logfile = os.path.join(basedir, venv, 'install.log')
    print ("Install log at %s" % logfile)
    with open(logfile, 'wb') as f:
        check_call([py, 'setup.py', 'install'], stderr=STDOUT, stdout=f)
    #toc = time.time()
    #print ("Installed NetworkX in %.1fs" % (toc-tic))
    os.chdir(basedir)

    # Remove PYTHONPATH if present
    os.environ.pop("PYTHONPATH", None)

    # check that the right NetworkX is imported. Also catch exception if
    # the pull request breaks "import networkx as nx"
    try:
        cmd_file = [py, '-c', 'import networkx as nx; print(nx.__file__)']
        nx_file = check_output(cmd_file, stderr=STDOUT)
    except CalledProcessError as e:
        return False, e.output.decode('utf-8')

    nx_file = nx_file.strip().decode('utf-8')
    if not nx_file.startswith(os.path.join(basedir, venv)):
        msg = u"NetworkX does not appear to be in the venv: %s" % nx_file
        msg += u"\nDo you use setupegg.py develop?"
        print(msg, file=sys.stderr)
        return False, msg

    # Run tests: this is different than in ipython's test_pr, they use
    # a script for running their tests. It gets installed at
    # os.path.join(basedir, venv, 'bin', 'iptest')
    print("\nRunning tests with %s ..." % version)
    cmd = [py, '-c', 'import networkx as nx; nx.test(verbosity=2,doctest=True)']
    try:
        return True, check_output(cmd, stderr=STDOUT).decode('utf-8')
    except CalledProcessError as e:
        return False, e.output.decode('utf-8')

def test_pr(num, post_results=True):
    # Get Github authorisation first, so that the user is prompted straight away
    # if their login is needed.
    if post_results:
        gh_api.get_auth_token()

    testrun = TestRun(num)

    testrun.get_branch()

    testrun.run()

    testrun.dump_results()

    testrun.save_logs()
    testrun.print_results()

    if post_results:
        results_urls = testrun.post_logs()
        testrun.post_results_comment()
        print("(Posted to Github)")
    else:
        post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
        print("To post the results to Github, run", post_script)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description="Test a pull request for NetworkX")
    parser.add_argument('-p', '--publish', action='store_true',
                        help="Publish the results to Github")
    parser.add_argument('number', type=int, help="The pull request number")

    args = parser.parse_args()

    # Test for requests version.
    import requests
    major, minor, rev = map(int, requests.__version__.split('.'))
    if minor < 10:
        print("test_pr.py:")
        print("The requests python library must be version 0.10.0",
                "or above, you have version",
                "{0}.{1}.{2} installed".format(major, minor, rev))
        print()
        sys.exit(1)

    test_pr(args.number, post_results=args.publish)
<commit_msg>Remove rebase suggestion when clean merge fails<commit_after>#!/usr/bin/env python
"""
This is a script for testing pull requests for NetworkX. It merges the pull
request with current master, installs and tests on all available versions of
Python, and posts the results to Gist if any tests fail.

This script is heavily based on IPython's test_pr.py and friends. See:

http://github.com/ipython/ipython/tree/master/tools

Usage:
    python test_pr.py 742
"""
from __future__ import print_function

import errno
from glob import glob
import io
import json
import os
import pickle
import re
import requests
import shutil
import time
from subprocess import call, check_call, check_output, PIPE, STDOUT, CalledProcessError
import sys

import gh_api
from gh_api import Obj

basedir = os.path.join(os.path.expanduser("~"), ".nx_pr_tests")
repodir = os.path.join(basedir, "networkx")
nx_repository = 'git://github.com/networkx/networkx.git'
nx_http_repository = 'http://github.com/networkx/networkx.git'
gh_project="networkx/networkx"

# TODO Add PyPy support
supported_pythons = ['python2.6', 'python2.7', 'python3.2']

# Report missing libraries during tests and number of skipped
# and passed tests.
missing_libs_re = re.compile('SKIP: (\w+) not available')
def get_missing_libraries(log):
    libs = set()
    for line in log.split('\n'):
        m = missing_libs_re.search(line)
        if m:
            libs.add(m.group(1).lower())
    if libs:
        return ", ".join(libs)

skipped_re = re.compile('SKIP=(\d+)')
def get_skipped(log):
    m = skipped_re.search(log)
    if m:
        return m.group(1)

number_tests_re = re.compile('Ran (\d+) tests in')
def get_number_tests(log):
    m = number_tests_re.search(log)
    if m:
        return m.group(1)

class TestRun(object):
    def __init__(self, pr_num):
        self.unavailable_pythons = []
        self.venvs = []
        self.pr_num = pr_num

        self.pr = gh_api.get_pull_request(gh_project, pr_num)

        self.setup()

        self.results = []

    def available_python_versions(self):
        """Get the executable names of available versions of Python on the system.
        """
        for py in supported_pythons:
            try:
                check_call([py, '-c', 'import nose'], stdout=PIPE)
                yield py
            except (OSError, CalledProcessError):
                self.unavailable_pythons.append(py)

    def setup(self):
        """Prepare the repository and virtualenvs."""
        try:
            os.mkdir(basedir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        os.chdir(basedir)

        # Delete virtualenvs and recreate
        for venv in glob('venv-*'):
            shutil.rmtree(venv)
        for py in self.available_python_versions():
            check_call(['virtualenv', '-p', py,
                            '--system-site-packages', 'venv-%s' % py])

            self.venvs.append((py, 'venv-%s' % py))

        # Check out and update the repository
        if not os.path.exists('networkx'):
            try :
                check_call(['git', 'clone', nx_repository])
            except CalledProcessError:
                check_call(['git', 'clone', nx_http_repository])
        os.chdir(repodir)
        check_call(['git', 'checkout', 'master'])
        try :
            check_call(['git', 'pull', 'origin', 'master'])
        except CalledProcessError:
            check_call(['git', 'pull', nx_http_repository, 'master'])
        self.master_sha = check_output(['git', 'log', '-1',
                                        '--format=%h']).decode('ascii').strip()
        os.chdir(basedir)

    def get_branch(self):
        repo = self.pr['head']['repo']['clone_url']
        branch = self.pr['head']['ref']
        owner = self.pr['head']['repo']['owner']['login']
        mergeable = self.pr['mergeable']

        os.chdir(repodir)
        if mergeable:
            merged_branch = "%s-%s" % (owner, branch)
            # Delete the branch first
            call(['git', 'branch', '-D', merged_branch])
            check_call(['git', 'checkout', '-b', merged_branch])
            check_call(['git', 'pull', '--no-ff', '--no-commit', repo, branch])
            check_call(['git', 'commit', '-m', "merge %s/%s" % (repo, branch)])
        else:
            # Fetch the branch without merging it.
            check_call(['git', 'fetch', repo, branch])
            check_call(['git', 'checkout', 'FETCH_HEAD'])
        os.chdir(basedir)

    def markdown_format(self):
        def format_result(result):
            s = "* %s: " % result.py
            if result.passed:
                s += "%s OK (SKIP=%s) Ran %s tests" % \
                                        (ok, result.skipped, result.num_tests)
            else:
                s += "%s Failed, log at %s" % (fail, result.log_url)
            if result.missing_libraries:
                s += " (libraries not available: " + result.missing_libraries + ")"
            return s
        pr_num = self.pr_num
        branch = self.pr['head']['ref']
        branch_url = self.pr['head']['repo']['html_url'] + '/tree/' + branch
        owner = self.pr['head']['repo']['owner']['login']
        mergeable = self.pr['mergeable']
        master_sha = self.master_sha
        branch_sha = self.pr['head']['sha'][:7]
        ok = ':eight_spoked_asterisk:'
        fail = ':red_circle:'

        header = "**NetworkX: Test results for pull request #%s " % pr_num
        header += "([%s '%s' branch](%s))**" % (owner, branch, branch_url)
        if mergeable:
            mrg = "%s This pull request can be merged cleanly " % ok
        else:
            mrg = "%s This pull request **cannot** be merged cleanly " % fail
        mrg += "(commit %s into NetworkX master %s)" % (branch_sha, master_sha)
        lines = [header,
                 mrg,
                 "Platform: " + sys.platform,
                 ""] + \
                [format_result(r) for r in self.results]
        if self.unavailable_pythons:
            lines += ["",
                        "Not available for testing: "  \
                            + ", ".join(self.unavailable_pythons)]
        return "\n".join(lines)

    def post_results_comment(self):
        body = self.markdown_format()
        gh_api.post_issue_comment(gh_project, self.pr_num, body)

    def print_results(self):
        pr_num = self.pr_num
        branch = self.pr['head']['ref']
        branch_url = self.pr['head']['repo']['html_url'] + '/tree/' + branch
        owner = self.pr['head']['repo']['owner']['login']
        mergeable = self.pr['mergeable']
        master_sha = self.master_sha
        branch_sha = self.pr['head']['sha'][:7]

        print("\n")
        print("**NetworkX: Test results for pull request %s " % pr_num,
              "(%s '%s' branch at %s)**" % (owner, branch, branch_url))
        if mergeable:
            mrg = "OK: This pull request can be merged cleanly "
        else:
            mrg = "FAIL: This pull request **cannot** be merged cleanly "
        mrg += "(commit %s into NetworkX master %s)" % (branch_sha, master_sha)
        print(mrg)
        print("Platform:", sys.platform)
        for result in self.results:
            if result.passed:
                print(result.py, ":", "OK (SKIP=%s) Ran %s tests" % \
                                        (result.skipped, result.num_tests))
            else:
                print(result.py, ":", "Failed")
                print("    Test log:", result.get('log_url') or result.log_file)
            if result.missing_libraries:
                print("    Libraries not available:", result.missing_libraries)
        if self.unavailable_pythons:
            print("Not available for testing:",
                    ", ".join(self.unavailable_pythons))

    def dump_results(self):
        with open(os.path.join(basedir, 'lastresults.pkl'), 'wb') as f:
            pickle.dump(self, f)

    @staticmethod
    def load_results():
        with open(os.path.join(basedir, 'lastresults.pkl'), 'rb') as f:
            return pickle.load(f)

    def save_logs(self):
        for result in self.results:
            if not result.passed:
                result_locn = os.path.abspath(os.path.join('venv-%s' % result.py,
                                            self.pr['head']['sha'][:7]+".log"))
                with io.open(result_locn, 'w', encoding='utf-8') as f:
                    f.write(result.log)

                result.log_file = result_locn

    def post_logs(self):
        for result in self.results:
            if not result.passed:
                result.log_url = gh_api.post_gist(result.log,
                                                description='NetworkX test log',
                                                filename="results.log", auth=True)

    def run(self):
        for py, venv in self.venvs:
            tic = time.time()
            passed, log = run_tests(venv)
            elapsed = int(time.time() - tic)
            print("Ran tests with %s in %is" % (py, elapsed))
            missing_libraries = get_missing_libraries(log)
            skipped = get_skipped(log)
            num_tests = get_number_tests(log)

            self.results.append(Obj(py=py,
                                    passed=passed,
                                    log=log,
                                    missing_libraries=missing_libraries,
                                    skipped=skipped,
                                    num_tests=num_tests
                                  )
                               )

def run_tests(venv):
    version = venv.split('-')[1]
    py = os.path.join(basedir, venv, 'bin', 'python')
    os.chdir(repodir)
    # cleanup build-dir
    if os.path.exists('build'):
        shutil.rmtree('build')
    #tic = time.time()
    print ("\nInstalling NetworkX with %s" % py)
    logfile = os.path.join(basedir, venv, 'install.log')
    print ("Install log at %s" % logfile)
    with open(logfile, 'wb') as f:
        check_call([py, 'setup.py', 'install'], stderr=STDOUT, stdout=f)
    #toc = time.time()
    #print ("Installed NetworkX in %.1fs" % (toc-tic))
    os.chdir(basedir)

    # Remove PYTHONPATH if present
    os.environ.pop("PYTHONPATH", None)

    # check that the right NetworkX is imported. Also catch exception if
    # the pull request breaks "import networkx as nx"
    try:
        cmd_file = [py, '-c', 'import networkx as nx; print(nx.__file__)']
        nx_file = check_output(cmd_file, stderr=STDOUT)
    except CalledProcessError as e:
        return False, e.output.decode('utf-8')

    nx_file = nx_file.strip().decode('utf-8')
    if not nx_file.startswith(os.path.join(basedir, venv)):
        msg = u"NetworkX does not appear to be in the venv: %s" % nx_file
        msg += u"\nDo you use setupegg.py develop?"
        print(msg, file=sys.stderr)
        return False, msg

    # Run tests: this is different than in ipython's test_pr, they use
    # a script for running their tests. It gets installed at
    # os.path.join(basedir, venv, 'bin', 'iptest')
    print("\nRunning tests with %s ..." % version)
    cmd = [py, '-c', 'import networkx as nx; nx.test(verbosity=2,doctest=True)']
    try:
        return True, check_output(cmd, stderr=STDOUT).decode('utf-8')
    except CalledProcessError as e:
        return False, e.output.decode('utf-8')

def test_pr(num, post_results=True):
    # Get Github authorisation first, so that the user is prompted straight away
    # if their login is needed.
    if post_results:
        gh_api.get_auth_token()

    testrun = TestRun(num)

    testrun.get_branch()

    testrun.run()

    testrun.dump_results()

    testrun.save_logs()
    testrun.print_results()

    if post_results:
        results_urls = testrun.post_logs()
        testrun.post_results_comment()
        print("(Posted to Github)")
    else:
        post_script = os.path.join(os.path.dirname(sys.argv[0]), "post_pr_test.py")
        print("To post the results to Github, run", post_script)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description="Test a pull request for NetworkX")
    parser.add_argument('-p', '--publish', action='store_true',
                        help="Publish the results to Github")
    parser.add_argument('number', type=int, help="The pull request number")

    args = parser.parse_args()

    # Test for requests version.
    import requests
    major, minor, rev = map(int, requests.__version__.split('.'))
    if minor < 10:
        print("test_pr.py:")
        print("The requests python library must be version 0.10.0",
                "or above, you have version",
                "{0}.{1}.{2} installed".format(major, minor, rev))
        print()
        sys.exit(1)

    test_pr(args.number, post_results=args.publish)
<|endoftext|>