<commit_before>"""Utilities for managing the fiscal calendar."""

from __future__ import division

import calendar
import datetime

# The first month at the start of a new fiscal year.
# By default, use the U.S. federal government's fiscal year,
# which starts on October 1st and ends on September 30th.
START_DAY = 1
START_MONTH = 10

# Number of months in each quarter
MONTHS_PER_QUARTER = 12 // 4

MIN_QUARTER = 1
MAX_QUARTER = 4

def _check_int(value):
    """Check if value is an int or int-like string.

    :param value: The value to test
    :return: The value
    :rtype: int
    :raises TypeError: If value is not an int or int-like string
    """
    if isinstance(value, int):
        return value
    elif isinstance(value, str) and value.is_digit():
        return int(value)
    else:
        raise TypeError('an int or int-like string is required (got %s)' % (
            type(value).__name__))

def _check_year(year):
    """Check if year is a valid year.

    :param year: The year to test
    :return: The year
    :rtype: int
    :raises ValueError: If the year is out of range
    """
    year = _check_int(year)

    if datetime.MINYEAR <= year <= datetime.MAXYEAR:
        return year
    else:
        raise ValueError('year must be in %d..%d' % (
            datetime.MINYEAR, datetime.MAXYEAR), year)

def _check_quarter(quarter):
    """Check if quarter is a valid quarter.

    :param quarter: The quarter to test
    :return: The quarter
    :rtype: int
    :raises ValueError: If the quarter is out of range
    """
    quarter = _check_int(quarter)

    if MIN_QUARTER <= quarter <= MAX_QUARTER:
        return quarter
    else:
        raise ValueError('quarter must be in %d..%d' % (
            MIN_QUARTER, MAX_QUARTER), quarter)

class FiscalYear(object):
    """A class representing a single fiscal year."""

    __slots__ = '_fiscal_year'

    def __new__(cls, fiscal_year):
        """Constructor.

        :param fiscal_year: The fiscal year
        :type fiscal_year: int or str
        :returns: A newly constructed FiscalYear object
        :rtype: FiscalYear
        :raises TypeError: If value is not an int or int-like string
        :raises ValueError: If the year is out of range
        """
        fiscal_year = _check_year(fiscal_year)

        self = super(FiscalYear, cls).__new__(cls)
        self._fiscal_year = fiscal_year
        return self

    def __repr__(self):
        """Convert to formal string, for repr().

        >>> fy = FiscalYear(2017)
        >>> repr(fy)
        'fiscalyear.FiscalYear(2017)'
        """
        return "%s.%s(%d)" % (self.__class__.__module__,
                              self.__class__.__name__,
                              self._fiscal_year)

    def isoformat(self):
        """Return the date range formatted according to ISO.

        This is 'YYYY-MM-DD'.

        References:
        - http://www.w3.org/TR/NOTE-datetime
        - http://www.cl.cam.ac.uk/~mgk25/iso-time.html
        """
        pass

    __str__ = isoformat

    # TODO: Implement __format__ so that you can print
    # fiscal year as 17 or 2017 (%y or %Y)

    # Read-only field accessors
    @property
    def fiscal_year(self):
        """Fiscal year."""
        return self._fiscal_year

    @property
    def start(self):
        """Start of the fiscal year.

        :rtype: datetime.datetime
        """
        return self.q1.start

    @property
    def end(self):
        """End of the fiscal year.

        :rtype: datetime.datetime
        """
        return self.q4.end

    @property
    def q1(self):
        """The first quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 1)

    @property
    def q2(self):
        """The second quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 2)

    @property
    def q3(self):
        """The third quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 3)

    @property
    def q4(self):
        """The fourth quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 4)

    # Comparisons of FiscalYear objects with other

    def __lt__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year < other._fiscal_year
        return NotImplemented

    def __le__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year <= other._fiscal_year
        return NotImplemented

    def __eq__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year == other._fiscal_year
        return NotImplemented

    def __ne__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year != other._fiscal_year
        return NotImplemented

    def __gt__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year > other._fiscal_year
        return NotImplemented

    def __ge__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year >= other._fiscal_year
        return NotImplemented

class FiscalQuarter:
    def __init__(self, fiscal_year, quarter):
        assert isinstance(fiscal_year, int)
        assert isinstance(quarter, int)
        assert datetime.MINYEAR <= fiscal_year <= datetime.MAXYEAR
        assert 1 <= quarter <= 4

        self.__fiscal_year = fiscal_year
        self.__quarter = quarter

    @property
    def fiscal_year(self):
        return self.__fiscal_year

    @property
    def quarter(self):
        return self.__quarter

    @property
    def start(self):
        # Find the first month of the fiscal quarter
        month = START_MONTH
        month += (self.quarter - 1) * MONTHS_PER_QUARTER
        month %= 12
        if month == 0:
            month = 12

        # Find the calendar year of the start of the fiscal quarter
        year = self.fiscal_year
        if month >= START_MONTH:
            year -= 1

        return datetime.datetime(year, month, START_DAY, 0, 0, 0)

    @property
    def end(self):
        # Find the last month of the fiscal quarter
        month = START_MONTH
        month += self.quarter * MONTHS_PER_QUARTER - 1
        month %= 12
        if month == 0:
            month = 12

        # Find the calendar year of the end of the fiscal quarter
        year = self.fiscal_year
        if month >= START_MONTH:
            year -= 1

        # Find the last day of the last month of the fiscal quarter
        day = calendar.monthrange(year, month)[1]

        return datetime.datetime(year, month, day, 23, 59, 59)

class FiscalDate(datetime.date):
    """Wrapper around datetime.date that provides the following
    additional features:

    Attributes:
        fiscal_year: the fiscal year    [int]
        quarter:     the fiscal quarter [int: 1-4]

    Methods:
        is_q1_start: True if date is the first day of the 1st quarter
                     of the fiscal year, else False
        ...
        is_q4_start: True if date is the first day of the 4th quarter
                     of the fiscal year, else False

        is_q1_end:   True if date is the last day of the 1st quarter
                     of the fiscal year, else False
        ...
        is_q4_end:   True if date is the last day of the 4th quarter
                     of the fiscal year, else False

    The fiscal year starts on the first day of October
    and ends on the last day of September. For example, the 2018
    fiscal year starts on 10/1/2017 and ends on 9/30/2018.
    """

    @property
    def fiscal_year(self):
        """Returns the fiscal year."""

        fiscal_year = self.year
        if self.month >= START_MONTH:
            fiscal_year += 1

        return fiscal_year

    @property
    def prev_quarter_fiscal_year(self):
        """Returns the fiscal year of the previous quarter."""

        fiscal_year = self.fiscal_year
        if self.quarter == 1:
            fiscal_year -= 1

        return fiscal_year

    @property
    def next_quarter_fiscal_year(self):
        """Returns the fiscal year of the next quarter."""

        fiscal_year = self.fiscal_year
        if self.quarter == 4:
            fiscal_year += 1

        return fiscal_year

    @property
    def quarter(self):
        """Returns the quarter of the fiscal year."""

        month = self.month
        month -= START_MONTH
        if month < 0:
            month += 12
        quarter = month // MONTHS_PER_QUARTER + 1

        return quarter

    @property
    def prev_quarter(self):
        """Returns the previous quarter."""

        quarter = self.quarter - 1
        if quarter == 0:
            quarter = 4

        return quarter

    @property
    def next_quarter(self):
        """Returns the next quarter."""

        quarter = self.quarter + 1
        if quarter == 5:
            quarter = 1

        return quarter

    def is_quarter_start(self, quarter):
        start = FiscalQuarter(self.fiscal_year, quarter).start.date()

        return self == start

    def is_q1_start(self):
        return self.is_quarter_start(1)

    def is_q2_start(self):
        return self.is_quarter_start(2)

    def is_q3_start(self):
        return self.is_quarter_start(3)

    def is_q4_start(self):
        return self.is_quarter_start(4)

    def is_quarter_end(self, quarter):
        end = FiscalQuarter(self.fiscal_year, quarter).end.date()

        return self == end

    def is_q1_end(self):
        return self.is_quarter_end(1)

    def is_q2_end(self):
        return self.is_quarter_end(2)

    def is_q3_end(self):
        return self.is_quarter_end(3)

    def is_q4_end(self):
        return self.is_quarter_end(4)
<commit_msg>Flesh out FiscalQuarter class<commit_after>"""Utilities for managing the fiscal calendar."""

from __future__ import division

import calendar
import datetime

# The first month at the start of a new fiscal year.
# By default, use the U.S. federal government's fiscal year,
# which starts on October 1st and ends on September 30th.
START_MONTH = 10
START_DAY = 1

# Number of months in each quarter
MONTHS_PER_QUARTER = 12 // 4

MIN_QUARTER = 1
MAX_QUARTER = 4

def _check_int(value):
    """Check if value is an int or int-like string.

    :param value: The value to test
    :return: The value
    :rtype: int
    :raises TypeError: If value is not an int or int-like string
    """
    if isinstance(value, int):
        return value
    elif isinstance(value, str) and value.isdigit():
        return int(value)
    else:
        raise TypeError('an int or int-like string is required (got %s)' % (
            type(value).__name__))

def _check_year(year):
    """Check if year is a valid year.

    :param year: The year to test
    :return: The year
    :rtype: int
    :raises ValueError: If the year is out of range
    """
    year = _check_int(year)

    if datetime.MINYEAR <= year <= datetime.MAXYEAR:
        return year
    else:
        raise ValueError('year must be in %d..%d' % (
            datetime.MINYEAR, datetime.MAXYEAR), year)

def _check_quarter(quarter):
    """Check if quarter is a valid quarter.

    :param quarter: The quarter to test
    :return: The quarter
    :rtype: int
    :raises ValueError: If the quarter is out of range
    """
    quarter = _check_int(quarter)

    if MIN_QUARTER <= quarter <= MAX_QUARTER:
        return quarter
    else:
        raise ValueError('quarter must be in %d..%d' % (
            MIN_QUARTER, MAX_QUARTER), quarter)

class FiscalYear(object):
    """A class representing a single fiscal year."""

    __slots__ = '_fiscal_year'

    def __new__(cls, fiscal_year):
        """Constructor.

        :param fiscal_year: The fiscal year
        :type fiscal_year: int or str
        :returns: A newly constructed FiscalYear object
        :rtype: FiscalYear
        :raises TypeError: If the fiscal_year is not an int or int-like string
        :raises ValueError: If the fiscal_year is out of range
        """
        fiscal_year = _check_year(fiscal_year)

        self = super(FiscalYear, cls).__new__(cls)
        self._fiscal_year = fiscal_year
        return self

    def __repr__(self):
        """Convert to formal string, for repr().

        >>> fy = FiscalYear(2017)
        >>> repr(fy)
        'fiscalyear.FiscalYear(2017)'
        """
        return '%s.%s(%d)' % (self.__class__.__module__,
                              self.__class__.__name__,
                              self._fiscal_year)

    def __str__(self):
        """Convert to informal string, for str().

        >>> fy = FiscalYear(2017)
        >>> str(fy)
        'FY2017'
        """
        return 'FY%d' % (self._fiscal_year)

    # TODO: Implement __format__ so that you can print
    # fiscal year as 17 or 2017 (%y or %Y)

    # Read-only field accessors

    @property
    def fiscal_year(self):
        """Fiscal year."""
        return self._fiscal_year

    @property
    def start(self):
        """Start of the fiscal year.

        :rtype: datetime.datetime
        """
        return self.q1.start

    @property
    def end(self):
        """End of the fiscal year.

        :rtype: datetime.datetime
        """
        return self.q4.end

    @property
    def q1(self):
        """The first quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 1)

    @property
    def q2(self):
        """The second quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 2)

    @property
    def q3(self):
        """The third quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 3)

    @property
    def q4(self):
        """The fourth quarter of the fiscal year.

        :rtype: FiscalQuarter
        """
        return FiscalQuarter(self.fiscal_year, 4)

    # Comparisons of FiscalYear objects with other

    def __lt__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year < other._fiscal_year
        return NotImplemented

    def __le__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year <= other._fiscal_year
        return NotImplemented

    def __eq__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year == other._fiscal_year
        return NotImplemented

    def __ne__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year != other._fiscal_year
        return NotImplemented

    def __gt__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year > other._fiscal_year
        return NotImplemented

    def __ge__(self, other):
        if isinstance(other, FiscalYear):
            return self._fiscal_year >= other._fiscal_year
        return NotImplemented

class FiscalQuarter(object):
    """A class representing a single fiscal quarter."""

    __slots__ = ['_fiscal_year', '_quarter']

    def __new__(cls, fiscal_year, quarter):
        """Constructor.

        :param fiscal_year: The fiscal year
        :type fiscal_year: int or str
        :param quarter: The fiscal quarter [1 - 4]
        :type quarter: int or str
        :returns: A newly constructed FiscalQuarter object
        :rtype: FiscalQuarter
        :raises TypeError: If the fiscal_year or quarter is not
            an int or int-like string
        :raises ValueError: If the fiscal_year or quarter is out of range
        """
        fiscal_year = _check_year(fiscal_year)
        quarter = _check_quarter(quarter)

        self = super(FiscalQuarter, cls).__new__(cls)
        self._fiscal_year = fiscal_year
        self._quarter = quarter
        return self

    def __repr__(self):
        """Convert to formal string, for repr().

        >>> q3 = FiscalQuarter(2017, 3)
        >>> repr(q3)
        'fiscalyear.FiscalQuarter(2017, 3)'
        """
        return '%s.%s(%d, %d)' % (self.__class__.__module__,
                                  self.__class__.__name__,
                                  self._fiscal_year,
                                  self._quarter)

    def __str__(self):
        """Convert to informal string, for str().

        >>> q3 = FiscalQuarter(2017, 3)
        >>> str(q3)
        'FY2017 Q3'
        """
        return 'FY%d Q%d' % (self._fiscal_year,
                             self._quarter)

    # TODO: Implement __format__ so that you can print
    # fiscal year as 17 or 2017 (%y or %Y)

    # Read-only field accessors

    @property
    def fiscal_year(self):
        """Fiscal year."""
        return self._fiscal_year

    @property
    def quarter(self):
        """Fiscal quarter."""
        return self._quarter

    @property
    def start(self):
        """Start of the fiscal quarter.

        :rtype: datetime.datetime
        """
        # Find the first month of the fiscal quarter
        month = START_MONTH
        month += (self.quarter - 1) * MONTHS_PER_QUARTER
        month %= 12
        if month == 0:
            month = 12

        # Find the calendar year of the start of the fiscal quarter
        year = self.fiscal_year
        if month >= START_MONTH:
            year -= 1

        return datetime.datetime(year, month, START_DAY, 0, 0, 0)

    @property
    def end(self):
        """End of the fiscal quarter.

        :rtype: datetime.datetime
        """
        # Find the last month of the fiscal quarter
        month = START_MONTH
        month += self.quarter * MONTHS_PER_QUARTER - 1
        month %= 12
        if month == 0:
            month = 12

        # Find the calendar year of the end of the fiscal quarter
        year = self.fiscal_year
        if month >= START_MONTH:
            year -= 1

        # Find the last day of the last month of the fiscal quarter
        day = calendar.monthrange(year, month)[1]

        return datetime.datetime(year, month, day, 23, 59, 59)

    # Comparisons of FiscalQuarter objects with other

    def __lt__(self, other):
        if isinstance(other, FiscalYear):
            return ((self._fiscal_year,  self._quarter) <
                    (other._fiscal_year, other._quarter))
        return NotImplemented

    def __le__(self, other):
        if isinstance(other, FiscalYear):
            return ((self._fiscal_year,  self._quarter) <=
                    (other._fiscal_year, other._quarter))
        return NotImplemented

    def __eq__(self, other):
        if isinstance(other, FiscalYear):
            return ((self._fiscal_year,  self._quarter) ==
                    (other._fiscal_year, other._quarter))
        return NotImplemented

    def __ne__(self, other):
        if isinstance(other, FiscalYear):
            return ((self._fiscal_year,  self._quarter) !=
                    (other._fiscal_year, other._quarter))
        return NotImplemented

    def __gt__(self, other):
        if isinstance(other, FiscalYear):
            return ((self._fiscal_year,  self._quarter) >
                    (other._fiscal_year, other._quarter))
        return NotImplemented

    def __ge__(self, other):
        if isinstance(other, FiscalYear):
            return ((self._fiscal_year,  self._quarter) >=
                    (other._fiscal_year, other._quarter))
        return NotImplemented

class FiscalDate(datetime.date):
    """Wrapper around datetime.date that provides the following
    additional features:

    Attributes:
        fiscal_year: the fiscal year    [int]
        quarter:     the fiscal quarter [int: 1-4]

    Methods:
        is_q1_start: True if date is the first day of the 1st quarter
                     of the fiscal year, else False
        ...
        is_q4_start: True if date is the first day of the 4th quarter
                     of the fiscal year, else False

        is_q1_end:   True if date is the last day of the 1st quarter
                     of the fiscal year, else False
        ...
        is_q4_end:   True if date is the last day of the 4th quarter
                     of the fiscal year, else False

    The fiscal year starts on the first day of October
    and ends on the last day of September. For example, the 2018
    fiscal year starts on 10/1/2017 and ends on 9/30/2018.
    """

    @property
    def fiscal_year(self):
        """Returns the fiscal year."""

        fiscal_year = self.year
        if self.month >= START_MONTH:
            fiscal_year += 1

        return fiscal_year

    @property
    def prev_quarter_fiscal_year(self):
        """Returns the fiscal year of the previous quarter."""

        fiscal_year = self.fiscal_year
        if self.quarter == 1:
            fiscal_year -= 1

        return fiscal_year

    @property
    def next_quarter_fiscal_year(self):
        """Returns the fiscal year of the next quarter."""

        fiscal_year = self.fiscal_year
        if self.quarter == 4:
            fiscal_year += 1

        return fiscal_year

    @property
    def quarter(self):
        """Returns the quarter of the fiscal year."""

        month = self.month
        month -= START_MONTH
        if month < 0:
            month += 12
        quarter = month // MONTHS_PER_QUARTER + 1

        return quarter

    @property
    def prev_quarter(self):
        """Returns the previous quarter."""

        quarter = self.quarter - 1
        if quarter == 0:
            quarter = 4

        return quarter

    @property
    def next_quarter(self):
        """Returns the next quarter."""

        quarter = self.quarter + 1
        if quarter == 5:
            quarter = 1

        return quarter

    def is_quarter_start(self, quarter):
        start = FiscalQuarter(self.fiscal_year, quarter).start.date()

        return self == start

    def is_q1_start(self):
        return self.is_quarter_start(1)

    def is_q2_start(self):
        return self.is_quarter_start(2)

    def is_q3_start(self):
        return self.is_quarter_start(3)

    def is_q4_start(self):
        return self.is_quarter_start(4)

    def is_quarter_end(self, quarter):
        end = FiscalQuarter(self.fiscal_year, quarter).end.date()

        return self == end

    def is_q1_end(self):
        return self.is_quarter_end(1)

    def is_q2_end(self):
        return self.is_quarter_end(2)

    def is_q3_end(self):
        return self.is_quarter_end(3)

    def is_q4_end(self):
        return self.is_quarter_end(4)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Create ssh executor system
'''
# Import python libs
import os
import copy
import json
import logging

# Import salt libs
import salt.client.ssh.shell
import salt.client.ssh.state
import salt.utils
import salt.utils.thin
import salt.roster
import salt.state
import salt.loader
import salt.minion
import salt.log

log = logging.getLogger(__name__)

def sls(mods, saltenv='base', test=None, exclude=None, env=None, **kwargs):
    '''
    Create the seed file for a state.sls run
    '''
    __opts__['grains'] = __grains__
    if env is not None:
        salt.utils.warn_until(
            'Boron',
            'Passing a salt environment should be done using \'saltenv\' '
            'not \'env\'. This functionality will be removed in Salt Boron.'
        )
        # Backwards compatibility
        saltenv = env

    __pillar__.update(kwargs.get('pillar', {}))
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    if isinstance(mods, str):
        mods = mods.split(',')
    high_data, errors = st_.render_highstate({saltenv: mods})
    if exclude:
        if isinstance(exclude, str):
            exclude = exclude.split(',')
        if '__exclude__' in high_data:
            high_data['__exclude__'].extend(exclude)
        else:
            high_data['__exclude__'] = exclude
    high_data, ext_errors = st_.state.reconcile_extend(high_data)
    errors += ext_errors
    errors += st_.state.verify_high(high_data)
    if errors:
        return errors
    high_data, req_in_errors = st_.state.requisite_in(high_data)
    errors += req_in_errors
    high_data = st_.state.apply_exclude(high_data)
    # Verify that the high data is structurally sound
    if errors:
        return errors
    # Compile and verify the raw chunks
    chunks = st_.state.compile_high_data(high_data)
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            __opts__['_thin_dir'],
            test,
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    try:
        return json.loads(stdout, object_hook=salt.utils.decode_dict)
    except Exception, e:
        log.error("JSON Render failed for: {0}".format(stdout))
        log.error(str(e))
    return stdout

def low(data, **kwargs):
    '''
    Execute a single low data call
    This function is mostly intended for testing the state system

    CLI Example:

    .. code-block:: bash

        salt '*' state.low '{"state": "pkg", "fun": "installed", "name": "vi"}'
    '''
    __opts__['grains'] = __grains__
    chunks = [data]
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    err = st_.verify_data(data)
    if err:
        return err
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format(
            __opts__['_thin_dir'],
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    return json.loads(stdout, object_hook=salt.utils.decode_dict)

def high(data, **kwargs):
    '''
    Execute the compound calls stored in a single set of high data
    This function is mostly intended for testing the state system

    CLI Example:

    .. code-block:: bash

        salt '*' state.high '{"vim": {"pkg": ["installed"]}}'
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    chunks = st_.state.compile_high_data(high)
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format(
            __opts__['_thin_dir'],
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    return json.loads(stdout, object_hook=salt.utils.decode_dict)

def highstate(test=None, **kwargs):
    '''
    Retrieve the state data from the salt master for this minion and execute it

    CLI Example:

    .. code-block:: bash

        salt '*' state.highstate

        salt '*' state.highstate exclude=sls_to_exclude
        salt '*' state.highstate exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]"
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    chunks = st_.compile_low_chunks()
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            __opts__['_thin_dir'],
            test,
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    try:
        stdout = json.loads(stdout, object_hook=salt.utils.decode_dict)
    except Exception, e:
        log.error('JSON Render failed for: {0}'.format(stdout))
        log.error(str(e))
    return stdout

def top(topfn, test=None, **kwargs):
    '''
    Execute a specific top file instead of the default

    CLI Example:

    .. code-block:: bash

        salt '*' state.top reverse_top.sls
        salt '*' state.top reverse_top.sls exclude=sls_to_exclude
        salt '*' state.top reverse_top.sls exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]"
    '''
    __opts__['grains'] = __grains__
    if salt.utils.test_mode(test=test, **kwargs):
        __opts__['test'] = True
    else:
        __opts__['test'] = __opts__.get('test', None)
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    st_.opts['state_top'] = os.path.join('salt://', topfn)
    chunks = st_.compile_low_chunks()
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            __opts__['_thin_dir'],
            test,
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    return json.loads(stdout, object_hook=salt.utils.decode_dict)

def show_highstate():
    '''
    Retrieve the highstate data from the salt master and display it

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_highstate
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    return st_.compile_highstate()

def show_lowstate():
    '''
    List out the low data that will be applied to this minion

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_lowstate
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    return st_.compile_low_chunks()

def show_sls(mods, saltenv='base', test=None, env=None, **kwargs):
    '''
    Display the state data from a specific sls or list of sls files on the
    master

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_sls core,edit.vim dev
    '''
    __opts__['grains'] = __grains__
    if env is not None:
        salt.utils.warn_until(
            'Boron',
            'Passing a salt environment should be done using \'saltenv\' '
            'not \'env\'. This functionality will be removed in Salt Boron.'
        )
        # Backwards compatibility
        saltenv = env

    opts = copy.copy(__opts__)
    if salt.utils.test_mode(test=test, **kwargs):
        opts['test'] = True
    else:
        opts['test'] = __opts__.get('test', None)
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    high_data, errors = st_.render_highstate({saltenv: mods})
    high_data, ext_errors = st_.state.reconcile_extend(high_data)
    errors += ext_errors
    errors += st_.state.verify_high(high_data)
    if errors:
        return errors
    high_data, req_in_errors = st_.state.requisite_in(high_data)
    errors += req_in_errors
    high_data = st_.state.apply_exclude(high_data)
    # Verify that the high data is structurally sound
    if errors:
        return errors
    return high_data

def show_top():
    '''
    Return the top data that the minion will use for a highstate

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_top
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    top_data = st_.get_top()
    errors = []
    errors += st_.verify_tops(top_data)
    if errors:
        return errors
    matches = st_.top_matches(top_data)
    return matches
<commit_msg>Fix #12810<commit_after># -*- coding: utf-8 -*-
'''
Create ssh executor system
'''
# Import python libs
import os
import copy
import json
import logging

# Import salt libs
import salt.client.ssh.shell
import salt.client.ssh.state
import salt.utils
import salt.utils.thin
import salt.roster
import salt.state
import salt.loader
import salt.minion
import salt.log

log = logging.getLogger(__name__)

def sls(mods, saltenv='base', test=None, exclude=None, env=None, **kwargs):
    '''
    Create the seed file for a state.sls run
    '''
    __opts__['grains'] = __grains__
    if env is not None:
        salt.utils.warn_until(
            'Boron',
            'Passing a salt environment should be done using \'saltenv\' '
            'not \'env\'. This functionality will be removed in Salt Boron.'
        )
        # Backwards compatibility
        saltenv = env

    __pillar__.update(kwargs.get('pillar', {}))
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    if isinstance(mods, str):
        mods = mods.split(',')
    high_data, errors = st_.render_highstate({saltenv: mods})
    if exclude:
        if isinstance(exclude, str):
            exclude = exclude.split(',')
        if '__exclude__' in high_data:
            high_data['__exclude__'].extend(exclude)
        else:
            high_data['__exclude__'] = exclude
    high_data, ext_errors = st_.state.reconcile_extend(high_data)
    errors += ext_errors
    errors += st_.state.verify_high(high_data)
    if errors:
        return errors
    high_data, req_in_errors = st_.state.requisite_in(high_data)
    errors += req_in_errors
    high_data = st_.state.apply_exclude(high_data)
    # Verify that the high data is structurally sound
    if errors:
        return errors
    # Compile and verify the raw chunks
    chunks = st_.state.compile_high_data(high_data)
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            __opts__['_thin_dir'],
            test,
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    try:
        return json.loads(stdout, object_hook=salt.utils.decode_dict)
    except Exception, e:
        log.error("JSON Render failed for: {0}".format(stdout))
        log.error(str(e))
    return stdout

def low(data, **kwargs):
    '''
    Execute a single low data call
    This function is mostly intended for testing the state system

    CLI Example:

    .. code-block:: bash

        salt '*' state.low '{"state": "pkg", "fun": "installed", "name": "vi"}'
    '''
    __opts__['grains'] = __grains__
    chunks = [data]
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    err = st_.verify_data(data)
    if err:
        return err
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format(
            __opts__['_thin_dir'],
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    return json.loads(stdout, object_hook=salt.utils.decode_dict)

def high(data, **kwargs):
    '''
    Execute the compound calls stored in a single set of high data
    This function is mostly intended for testing the state system

    CLI Example:

    .. code-block:: bash

        salt '*' state.high '{"vim": {"pkg": ["installed"]}}'
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    chunks = st_.state.compile_high_data(high)
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format(
            __opts__['_thin_dir'],
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    return json.loads(stdout, object_hook=salt.utils.decode_dict)

def highstate(test=None, **kwargs):
    '''
    Retrieve the state data from the salt master for this minion and execute it

    CLI Example:

    .. code-block:: bash

        salt '*' state.highstate

        salt '*' state.highstate exclude=sls_to_exclude
        salt '*' state.highstate exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]"
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    chunks = st_.compile_low_chunks()
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            __opts__['_thin_dir'],
            test,
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    try:
        stdout = json.loads(stdout, object_hook=salt.utils.decode_dict)
    except Exception, e:
        log.error('JSON Render failed for: {0}'.format(stdout))
        log.error(str(e))
    return stdout

def top(topfn, test=None, **kwargs):
    '''
    Execute a specific top file instead of the default

    CLI Example:

    .. code-block:: bash

        salt '*' state.top reverse_top.sls
        salt '*' state.top reverse_top.sls exclude=sls_to_exclude
        salt '*' state.top reverse_top.sls exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]"
    '''
    __opts__['grains'] = __grains__
    if salt.utils.test_mode(test=test, **kwargs):
        __opts__['test'] = True
    else:
        __opts__['test'] = __opts__.get('test', None)
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    st_.opts['state_top'] = os.path.join('salt://', topfn)
    chunks = st_.compile_low_chunks()
    file_refs = salt.client.ssh.state.lowstate_file_refs(chunks, kwargs.get('extra_filerefs', ''))
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __opts__,
            chunks,
            file_refs,
            __pillar__)
    trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            __opts__['_thin_dir'],
            test,
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            **__salt__.kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['_thin_dir']))
    stdout, stderr, _ = single.cmd_block()
    return json.loads(stdout, object_hook=salt.utils.decode_dict)

def show_highstate():
    '''
    Retrieve the highstate data from the salt master and display it

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_highstate
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    return st_.compile_highstate()

def show_lowstate():
    '''
    List out the low data that will be applied to this minion

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_lowstate
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    return st_.compile_low_chunks()

def show_sls(mods, saltenv='base', test=None, env=None, **kwargs):
    '''
    Display the state data from a specific sls or list of sls files on the
    master

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_sls core,edit.vim dev
    '''
    __opts__['grains'] = __grains__
    if env is not None:
        salt.utils.warn_until(
            'Boron',
            'Passing a salt environment should be done using \'saltenv\' '
            'not \'env\'. This functionality will be removed in Salt Boron.'
        )
        # Backwards compatibility
        saltenv = env

    opts = copy.copy(__opts__)
    if salt.utils.test_mode(test=test, **kwargs):
        opts['test'] = True
    else:
        opts['test'] = __opts__.get('test', None)
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    high_data, errors = st_.render_highstate({saltenv: mods})
    high_data, ext_errors = st_.state.reconcile_extend(high_data)
    errors += ext_errors
    errors += st_.state.verify_high(high_data)
    if errors:
        return errors
    high_data, req_in_errors = st_.state.requisite_in(high_data)
    errors += req_in_errors
    high_data = st_.state.apply_exclude(high_data)
    # Verify that the high data is structurally sound
    if errors:
        return errors
    return high_data

def show_top():
    '''
    Return the top data that the minion will use for a highstate

    CLI Example:

    .. code-block:: bash

        salt '*' state.show_top
    '''
    __opts__['grains'] = __grains__
    st_ = salt.client.ssh.state.SSHHighState(__opts__, __pillar__, __salt__)
    top_data = st_.get_top()
    errors = []
    errors += st_.verify_tops(top_data)
    if errors:
        return errors
    matches = st_.top_matches(top_data)
    return matches
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from libmozdata.bugzilla import Bugzilla
from auto_nag.bzcleaner import BzCleaner
import lzma
import shutil
from bugbug.models.regression import RegressionModel
from urllib.request import urlretrieve
import requests

class Regression(BzCleaner):
    def __init__(self):
        super(Regression, self).__init__()
        MODEL_URL = 'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_train.latest/artifacts/public/regressionmodel.xz'
        r = requests.head(MODEL_URL, allow_redirects=True)
        new_etag = r.headers['ETag']

        try:
            with open('regressionmodel.etag', 'r') as f:
                old_etag = f.read()
        except IOError:
            old_etag = None

        if old_etag != new_etag:
            urlretrieve(MODEL_URL, 'regressionmodel.xz')

            with lzma.open('regressionmodel.xz', 'rb') as input_f:
                with open('regressionmodel', 'wb') as output_f:
                    shutil.copyfileobj(input_f, output_f)

            with open('regressionmodel.etag', 'w') as f:
                f.write(new_etag)

        self.model = RegressionModel.load('regressionmodel')

    def description(self):
        return 'Get bugs with missing regression keyword'

    def name(self):
        return 'regression'

    def template(self):
        return 'regression.html'

    def subject(self):
        return '[Using ML] Bugs with missing regression keyword'

    def ignore_bug_summary(self):
        return False

    def all_include_fields(self):
        return True

    def columns(self):
        return ['id', 'summary']

    def get_bz_params(self, date):
        start_date, end_date = self.get_dates(date)
        resolution_blacklist = self.get_config('resolution_blacklist', default=[])
        resolution_blacklist = ' '.join(resolution_blacklist)
        params = {
            'f1': 'keywords',
            'o1': 'notsubstring',
            'v1': 'regression',
            'f2': 'longdesc',
            'o2': 'anywordssubstr',
            'v2': 'regress caus',
            'f3': 'resolution',
            'o3': 'nowords',
            'v3': resolution_blacklist,
            'f4': 'longdesc',
            'o4': 'changedafter',
            'v4': start_date,
            'f5': 'longdesc',
            'o5': 'changedbefore',
            'v5': end_date,
            # Ignore bugs filed by the intermittent failure bot.
            'emailtype1': 'notequals',
            'email1': 'intermittent-bug-filer@mozilla.bugs',
            'emailreporter1': 1,
        }

        return params

    def get_data(self):
        return {'regressions': set(), 'others': {}, 'summaries': {}}

    def bughandler(self, bug, data):
        keywords = bug.get('keywords', [])
        if 'regressionwindow-wanted' in keywords:
            data['regressions'].add(bug['id'])
        else:
            has_regression_range = bug.get('cf_has_regression_range', '---')
            if has_regression_range == 'yes':
                data['regressions'].add(bug['id'])
            else:
                data['others'][bug['id']] = bug
        data['summaries'][bug['id']] = self.get_summary(bug)

    def retrieve_comments(self, bugs):
        """Retrieve bug comments"""

        def comment_handler(bug, bug_id):
            bugs[int(bug_id)]['comments'] = bug['comments']

        Bugzilla(
            bugids=[bug_id for bug_id in bugs.keys()],
            commenthandler=comment_handler,
            comment_include_fields=['text'],
        ).get_data().wait()

    def retrieve_history(self, bugs):
        """Retrieve bug history"""

        def history_handler(bug):
            bugs[int(bug['id'])]['history'] = bug['history']

        Bugzilla(
            bugids=[bug_id for bug_id in bugs.keys()], historyhandler=history_handler
        ).get_data().wait()

    def remove_using_history(self, bugs):
        to_remove = set()
        for bug_id, bug in bugs.items():
            for h in bug['history']:
                changes = h.get('changes', [])
                for change in changes:
                    if (
                        change['field_name'] == 'keywords'
                        and change['removed'] == 'regression'
                    ):
                        to_remove.add(bug_id)
                        return

        for bug_id in to_remove:
            del bugs[bug_id]

    def retrieve_attachments(self, bugs):
        """Retrieve bug attachments"""

        def attachment_handler(bug, bug_id):
            bugs[int(bug_id)]['attachments'] = bug

        Bugzilla(
            bugids=[bug_id for bug_id in bugs.keys()],
            attachmenthandler=attachment_handler,
            attachment_include_fields=[
                'id',
                'is_obsolete',
                'flags',
                'is_patch',
                'creator',
                'content_type',
            ],
        ).get_data().wait()

    def get_bugs(self, date='today', bug_ids=[]):
        # Retrieve bugs to analyze.
        all_bug_data = super(Regression, self).get_bugs(date=date, bug_ids=bug_ids)

        bugs = all_bug_data['others']

        # Retrieve history.
        self.retrieve_history(bugs)

        # Remove bugs for which the regression keyword was set and removed in the past.
        self.remove_using_history(bugs)

        # Retrieve comments.
        self.retrieve_comments(bugs)

        # Retrieve attachments.
        self.retrieve_attachments(bugs)

        bugs = list(bugs.values())

        # Analyze bugs.
        probs = self.model.classify(bugs, True)

        reg_bugids = {bug['id'] for bug, prob in zip(bugs, probs) if prob[1] > 0.9}
        print(len(reg_bugids))

        # Add bugs that are certainly regressions.
        reg_bugids |= all_bug_data['regressions']

        # Attach summaries to bugs.
        reg_bugids = [(n, all_bug_data['summaries'][n]) for n in reg_bugids]

        bugs = {}
        for n in reg_bugids:
            bugid = str(n)
            bugs[bugid] = {'id': bugid, 'summary': all_bug_data['summaries'][n]}

if __name__ == '__main__':
    Regression().run()
<commit_msg>Fix bug in regression.py<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from libmozdata.bugzilla import Bugzilla
from auto_nag.bzcleaner import BzCleaner
import lzma
import shutil
from bugbug.models.regression import RegressionModel
from urllib.request import urlretrieve
import requests

class Regression(BzCleaner):
    def __init__(self):
        super(Regression, self).__init__()
        MODEL_URL = 'https://index.taskcluster.net/v1/task/project.releng.services.project.testing.bugbug_train.latest/artifacts/public/regressionmodel.xz'
        r = requests.head(MODEL_URL, allow_redirects=True)
        new_etag = r.headers['ETag']

        try:
            with open('regressionmodel.etag', 'r') as f:
                old_etag = f.read()
        except IOError:
            old_etag = None

        if old_etag != new_etag:
            urlretrieve(MODEL_URL, 'regressionmodel.xz')

            with lzma.open('regressionmodel.xz', 'rb') as input_f:
                with open('regressionmodel', 'wb') as output_f:
                    shutil.copyfileobj(input_f, output_f)

            with open('regressionmodel.etag', 'w') as f:
                f.write(new_etag)

        self.model = RegressionModel.load('regressionmodel')

    def description(self):
        return 'Get bugs with missing regression keyword'

    def name(self):
        return 'regression'

    def template(self):
        return 'regression.html'

    def subject(self):
        return '[Using ML] Bugs with missing regression keyword'

    def ignore_bug_summary(self):
        return False

    def all_include_fields(self):
        return True

    def columns(self):
        return ['id', 'summary']

    def get_bz_params(self, date):
        start_date, end_date = self.get_dates(date)
        resolution_blacklist = self.get_config('resolution_blacklist', default=[])
        resolution_blacklist = ' '.join(resolution_blacklist)
        params = {
            'f1': 'keywords',
            'o1': 'notsubstring',
            'v1': 'regression',
            'f2': 'longdesc',
            'o2': 'anywordssubstr',
            'v2': 'regress caus',
            'f3': 'resolution',
            'o3': 'nowords',
            'v3': resolution_blacklist,
            'f4': 'longdesc',
            'o4': 'changedafter',
            'v4': start_date,
            'f5': 'longdesc',
            'o5': 'changedbefore',
            'v5': end_date,
            # Ignore bugs filed by the intermittent failure bot.
            'emailtype1': 'notequals',
            'email1': 'intermittent-bug-filer@mozilla.bugs',
            'emailreporter1': 1,
        }

        return params

    def get_data(self):
        return {'regressions': set(), 'others': {}, 'summaries': {}}

    def bughandler(self, bug, data):
        keywords = bug.get('keywords', [])
        if 'regressionwindow-wanted' in keywords:
            data['regressions'].add(bug['id'])
        else:
            has_regression_range = bug.get('cf_has_regression_range', '---')
            if has_regression_range == 'yes':
                data['regressions'].add(bug['id'])
            else:
                data['others'][bug['id']] = bug
        data['summaries'][bug['id']] = self.get_summary(bug)

    def retrieve_comments(self, bugs):
        """Retrieve bug comments"""

        def comment_handler(bug, bug_id):
            bugs[int(bug_id)]['comments'] = bug['comments']

        Bugzilla(
            bugids=[bug_id for bug_id in bugs.keys()],
            commenthandler=comment_handler,
            comment_include_fields=['text'],
        ).get_data().wait()

    def retrieve_history(self, bugs):
        """Retrieve bug history"""

        def history_handler(bug):
            bugs[int(bug['id'])]['history'] = bug['history']

        Bugzilla(
            bugids=[bug_id for bug_id in bugs.keys()], historyhandler=history_handler
        ).get_data().wait()

    def remove_using_history(self, bugs):
        to_remove = set()
        for bug_id, bug in bugs.items():
            for h in bug['history']:
                changes = h.get('changes', [])
                for change in changes:
                    if (
                        change['field_name'] == 'keywords'
                        and change['removed'] == 'regression'
                    ):
                        to_remove.add(bug_id)
                        return

        for bug_id in to_remove:
            del bugs[bug_id]

    def retrieve_attachments(self, bugs):
        """Retrieve bug attachments"""

        def attachment_handler(bug, bug_id):
            bugs[int(bug_id)]['attachments'] = bug

        Bugzilla(
            bugids=[bug_id for bug_id in bugs.keys()],
            attachmenthandler=attachment_handler,
            attachment_include_fields=[
                'id',
                'is_obsolete',
                'flags',
                'is_patch',
                'creator',
                'content_type',
            ],
        ).get_data().wait()

    def get_bugs(self, date='today', bug_ids=[]):
        # Retrieve bugs to analyze.
        all_bug_data = super(Regression, self).get_bugs(date=date, bug_ids=bug_ids)

        bugs = all_bug_data['others']

        # Retrieve history.
        self.retrieve_history(bugs)

        # Remove bugs for which the regression keyword was set and removed in the past.
        self.remove_using_history(bugs)

        # Retrieve comments.
        self.retrieve_comments(bugs)

        # Retrieve attachments.
        self.retrieve_attachments(bugs)

        bugs = list(bugs.values())

        # Analyze bugs.
        probs = self.model.classify(bugs, True)

        reg_bugids = {bug['id'] for bug, prob in zip(bugs, probs) if prob[1] > 0.9}
        print(len(reg_bugids))

        # Add bugs that are certainly regressions.
        reg_bugids |= all_bug_data['regressions']

        bugs = {}
        for n in reg_bugids:
            bugid = str(n)
            bugs[bugid] = {'id': bugid, 'summary': all_bug_data['summaries'][n]}

        return bugs

if __name__ == '__main__':
    Regression().run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2012-2015 MUJIN Inc
# Mujin vision controller client for bin picking task

# system imports

# mujin imports
from mujincontrollerclient import zmqclient
from . import VisionControllerClientError

# logging
from logging import getLogger
log = getLogger(__name__)

class VisionControllerClient(object):
    """mujin vision controller client for bin picking task
    """

    def __init__(self, hostname, commandport, ctx=None):
        """connects to vision server, initializes vision server, and sets up parameters
        :param hostname: e.g. visioncontroller1
        :param commandport: e.g. 7004
        :param ctx: zmq context
        """
        self.hostname = hostname
        self.commandport = commandport
        self._zmqclient = zmqclient.ZmqClient(hostname, commandport, ctx)

    def __del__(self):
        self.Destroy()

    def Destroy(self):
        if self._zmqclient is not None:
            self._zmqclient.Destroy()
            self._zmqclient = None

    def _ExecuteCommand(self, command, timeout=1.0):
        response = self._zmqclient.SendCommand(command, timeout)
        if 'error' in response:
            raise VisionControllerClientError(response['error'])
        if 'computationtime' in response:
            log.verbose('%s took %f seconds' % (command['command'], response['computationtime'] / 1000.0))
        else:
            log.verbose('%s executed successfully' % (command['command']))
        return response

    def InitializeVisionServer(self, visionmanagerconfigname, detectorconfigname, imagesubscriberconfigname, targetname, controllerclient, timeout=10.0):
        """initializes vision server
        :param visionmanagerconfigname: name of visionmanager config
        :param detectorconfigname: name of detector config
        :param imagesubscribername: name of imagesubscriber config
        :param targetname: name of the target object
        :param controllerclient: pointer to the BinpickingControllerClient that connects to the mujin controller we want the vision server to talk to
        :param timeout in seconds
        """
        controllerusernamepass = '%s:%s' % (controllerclient.controllerusername, controllerclient.controllerpassword)
        command = {"command": "Initialize",
                   "visionmanagerconfigname": visionmanagerconfigname,
                   "detectorconfigname": detectorconfigname,
                   "imagesubscriberconfigname": imagesubscriberconfigname,
                   "mujinControllerIp": controllerclient.controllerIp,
                   "mujinControllerPort": controllerclient.controllerPort,
                   "mujinControllerUsernamePass": controllerusernamepass,
                   "robotControllerUri": controllerclient.GetRobotControllerUri(),
                   "robotDeviceIOUri": controllerclient.GetRobotDeviceIOUri(),
                   "binpickingTaskZmqPort": controllerclient.taskzmqport,
                   "binpickingTaskHeartbeatPort": controllerclient.taskheartbeatport,
                   "binpickingTaskHeartbeatTimeout": controllerclient.taskheartbeattimeout,
                   "binpickingTaskScenePk": controllerclient.scenepk,
                   "robotname": controllerclient.robotname,
                   "targetname": targetname,
                   "tasktype": controllerclient.tasktype
                   }

        log.verbose('Initializing vision system...')
        return self._ExecuteCommand(command, timeout)

    def IsDetectionRunning(self, timeout=10.0):
        log.verbose('checking detection status...')
        command = {'command': 'IsDetectionRunning'}
        return self._ExecuteCommand(command, timeout)

    def DetectObjects(self, regionname=None, cameranames=None, ignoreocclusion=None, maxage=None, timeout=10.0):
        """detects objects
        :param regionname: name of the bin
        :param cameranames: a list of names of cameras to use for detection, if None, then use all cameras available
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param timeout in seconds
        :return: detected objects in world frame in a json dictionary, the translation info is in milimeter, e.g. {'objects': [{'name': 'target_0', 'translation_': [1,2,3], 'quat_': [1,0,0,0], 'confidence': 0.8}]}
        """
        log.verbose('Detecting objects...')
        command = {"command": "DetectObjects",
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def StartDetectionThread(self, regionname=None, cameranames=None, voxelsize=None, pointsize=None, ignoreocclusion=None, maxage=None, obstaclename=None, timeout=1.0):
        """starts detection thread to continuously detect objects. the vision server will send detection results directly to mujin controller.
        :param regionname: name of the bin
        :param cameranames: a list of names of cameras to use for detection, if None, then use all cameras available
        :param voxelsize: in meter
        :param pointsize: in meter
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param obstaclename: name of the collision obstacle
        :param timeout in seconds
        :return: returns immediately once the call completes
        """
        log.verbose('Starting detection thread...')
        command = {"command": "StartDetectionLoop",
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if voxelsize is not None:
            command['voxelsize'] = voxelsize
        if pointsize is not None:
            command['pointsize'] = pointsize
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        if obstaclename is not None:
            command[obstaclename] = obstaclename
        return self._ExecuteCommand(command, timeout)

    def StopDetectionThread(self, timeout=1.0):
        """stops detection thread
        :param timeout in seconds
        """
        log.verbose('Stopping detection thread...')
        command = {"command": "StopDetectionLoop"}
        return self._ExecuteCommand(command, timeout)

    def SendPointCloudObstacleToController(self, regionname=None, cameranames=None, detectedobjects=None, voxelsize=None, pointsize=None, obstaclename=None, timeout=1.0):
        """Updates the point cloud obstacle with detected objects removed and sends it to mujin controller
        :param regionname: name of the region
        :param cameranames: a list of camera names to use for visualization, if None, then use all cameras available
        :param detectedobjects: a list of detected objects in world frame, the translation info is in meter, e.g. [{'name': 'target_0', 'translation_': [1,2,3], 'quat_': [1,0,0,0], 'confidence': 0.8}]
        :param voxelsize: in meter
        :param pointsize: in meter
        :param obstaclename: name of the obstacle
        :param timeout in seconds
        """
        log.verbose('Sending point cloud obstacle to mujin controller...')
        command = {'command': 'SendPointCloudObstacleToController'}
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if detectedobjects is not None:
            command['detectedobjects'] = list(detectedobjects)
        if voxelsize is not None:
            command['voxelsize'] = voxelsize
        if pointsize is not None:
            command['pointsize'] = pointsize
        if obstaclename is not None:
            command['obstaclename'] = obstaclename
        return self._ExecuteCommand(command, timeout)

    def DetectRegionTransform(self, regionname=None, cameranames=None, ignoreocclusion=None, maxage=None, timeout=1.0):
        """Detects the transform of the region
        :param regionname: name of the region
        :param cameranames: a list of camera names to use for visualization, if None, then use all cameras available
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param timeout in seconds
        """
        log.verbose('Detecting transform of region')
        command = {'command': 'DetectRegionTransform',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def VisualizePointCloudOnController(self, regionname=None, cameranames=None, pointsize=None, ignoreocclusion=None, maxage=None, timeout=1.0):
        """Visualizes the raw camera point clouds on mujin controller
        :param regionname: name of the region
        :param cameranames: a list of camera names to use for visualization, if None, then use all cameras available
        :param pointsize: in meter
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param timeout in seconds
        """
        log.verbose('sending camera point cloud to mujin controller...')
        command = {'command': 'VisualizePointCloudOnController',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if pointsize is not None:
            command['pointsize'] = pointsize
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def ClearVisualizationOnController(self, timeout=1.0):
        """Clears visualization made by VisualizePointCloudOnController
        :param timeout in seconds
        """
        log.verbose("clearing visualization on mujin controller...")
        command = {'command': 'ClearVisualizationOnController'}
        return self._ExecuteCommand(command, timeout)

    def GetVisionmanagerConfig(self, timeout=1.0):
        """Gets the current visionmanager config json string
        """
        log.verbose('getting current visionmanager config...')
        command = {'command': 'GetVisionmanagerConfig'}
        return self._ExecuteCommand(command, timeout)

    def GetDetectorConfig(self, timeout=1.0):
        """Gets the current detector config json string
        """
        log.verbose('getting current detector config...')
        command = {'command': 'GetDetectorConfig'}
        return self._ExecuteCommand(command, timeout)

    def GetImagesubscriberConfig(self, timeout=1.0):
        """Gets the current imagesubscriber config json string
        """
        log.verbose('getting current imagesubscriber config...')
        command = {'command': 'GetImagesubscriberConfig'}
        return self._ExecuteCommand(command, timeout)

    def SaveVisionmanagerConfig(self, visionmanagerconfigname, config="", timeout=1.0):
        """Saves the visionmanager config to disk
        :param visionmanagerconfigname name of the visionmanager config
        :param config if not specified, then saves the current config
        """
        log.verbose('saving visionmanager config to disk...')
        command = {'command': 'SaveVisionmanagerConfig'}
        if config != '':
            command['config'] = config
        return self._ExecuteCommand(command, timeout)

    def SaveDetectorConfig(self, detectorconfigname, config="", timeout=1.0):
        """Saves the detector config to disk
        :param detectorconfigname name of the detector config
        :param config if not specified, then saves the current config
        """
        log.verbose('saving detector config to disk...')
        command = {'command': 'SaveDetectorConfig'}
        if config != '':
            command['config'] = config
        return self._ExecuteCommand(command, timeout)

    def SaveImagesubscriberConfig(self, imagesubscriberconfigname, config="", timeout=1.0):
        """Saves the imagesubscriber config to disk
        :param imagesubscriberconfigname name of the imagesubscriber config
        :param config if not specified, then saves the current config
        """
        log.verbose('saving imagesubscriber config to disk...')
        command = {'command': 'SaveImagesubscriberConfig'}
        if config != '':
            command['config'] = config
        return self._ExecuteCommand(command, timeout)

    ############################
    # internal methods
    ############################

    def SaveSnapshot(self, regionname=None, ignoreocclusion=None, maxage=None, timeout=1.0):
        """makes each sensor save a snapshot, all files will be saved to the runtime directory of the vision server
        :param timeout in seconds
        """
        log.verbose('saving snapshot')
        command = {"command": "SaveSnapshot",
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def UpdateDetectedObjects(self, objects, sendtocontroller=False, timeout=1.0):
        """updates the list of objects the vision server maintains
        usage: user may want to process the object location locally and then update the list on the vision server to improve detection
        :param objects: list of dictionaries of object info in world frame, the translation info is in meter, e.g. [{'name':'target_0', 'translation': [1,2,3], 'rotationmat': [[1,0,0],[0,1,0],[0,0,1]], 'score': 0.8}]
        :param sendtocontroller: whether to send the list to mujin controller
        :param timeout in seconds
        """
        log.verbose('Updating objects...')
        command = {"command": "UpdateDetectedObjects",
                   "detectedobjects": objects,
                   "sendtocontroller": sendtocontroller}
        return self._ExecuteCommand(command, timeout)

    def SyncRegion(self, regionname=None, timeout=1.0):
        """updates vision server with the lastest caontainer info on mujin controller
        usage: user may want to update the region's transform on the vision server after it gets updated on the mujin controller
        :param regionname: name of the bin
        :param timeout in seconds
        """
        log.verbose('Updating region...')
        command = {'command': 'SyncRegion',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        return self._ExecuteCommand(command, timeout)

    def SyncCameras(self, regionname=None, cameranames=None, timeout=1.0):
        """updates vision server with the lastest camera info on mujin controller
        usage: user may want to update a camera's transform on the vision server after it gets updated on the mujin controller
        :param regionname: name of the bin, of which the relevant camera info gets updated
        :param cameranames: a list of names of cameras, if None, then use all cameras available
        :param timeout in seconds
        """
        log.verbose('Updating cameras...')
        command = {'command': 'SyncCameras',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        return self._ExecuteCommand(command, timeout)

    def GetCameraId(self, cameraname, timeout=1.0):
        """gets the id of the camera
        :param cameraname: name of the camera
        :param timeout in seconds
        """
        log.verbose("Getting camera id...")
        command = {'command': 'GetCameraId',
                   'cameraname': cameraname}
        return self._ExecuteCommand(command, timeout)

    def GetStatusPort(self, timeout=1.0):
        """gets the status port of visionmanager
        """
        log.verbose("Getting status port...")
        command = {'command': 'GetStatusPort'}
        return self._ExecuteCommand(command, timeout)

    def GetConfigPort(self, timeout=1.0):
        """gets the config port of visionmanager
        """
        log.verbose("Getting config port...")
        command = {'command': 'GetConfigPort'}
        return self._ExecuteCommand(command, timeout)

    def GetLatestDetectedObjects(self, timeout=1.0):
        """gets the latest detected objects
        """
        log.verbose("Getting latest detected objects...")
        command = {'command': 'GetLatestDetectedObjects'}
        return self._ExecuteCommand(command, timeout)
<commit_msg>updated InitializeVisionServer()<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2012-2015 MUJIN Inc
# Mujin vision controller client for bin picking task

# system imports

# mujin imports
from mujincontrollerclient import zmqclient
from . import VisionControllerClientError

# logging
from logging import getLogger
log = getLogger(__name__)

class VisionControllerClient(object):
    """mujin vision controller client for bin picking task
    """

    def __init__(self, hostname, commandport, ctx=None):
        """connects to vision server, initializes vision server, and sets up parameters
        :param hostname: e.g. visioncontroller1
        :param commandport: e.g. 7004
        :param ctx: zmq context
        """
        self.hostname = hostname
        self.commandport = commandport
        self._zmqclient = zmqclient.ZmqClient(hostname, commandport, ctx)

    def __del__(self):
        self.Destroy()

    def Destroy(self):
        if self._zmqclient is not None:
            self._zmqclient.Destroy()
            self._zmqclient = None

    def _ExecuteCommand(self, command, timeout=1.0):
        response = self._zmqclient.SendCommand(command, timeout)
        if 'error' in response:
            raise VisionControllerClientError(response['error'])
        if 'computationtime' in response:
            log.verbose('%s took %f seconds' % (command['command'], response['computationtime'] / 1000.0))
        else:
            log.verbose('%s executed successfully' % (command['command']))
        return response

    def InitializeVisionServer(self, visionmanagerconfigname, detectorconfigname, imagesubscriberconfigname, targetname, streameruris, controllerclient, timeout=10.0):
        """initializes vision server
        :param visionmanagerconfigname: name of visionmanager config
        :param detectorconfigname: name of detector config
        :param imagesubscribername: name of imagesubscriber config
        :param targetname: name of the target object
        :param controllerclient: pointer to the BinpickingControllerClient that connects to the mujin controller we want the vision server to talk to
        :param timeout in seconds
        """
        controllerusernamepass = '%s:%s' % (controllerclient.controllerusername, controllerclient.controllerpassword)
        command = {"command": "Initialize",
                   "visionmanagerconfigname": visionmanagerconfigname,
                   "detectorconfigname": detectorconfigname,
                   "imagesubscriberconfigname": imagesubscriberconfigname,
                   "mujinControllerIp": controllerclient.controllerIp,
                   "mujinControllerPort": controllerclient.controllerPort,
                   "mujinControllerUsernamePass": controllerusernamepass,
                   "robotControllerUri": controllerclient.GetRobotControllerUri(),
                   "robotDeviceIOUri": controllerclient.GetRobotDeviceIOUri(),
                   "binpickingTaskZmqPort": controllerclient.taskzmqport,
                   "binpickingTaskHeartbeatPort": controllerclient.taskheartbeatport,
                   "binpickingTaskHeartbeatTimeout": controllerclient.taskheartbeattimeout,
                   "binpickingTaskScenePk": controllerclient.scenepk,
                   "robotname": controllerclient.robotname,
                   "targetname": targetname,
                   "streamerUris": streameruris,
                   "tasktype": controllerclient.tasktype
                   }

        log.verbose('Initializing vision system...')
        return self._ExecuteCommand(command, timeout)

    def IsDetectionRunning(self, timeout=10.0):
        log.verbose('checking detection status...')
        command = {'command': 'IsDetectionRunning'}
        return self._ExecuteCommand(command, timeout)

    def DetectObjects(self, regionname=None, cameranames=None, ignoreocclusion=None, maxage=None, timeout=10.0):
        """detects objects
        :param regionname: name of the bin
        :param cameranames: a list of names of cameras to use for detection, if None, then use all cameras available
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param timeout in seconds
        :return: detected objects in world frame in a json dictionary, the translation info is in milimeter, e.g. {'objects': [{'name': 'target_0', 'translation_': [1,2,3], 'quat_': [1,0,0,0], 'confidence': 0.8}]}
        """
        log.verbose('Detecting objects...')
        command = {"command": "DetectObjects",
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def StartDetectionThread(self, regionname=None, cameranames=None, voxelsize=None, pointsize=None, ignoreocclusion=None, maxage=None, obstaclename=None, timeout=1.0):
        """starts detection thread to continuously detect objects. the vision server will send detection results directly to mujin controller.
        :param regionname: name of the bin
        :param cameranames: a list of names of cameras to use for detection, if None, then use all cameras available
        :param voxelsize: in meter
        :param pointsize: in meter
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param obstaclename: name of the collision obstacle
        :param timeout in seconds
        :return: returns immediately once the call completes
        """
        log.verbose('Starting detection thread...')
        command = {"command": "StartDetectionLoop",
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if voxelsize is not None:
            command['voxelsize'] = voxelsize
        if pointsize is not None:
            command['pointsize'] = pointsize
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        if obstaclename is not None:
            command[obstaclename] = obstaclename
        return self._ExecuteCommand(command, timeout)

    def StopDetectionThread(self, timeout=1.0):
        """stops detection thread
        :param timeout in seconds
        """
        log.verbose('Stopping detection thread...')
        command = {"command": "StopDetectionLoop"}
        return self._ExecuteCommand(command, timeout)

    def SendPointCloudObstacleToController(self, regionname=None, cameranames=None, detectedobjects=None, voxelsize=None, pointsize=None, obstaclename=None, timeout=1.0):
        """Updates the point cloud obstacle with detected objects removed and sends it to mujin controller
        :param regionname: name of the region
        :param cameranames: a list of camera names to use for visualization, if None, then use all cameras available
        :param detectedobjects: a list of detected objects in world frame, the translation info is in meter, e.g. [{'name': 'target_0', 'translation_': [1,2,3], 'quat_': [1,0,0,0], 'confidence': 0.8}]
        :param voxelsize: in meter
        :param pointsize: in meter
        :param obstaclename: name of the obstacle
        :param timeout in seconds
        """
        log.verbose('Sending point cloud obstacle to mujin controller...')
        command = {'command': 'SendPointCloudObstacleToController'}
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if detectedobjects is not None:
            command['detectedobjects'] = list(detectedobjects)
        if voxelsize is not None:
            command['voxelsize'] = voxelsize
        if pointsize is not None:
            command['pointsize'] = pointsize
        if obstaclename is not None:
            command['obstaclename'] = obstaclename
        return self._ExecuteCommand(command, timeout)

    def DetectRegionTransform(self, regionname=None, cameranames=None, ignoreocclusion=None, maxage=None, timeout=1.0):
        """Detects the transform of the region
        :param regionname: name of the region
        :param cameranames: a list of camera names to use for visualization, if None, then use all cameras available
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param timeout in seconds
        """
        log.verbose('Detecting transform of region')
        command = {'command': 'DetectRegionTransform',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def VisualizePointCloudOnController(self, regionname=None, cameranames=None, pointsize=None, ignoreocclusion=None, maxage=None, timeout=1.0):
        """Visualizes the raw camera point clouds on mujin controller
        :param regionname: name of the region
        :param cameranames: a list of camera names to use for visualization, if None, then use all cameras available
        :param pointsize: in meter
        :param ignoreocclusion: whether to skip occlusion check
        :param maxage: max time difference in ms allowed between the current time and the timestamp of image used for detection, 0 means infinity
        :param timeout in seconds
        """
        log.verbose('sending camera point cloud to mujin controller...')
        command = {'command': 'VisualizePointCloudOnController',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        if pointsize is not None:
            command['pointsize'] = pointsize
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def ClearVisualizationOnController(self, timeout=1.0):
        """Clears visualization made by VisualizePointCloudOnController
        :param timeout in seconds
        """
        log.verbose("clearing visualization on mujin controller...")
        command = {'command': 'ClearVisualizationOnController'}
        return self._ExecuteCommand(command, timeout)

    def GetVisionmanagerConfig(self, timeout=1.0):
        """Gets the current visionmanager config json string
        """
        log.verbose('getting current visionmanager config...')
        command = {'command': 'GetVisionmanagerConfig'}
        return self._ExecuteCommand(command, timeout)

    def GetDetectorConfig(self, timeout=1.0):
        """Gets the current detector config json string
        """
        log.verbose('getting current detector config...')
        command = {'command': 'GetDetectorConfig'}
        return self._ExecuteCommand(command, timeout)

    def GetImagesubscriberConfig(self, timeout=1.0):
        """Gets the current imagesubscriber config json string
        """
        log.verbose('getting current imagesubscriber config...')
        command = {'command': 'GetImagesubscriberConfig'}
        return self._ExecuteCommand(command, timeout)

    def SaveVisionmanagerConfig(self, visionmanagerconfigname, config="", timeout=1.0):
        """Saves the visionmanager config to disk
        :param visionmanagerconfigname name of the visionmanager config
        :param config if not specified, then saves the current config
        """
        log.verbose('saving visionmanager config to disk...')
        command = {'command': 'SaveVisionmanagerConfig'}
        if config != '':
            command['config'] = config
        return self._ExecuteCommand(command, timeout)

    def SaveDetectorConfig(self, detectorconfigname, config="", timeout=1.0):
        """Saves the detector config to disk
        :param detectorconfigname name of the detector config
        :param config if not specified, then saves the current config
        """
        log.verbose('saving detector config to disk...')
        command = {'command': 'SaveDetectorConfig'}
        if config != '':
            command['config'] = config
        return self._ExecuteCommand(command, timeout)

    def SaveImagesubscriberConfig(self, imagesubscriberconfigname, config="", timeout=1.0):
        """Saves the imagesubscriber config to disk
        :param imagesubscriberconfigname name of the imagesubscriber config
        :param config if not specified, then saves the current config
        """
        log.verbose('saving imagesubscriber config to disk...')
        command = {'command': 'SaveImagesubscriberConfig'}
        if config != '':
            command['config'] = config
        return self._ExecuteCommand(command, timeout)

    ############################
    # internal methods
    ############################

    def SaveSnapshot(self, regionname=None, ignoreocclusion=None, maxage=None, timeout=1.0):
        """makes each sensor save a snapshot, all files will be saved to the runtime directory of the vision server
        :param timeout in seconds
        """
        log.verbose('saving snapshot')
        command = {"command": "SaveSnapshot",
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if ignoreocclusion is not None:
            command['ignoreocclusion'] = 1 if ignoreocclusion is True else 0
        if maxage is not None:
            command['maxage'] = maxage
        return self._ExecuteCommand(command, timeout)

    def UpdateDetectedObjects(self, objects, sendtocontroller=False, timeout=1.0):
        """updates the list of objects the vision server maintains
        usage: user may want to process the object location locally and then update the list on the vision server to improve detection
        :param objects: list of dictionaries of object info in world frame, the translation info is in meter, e.g. [{'name':'target_0', 'translation': [1,2,3], 'rotationmat': [[1,0,0],[0,1,0],[0,0,1]], 'score': 0.8}]
        :param sendtocontroller: whether to send the list to mujin controller
        :param timeout in seconds
        """
        log.verbose('Updating objects...')
        command = {"command": "UpdateDetectedObjects",
                   "detectedobjects": objects,
                   "sendtocontroller": sendtocontroller}
        return self._ExecuteCommand(command, timeout)

    def SyncRegion(self, regionname=None, timeout=1.0):
        """updates vision server with the lastest caontainer info on mujin controller
        usage: user may want to update the region's transform on the vision server after it gets updated on the mujin controller
        :param regionname: name of the bin
        :param timeout in seconds
        """
        log.verbose('Updating region...')
        command = {'command': 'SyncRegion',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        return self._ExecuteCommand(command, timeout)

    def SyncCameras(self, regionname=None, cameranames=None, timeout=1.0):
        """updates vision server with the lastest camera info on mujin controller
        usage: user may want to update a camera's transform on the vision server after it gets updated on the mujin controller
        :param regionname: name of the bin, of which the relevant camera info gets updated
        :param cameranames: a list of names of cameras, if None, then use all cameras available
        :param timeout in seconds
        """
        log.verbose('Updating cameras...')
        command = {'command': 'SyncCameras',
                   }
        if regionname is not None:
            command['regionname'] = regionname
        if cameranames is not None:
            command['cameranames'] = list(cameranames)
        return self._ExecuteCommand(command, timeout)

    def GetCameraId(self, cameraname, timeout=1.0):
        """gets the id of the camera
        :param cameraname: name of the camera
        :param timeout in seconds
        """
        log.verbose("Getting camera id...")
        command = {'command': 'GetCameraId',
                   'cameraname': cameraname}
        return self._ExecuteCommand(command, timeout)

    def GetStatusPort(self, timeout=1.0):
        """gets the status port of visionmanager
        """
        log.verbose("Getting status port...")
        command = {'command': 'GetStatusPort'}
        return self._ExecuteCommand(command, timeout)

    def GetConfigPort(self, timeout=1.0):
        """gets the config port of visionmanager
        """
        log.verbose("Getting config port...")
        command = {'command': 'GetConfigPort'}
        return self._ExecuteCommand(command, timeout)

    def GetLatestDetectedObjects(self, timeout=1.0):
        """gets the latest detected objects
        """
        log.verbose("Getting latest detected objects...")
        command = {'command': 'GetLatestDetectedObjects'}
        return self._ExecuteCommand(command, timeout)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: set sw=3 sts=3 ts=3

"""(Example) Simple UDP Server

A trivial simple example of using circuits to build a simple
UDP Socket Server.

This example demonstrates:
	* Basic Component creation.
	* Basic Event handling.
	* Basic Networking

This example makes use of:
	* Component
	* Event
	* Manager
	* lib.sockets.UDPServer
"""

from circuits import listener, Manager
from circuits.lib.sockets import UDPServer

class EchoServer(UDPServer):

	@listener("read")
	def onREAD(self, address, data):
		print "%s, %s:" % address
		print data.strip()
		self.write(address, data.strip())

def main():
	manager = Manager()
	server = EchoServer(8000)
	manager += server

	while True:
		try:
			manager.flush()
			server.poll()
		except KeyboardInterrupt:
			break

if __name__ == "__main__":
	main()
<commit_msg>examples/udpserver: Fixed and simplified<commit_after>#!/usr/bin/env python

from circuits.lib.sockets import UDPServer

class EchoServer(UDPServer):

    def read(self, address, data):
        print "%s: %s" % (address, data.strip())
        self.write(address, data.strip())

if __name__ == "__main__":
    EchoServer(8000).run()
<|endoftext|>