<commit_before>from yodatools.converter.Abstract import iOutputs
from odm2api.ODMconnection import dbconnection
from odm2api.ODM2.services import *
from odm2api.ODM2.models import setSchema, Sites, Results, SamplingFeatures, Specimens, MeasurementResults, TimeSeriesResults
import logging
import sqlite3

class dbOutput(iOutputs):

    def __init__(self, file_path=None, connection_string = None):
        if connection_string:
            self.connect_to_db(connection_string)
        self.added_objs= {}

    def connect_to_db(self, connection_string):
        self.session_factory_out = dbconnection.createConnectionFromString(connection_string)
        self._session_out = self.session_factory_out.getSession()
        self._engine_out = self.session_factory_out.engine
        setSchema(self._engine_out)
        return self._session_out

    def save(self, session, connection_string):
        self.data = self.parseObjects(session)
        self.connect_to_db(connection_string)

        #datasets
        self.check("datasets", self.data)
        # organization
        self.check("organizations", self.data)
        # people
        self.check("people", self.data)
        # affiliations
        self.check("affiliations", self.data)
        # citations
        self.check("citations", self.data)
        # authorlists
        self.check("authorlists", self.data)
        # datasetcitations
        self.check("datasetcitations", self.data)
        # spatialreferences
        self.check("spatialreferences", self.data)
        # samplingfeatures:
        self.check("samplingfeatures", self.data)
        # relatedfeatures
        self.check("relatedfeatures", self.data)
        # units
        self.check("units", self.data)
        # annotations
        self.check("annotations", self.data)
        # methods
        self.check("methods", self.data)
        # variables
        self.check("variables", self.data)
        # proc level
        self.check("processinglevels", self.data)
        # action
        self.check("actions", self.data)
        # featureaction
        self.check("featureactions", self.data)
        # actionby
        self.check("actionby", self.data)
        # relatedActions
        self.check("relatedactions", self.data)
        # result
        self.check("results", self.data)
        # datasetresults
        self.check("datasetsresults", self.data)
        # measurementResultValues
        self.check("measurementresultvalues", self.data)
        # MeasurementResultValueAnnotations
        self.check("measurementresultvalueannotations", self.data)
        # timeseriesresultvalues - ColumnDefinitions:, data:
        val = "timeseriesresultvalues"
        if val in self.data:
            self.save_ts(self.data[val])

        self.session_out.commit()

    def save_ts(self, values):
        pass

    def check(self, objname, data):

        if objname in data:
            vals = self.add_to_db( data[objname])
            self.added_objs[objname] =vals

    def add_to_db(self,  values):
        added = []
        from odm2api.ODM2.models import _changeSchema
        _changeSchema(None)
        for obj in values:
            try:
                valuedict = obj.__dict__.copy()
                valuedict = self.get_new_objects(obj, valuedict)
                valuedict.pop("_sa_instance_state")

                #delete primary key
                for v in valuedict.keys():
                    if v.lower() == obj.__mapper__.primary_key[0].name:
                        del valuedict[v]
                    elif "obj" in v.lower():
                        del valuedict[v]
                setSchema(self._engine_out)
                model = self.check_model(attr=obj)
                added.append(self.get_or_create(self._session_out, model, **valuedict))

            except Exception as e:
                print e
                self._session_out.rollback()
                # raise e
        return added

    def check_model(self, attr):
        model = type(attr)
        # if isinstance(attr, Sites) or isinstance(attr, Specimens):
        #     model = SamplingFeatures
        # elif isinstance(attr, MeasurementResults) or isinstance(attr, TimeSeriesResults):
        #     return Results

        return model

    def get_new_objects(self, obj, valuedict):
        for key in dir(obj):
            if "obj" in key.lower():  # key.contains("Obj"):
                try:
                    att = getattr(obj, key)
                    if att is not None:
                        attdict = att.__dict__.copy()
                        # for v in attdict.keys():
                        #     if "id" in v.lower() and "uuid" not in v.lower():
                        #         del attdict[v]
                        pk = None
                        for k in attdict.keys():
                            if k.lower() == att.__mapper__.primary_key[0].name:
                                pk = k
                                del attdict[k]
                            elif "obj" in k.lower():
                                del attdict[k]
                        attdict.pop("_sa_instance_state")
                        attdict = self.get_new_objects(att, attdict)

                        model = self.check_model(attr =att)
                        setSchema(self._engine_out)

                        get = getattr(model, pk)

                        # new_obj = self._session_out.query(model).filter_by(**attdict).first()
                        new_obj = self.get_inherited(self._session_out, model, **attdict)
                        # for s in self.added_objs["samplingfeatures"]:
                        #     if s == att:
                        #         new_obj = s

                        objkey = key.replace("Obj", "ID")
                        newkey = objkey
                        if objkey == "RelatedFeatureID":
                            newkey = "SamplingFeatureID"
                        elif objkey == "RelatedActionID":
                            newkey = "ActionID"
                        valuedict[objkey] = getattr(new_obj, newkey)

                except Exception as e:
                    print ("cannot find {} in {}. Error:{} in dbOutput".format(key, obj.__class__.__name__, e))
        return valuedict

    def get_inherited(self, sess, model, **kwargs):
        uuid = {}
        for key in kwargs.keys():
            if "uuid" in key.lower():
                uuid[key] = kwargs[key]
                break
        try:
            if len(uuid) > 0:
                instance = sess.query(model).filter_by(**uuid).first()
            else:
                instance = sess.query(model).filter_by(**kwargs).first()
            return instance
        except:
            return None

    def get_or_create(self, sess, model, **kwargs):
        # instance = sess.query(model).filter_by(**kwargs).first()
        instance = self.get_inherited(sess, model, **kwargs)

        if instance:
            return instance
        else:
            instance = model(**kwargs)
            print instance
            new_instance = sess.merge(instance)
            sess.flush()
            return new_instance

<commit_msg>update db output<commit_after>from yodatools.converter.Abstract import iOutputs
from odm2api.ODMconnection import dbconnection
from odm2api.ODM2.services import *
from odm2api.ODM2.models import setSchema, Sites, Results, SamplingFeatures, Specimens, MeasurementResults, TimeSeriesResults
import logging
import sqlite3

class dbOutput(iOutputs):

    def __init__(self, file_path=None, connection_string = None):
        if connection_string:
            self.connect_to_db(connection_string)
        self.added_objs= {}

    def connect_to_db(self, connection_string):
        self.session_factory_out = dbconnection.createConnectionFromString(connection_string)
        self._session_out = self.session_factory_out.getSession()
        self._engine_out = self.session_factory_out.engine
        setSchema(self._engine_out)
        return self._session_out

    def save(self, session, connection_string):
        self.data = self.parseObjects(session)
        self.connect_to_db(connection_string)

        #datasets
        self.check("datasets", self.data)
        # organization
        self.check("organizations", self.data)
        # people
        self.check("people", self.data)
        # affiliations
        self.check("affiliations", self.data)
        # citations
        self.check("citations", self.data)
        # authorlists
        self.check("authorlists", self.data)
        # datasetcitations
        self.check("datasetcitations", self.data)
        # spatialreferences
        self.check("spatialreferences", self.data)
        # samplingfeatures:
        self.check("samplingfeatures", self.data)
        # relatedfeatures
        self.check("relatedfeatures", self.data)
        # units
        self.check("units", self.data)
        # annotations
        self.check("annotations", self.data)
        # methods
        self.check("methods", self.data)
        # variables
        self.check("variables", self.data)
        # proc level
        self.check("processinglevels", self.data)
        # action
        self.check("actions", self.data)
        # featureaction
        self.check("featureactions", self.data)
        # actionby
        self.check("actionby", self.data)
        # relatedActions
        self.check("relatedactions", self.data)
        # result
        self.check("results", self.data)
        # datasetresults
        self.check("datasetsresults", self.data)
        # measurementResultValues
        self.check("measurementresultvalues", self.data)
        # MeasurementResultValueAnnotations
        self.check("measurementresultvalueannotations", self.data)
        # timeseriesresultvalues - ColumnDefinitions:, data:
        val = "timeseriesresultvalues"
        if val in self.data:
            self.save_ts(self.data[val])

        self._session_out.commit()

    def save_ts(self, values):
        pass

    def check(self, objname, data):

        if objname in data:
            vals = self.add_to_db( data[objname])
            self.added_objs[objname] =vals

    def add_to_db(self,  values):
        added = []
        from odm2api.ODM2.models import _changeSchema
        _changeSchema(None)
        for obj in values:
            try:
                valuedict = obj.__dict__.copy()
                valuedict = self.get_new_objects(obj, valuedict)
                valuedict.pop("_sa_instance_state")

                #delete primary key
                for v in valuedict.keys():
                    if v.lower() == obj.__mapper__.primary_key[0].name:
                        del valuedict[v]
                    elif "obj" in v.lower():
                        del valuedict[v]
                setSchema(self._engine_out)
                model = self.check_model(attr=obj)
                added.append(self.get_or_create(self._session_out, model, **valuedict))

            except Exception as e:
                print e
                self._session_out.rollback()
                # raise e
        return added

    def check_model(self, attr):
        model = type(attr)
        # if isinstance(attr, Sites) or isinstance(attr, Specimens):
        #     model = SamplingFeatures
        # elif isinstance(attr, MeasurementResults) or isinstance(attr, TimeSeriesResults):
        #     return Results

        return model

    def get_new_objects(self, obj, valuedict):
        for key in dir(obj):
            if "obj" in key.lower():  # key.contains("Obj"):
                try:
                    att = getattr(obj, key)
                    if att is not None:
                        attdict = att.__dict__.copy()
                        # for v in attdict.keys():
                        #     if "id" in v.lower() and "uuid" not in v.lower():
                        #         del attdict[v]
                        pk = None
                        for k in attdict.keys():
                            if k.lower() == att.__mapper__.primary_key[0].name:
                                pk = k
                                del attdict[k]
                            elif "obj" in k.lower():
                                del attdict[k]
                        attdict.pop("_sa_instance_state")
                        attdict = self.get_new_objects(att, attdict)

                        model = self.check_model(attr =att)
                        setSchema(self._engine_out)

                        get = getattr(model, pk)

                        # new_obj = self._session_out.query(model).filter_by(**attdict).first()
                        new_obj = self.get_inherited(self._session_out, model, **attdict)
                        # for s in self.added_objs["samplingfeatures"]:
                        #     if s == att:
                        #         new_obj = s

                        objkey = key.replace("Obj", "ID")
                        newkey = objkey
                        if objkey == "RelatedFeatureID":
                            newkey = "SamplingFeatureID"
                        elif objkey == "RelatedActionID":
                            newkey = "ActionID"
                        valuedict[objkey] = getattr(new_obj, newkey)

                except Exception as e:
                    print ("cannot find {} in {}. Error:{} in dbOutput".format(key, obj.__class__.__name__, e))
        return valuedict

    def get_inherited(self, sess, model, **kwargs):
        uuid = {}
        for key in kwargs.keys():
            if "uuid" in key.lower():
                uuid[key] = kwargs[key]
                break
        try:
            if len(uuid) > 0:
                instance = sess.query(model).filter_by(**uuid).first()
            else:
                instance = sess.query(model).filter_by(**kwargs).first()
            return instance
        except:
            return None

    def get_or_create(self, sess, model, **kwargs):
        # instance = sess.query(model).filter_by(**kwargs).first()
        instance = self.get_inherited(sess, model, **kwargs)

        if instance:
            return instance
        else:
            instance = model(**kwargs)
            # print instance
            new_instance = sess.merge(instance)
            sess.flush()
            return new_instance

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Display the list of current IPs.

This can exclude specified IPs and displays "no connection" if there are no IPs
to display.

Configuration parameters:
    cache_timeout: how often we refresh this module in seconds.
        (default 30)
    format: format of the output.
        (default 'Network: {format_iface}')
    format_iface: format string for the list of IPs of each interface.
        (default '{iface}: v4{{{ip}}} v6{{{ip6}}}')
    format_no_ip: string to show if there are no IPs to display.
        (default 'no connection')
    iface_blacklist: list of interfaces to ignore. Accepts shell-style wildcards.
        (default ['lo'])
    iface_sep: string to write between interfaces.
        (default ' ')
    ip_blacklist: list of IPs to ignore. Accepts shell-style wildcards.
        (default [])
    ip_sep: string to write between IP addresses.
        (default ',')
    remove_empty: do not show interfaces with no IP.
        (default True)

Format placeholders:
    {format_iface} the format_iface string.

Format placeholders for format_iface:
    {iface} name of the interface.
    {ip} list of IPv4 of the interface.
    {ip6} list of IPv6 of the interface.

Color options:
    color_bad: no IPs to show
    color_good: IPs to show

Example:

```
net_iplist {
    iface_blacklist = []
    ip_blacklist = ['127.*', '::1']
}
```

Requires:
    ip: utility found in iproute2 package
"""

import re
from subprocess import check_output
from fnmatch import fnmatch

class Py3status:
    cache_timeout = 30
    format = 'Network: {format_iface}'
    format_iface = '{iface}: v4{{{ip}}} v6{{{ip6}}}'
    format_no_ip = 'no connection'
    iface_blacklist = ['lo']
    iface_sep = ' '
    ip_blacklist = []
    ip_sep = ','
    remove_empty = True

    def __init__(self):
        self.iface_re = re.compile(r'\d+: (\w+):')
        self.ip_re = re.compile(r'\s+inet (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})/')
        self.ip6_re = re.compile(r'\s+inet6 ([\da-f:]+)/')

    def ip_list(self):
        response = {
            'cached_until': self.py3.time_in(seconds=self.cache_timeout),
            'full_text': ''
        }

        connection = False
        data = self._get_data()
        iface_list = []
        for iface, ips in data.items():
            if not self._check_blacklist(iface, self.iface_blacklist):
                continue

            ip_list = []
            ip6_list = []
            for ip in ips.get('ip', []):
                if self._check_blacklist(ip, self.ip_blacklist):
                    connection = True
                    ip_list.append(ip)
            for ip6 in ips.get('ip6', []):
                if self._check_blacklist(ip6, self.ip_blacklist):
                    connection = True
                    ip6_list.append(ip6)
            iface_list.append(self.py3.safe_format(self.format_iface,
                                                   {'iface': iface,
                                                    'ip': self.ip_sep.join(ip_list),
                                                    'ip6': self.ip_sep.join(ip6_list)}))
        if not connection:
            response['full_text'] = self.py3.safe_format(self.format_no_ip,
                                                         {'format_iface':
                                                             self.iface_sep.join(iface_list)})
            response['color'] = self.py3.COLOR_BAD
        else:
            response['full_text'] = self.py3.safe_format(self.format,
                                                         {'format_iface':
                                                             self.iface_sep.join(iface_list)})
            response['color'] = self.py3.COLOR_GOOD

        return response

    def _get_data(self):
        txt = check_output(['ip', 'address', 'show']).decode('utf-8').splitlines()
        len_txt = len(txt)

        idx = 0
        data = {}
        while idx < len_txt:
            iface = self.iface_re.match(txt[idx])
            idx += 1

            if iface is None:
                continue

            iface = iface.groups()[0]
            iface_data = {}
            while idx < len_txt and not self.iface_re.match(txt[idx]):
                ip = self.ip_re.match(txt[idx])
                if ip:
                    iface_data.setdefault('ip', []).append(ip.groups()[0])
                    idx += 1
                    continue
                ip6 = self.ip6_re.match(txt[idx])
                if ip6:
                    iface_data.setdefault('ip6', []).append(ip6.groups()[0])
                    idx += 1
                    continue
                idx += 1
            if self.remove_empty and iface_data == {}:
                continue
            data[iface] = iface_data

        return data

    def _check_blacklist(self, string, blacklist):
        for ignore in blacklist:
            if fnmatch(string, ignore):
                return False
        return True

if __name__ == "__main__":
    """
    Test this module by calling it directly.
    """
    from py3status.module_test import module_test
    module_test(Py3status)
<commit_msg>Add named groups in regexps<commit_after># -*- coding: utf-8 -*-
"""
Display the list of current IPs.

This can exclude specified IPs and displays "no connection" if there are no IPs
to display.

Configuration parameters:
    cache_timeout: how often we refresh this module in seconds.
        (default 30)
    format: format of the output.
        (default 'Network: {format_iface}')
    format_iface: format string for the list of IPs of each interface.
        (default '{iface}: v4{{{ip}}} v6{{{ip6}}}')
    format_no_ip: string to show if there are no IPs to display.
        (default 'no connection')
    iface_blacklist: list of interfaces to ignore. Accepts shell-style wildcards.
        (default ['lo'])
    iface_sep: string to write between interfaces.
        (default ' ')
    ip_blacklist: list of IPs to ignore. Accepts shell-style wildcards.
        (default [])
    ip_sep: string to write between IP addresses.
        (default ',')
    remove_empty: do not show interfaces with no IP.
        (default True)

Format placeholders:
    {format_iface} the format_iface string.

Format placeholders for format_iface:
    {iface} name of the interface.
    {ip} list of IPv4 of the interface.
    {ip6} list of IPv6 of the interface.

Color options:
    color_bad: no IPs to show
    color_good: IPs to show

Example:

```
net_iplist {
    iface_blacklist = []
    ip_blacklist = ['127.*', '::1']
}
```

Requires:
    ip: utility found in iproute2 package
"""

import re
from subprocess import check_output
from fnmatch import fnmatch

class Py3status:
    cache_timeout = 30
    format = 'Network: {format_iface}'
    format_iface = '{iface}: v4{{{ip}}} v6{{{ip6}}}'
    format_no_ip = 'no connection'
    iface_blacklist = ['lo']
    iface_sep = ' '
    ip_blacklist = []
    ip_sep = ','
    remove_empty = True

    def __init__(self):
        self.iface_re = re.compile(r'\d+: (?P<iface>\w+):')
        self.ip_re = re.compile(r'\s+inet (?P<ip>\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})/')
        self.ip6_re = re.compile(r'\s+inet6 (?P<ip6>[\da-f:]+)/')

    def ip_list(self):
        response = {
            'cached_until': self.py3.time_in(seconds=self.cache_timeout),
            'full_text': ''
        }

        connection = False
        data = self._get_data()
        iface_list = []
        for iface, ips in data.items():
            if not self._check_blacklist(iface, self.iface_blacklist):
                continue

            ip_list = []
            ip6_list = []
            for ip in ips.get('ip', []):
                if self._check_blacklist(ip, self.ip_blacklist):
                    connection = True
                    ip_list.append(ip)
            for ip6 in ips.get('ip6', []):
                if self._check_blacklist(ip6, self.ip_blacklist):
                    connection = True
                    ip6_list.append(ip6)
            iface_list.append(self.py3.safe_format(self.format_iface,
                                                   {'iface': iface,
                                                    'ip': self.ip_sep.join(ip_list),
                                                    'ip6': self.ip_sep.join(ip6_list)}))
        if not connection:
            response['full_text'] = self.py3.safe_format(self.format_no_ip,
                                                         {'format_iface':
                                                             self.iface_sep.join(iface_list)})
            response['color'] = self.py3.COLOR_BAD
        else:
            response['full_text'] = self.py3.safe_format(self.format,
                                                         {'format_iface':
                                                             self.iface_sep.join(iface_list)})
            response['color'] = self.py3.COLOR_GOOD

        return response

    def _get_data(self):
        txt = check_output(['ip', 'address', 'show']).decode('utf-8').splitlines()
        len_txt = len(txt)

        idx = 0
        data = {}
        while idx < len_txt:
            iface = self.iface_re.match(txt[idx])
            idx += 1

            if iface is None:
                continue

            iface = iface.group('iface')
            iface_data = {}
            while idx < len_txt and not self.iface_re.match(txt[idx]):
                ip = self.ip_re.match(txt[idx])
                if ip:
                    iface_data.setdefault('ip', []).append(ip.group('ip'))
                    idx += 1
                    continue
                ip6 = self.ip6_re.match(txt[idx])
                if ip6:
                    iface_data.setdefault('ip6', []).append(ip6.group('ip6'))
                    idx += 1
                    continue
                idx += 1
            if self.remove_empty and iface_data == {}:
                continue
            data[iface] = iface_data

        return data

    def _check_blacklist(self, string, blacklist):
        for ignore in blacklist:
            if fnmatch(string, ignore):
                return False
        return True

if __name__ == "__main__":
    """
    Test this module by calling it directly.
    """
    from py3status.module_test import module_test
    module_test(Py3status)
<|endoftext|>


<commit_before>import cupy
from cupy import core
from cupy.core import fusion
from cupy import _util

from cupy.core import _routines_indexing as _indexing
from cupy.core import _routines_statistics as _statistics

def argmax(a, axis=None, dtype=None, out=None, keepdims=False):
    """Returns the indices of the maximum along an axis.

    Args:
        a (cupy.ndarray): Array to take argmax.
        axis (int): Along which axis to find the maximum. ``a`` is flattened by
            default.
        dtype: Data type specifier.
        out (cupy.ndarray): Output array.
        keepdims (bool): If ``True``, the axis ``axis`` is preserved as an axis
            of length one.

    Returns:
        cupy.ndarray: The indices of the maximum of ``a`` along an axis.

    .. note::
       ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
       not in NumPy.

    .. note::
       ``axis`` argument accepts a tuple of ints, but this is specific to
       CuPy. NumPy does not support it.

    .. seealso:: :func:`numpy.argmax`

    """
    # TODO(okuta): check type
    return a.argmax(axis=axis, dtype=dtype, out=out, keepdims=keepdims)

def nanargmax(a, axis=None, dtype=None, out=None, keepdims=False):
    """Return the indices of the maximum values in the specified axis ignoring
    NaNs. For all-NaN slice ``-1`` is returned.
    Subclass cannot be passed yet, subok=True still unsupported

    Args:
        a (cupy.ndarray): Array to take nanargmax.
        axis (int): Along which axis to find the maximum. ``a`` is flattened by
            default.

    Returns:
        cupy.ndarray: The indices of the maximum of ``a``
            along an axis ignoring NaN values.

    .. note:: For performance reasons, ``cupy.nanargmax`` returns
            ``out of range values`` for all-NaN slice
            whereas ``numpy.nanargmax`` raises ``ValueError``
    .. seealso:: :func:`numpy.nanargmax`
    """
    if a.dtype.kind in 'biu':
        return argmax(a, axis=axis)

    return _statistics._nanargmax(a, axis, dtype, out, keepdims)

def argmin(a, axis=None, dtype=None, out=None, keepdims=False):
    """Returns the indices of the minimum along an axis.

    Args:
        a (cupy.ndarray): Array to take argmin.
        axis (int): Along which axis to find the minimum. ``a`` is flattened by
            default.
        dtype: Data type specifier.
        out (cupy.ndarray): Output array.
        keepdims (bool): If ``True``, the axis ``axis`` is preserved as an axis
            of length one.

    Returns:
        cupy.ndarray: The indices of the minimum of ``a`` along an axis.

    .. note::
       ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
       not in NumPy.

    .. note::
       ``axis`` argument accepts a tuple of ints, but this is specific to
       CuPy. NumPy does not support it.

    .. seealso:: :func:`numpy.argmin`

    """
    # TODO(okuta): check type
    return a.argmin(axis=axis, dtype=dtype, out=out, keepdims=keepdims)

def nanargmin(a, axis=None, dtype=None, out=None, keepdims=False):
    """Return the indices of the minimum values in the specified axis ignoring
    NaNs. For all-NaN slice ``-1`` is returned.
    Subclass cannot be passed yet, subok=True still unsupported

    Args:
        a (cupy.ndarray): Array to take nanargmin.
        axis (int): Along which axis to find the minimum. ``a`` is flattened by
            default.

    Returns:
        cupy.ndarray: The indices of the minimum of ``a``
            along an axis ignoring NaN values.

    .. note:: For performance reasons, ``cupy.nanargmin`` returns
            ``out of range values`` for all-NaN slice
            whereas ``numpy.nanargmin`` raises ``ValueError``
    .. seealso:: :func:`numpy.nanargmin`
    """
    if a.dtype.kind in 'biu':
        return argmin(a, axis=axis)

    return _statistics._nanargmin(a, axis, dtype, out, keepdims)

def nonzero(a):
    """Return the indices of the elements that are non-zero.

    Returns a tuple of arrays, one for each dimension of a,
    containing the indices of the non-zero elements in that dimension.

    Args:
        a (cupy.ndarray): array

    Returns:
        tuple of arrays: Indices of elements that are non-zero.

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.nonzero`

    """
    _util.check_array(a, arg_name='a')
    return a.nonzero()

def flatnonzero(a):
    """Return indices that are non-zero in the flattened version of a.

    This is equivalent to a.ravel().nonzero()[0].

    Args:
        a (cupy.ndarray): input array

    Returns:
        cupy.ndarray: Output array,
        containing the indices of the elements of a.ravel() that are non-zero.

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.flatnonzero`
    """
    _util.check_array(a, arg_name='a')
    return a.ravel().nonzero()[0]

_where_ufunc = core.create_ufunc(
    'cupy_where',
    ('???->?', '?bb->b', '?BB->B', '?hh->h', '?HH->H', '?ii->i', '?II->I',
     '?ll->l', '?LL->L', '?qq->q', '?QQ->Q', '?ee->e', '?ff->f',
     # On CUDA 6.5 these combinations don't work correctly (on CUDA >=7.0, it
     # works).
     # See issue #551.
     '?hd->d', '?Hd->d',
     '?dd->d', '?FF->F', '?DD->D'),
    'out0 = in0 ? in1 : in2')

def where(condition, x=None, y=None):
    """Return elements, either from x or y, depending on condition.

    If only condition is given, return ``condition.nonzero()``.

    Args:
        condition (cupy.ndarray): When True, take x, otherwise take y.
        x (cupy.ndarray): Values from which to choose on ``True``.
        y (cupy.ndarray): Values from which to choose on ``False``.

    Returns:
        cupy.ndarray: Each element of output contains elements of ``x`` when
            ``condition`` is ``True``, otherwise elements of ``y``. If only
            ``condition`` is given, return the tuple ``condition.nonzero()``,
            the indices where ``condition`` is True.

    .. warning::

        This function may synchronize the device if both ``x`` and ``y`` are
        omitted.

    .. seealso:: :func:`numpy.where`

    """

    missing = (x is None, y is None).count(True)

    if missing == 1:
        raise ValueError('Must provide both \'x\' and \'y\' or neither.')
    if missing == 2:
        return nonzero(condition)  # may synchronize

    if fusion._is_fusing():
        return fusion._call_ufunc(_where_ufunc, condition, x, y)
    return _where_ufunc(condition.astype('?'), x, y)

def argwhere(a):
    """Return the indices of the elements that are non-zero.

    Returns a (N, ndim) dimantional array containing the
    indices of the non-zero elements. Where `N` is number of
    non-zero elements and `ndim` is dimension of the given array.

    Args:
        a (cupy.ndarray): array

    Returns:
        cupy.ndarray: Indices of elements that are non-zero.

    .. seealso:: :func:`numpy.argwhere`

    """
    _util.check_array(a, arg_name='a')
    return _indexing._ndarray_argwhere(a)

# This is to allow using the same kernels for all dtypes, ints & floats
# as nan is a special case
_preamble = '''
template<typename T>
__device__ bool _isnan(T val) {
    return val != val;
}
'''

_searchsorted_kernel = core.ElementwiseKernel(
    'S x, raw T bins, int64 n_bins, bool side_is_right, '
    'bool assume_increassing',
    'int64 y',
    '''
    #ifdef __HIP_DEVICE_COMPILE__
    bool is_done = false;
    #endif

    // Array is assumed to be monotonically
    // increasing unless a check is requested with the
    // `assume_increassing = False` parameter.
    // `digitize` allows increasing and decreasing arrays.
    bool inc = true;
    if (!assume_increassing && n_bins >= 2) {
        // In the case all the bins are nan the array is considered
        // to be decreasing in numpy
        inc = (bins[0] <= bins[n_bins-1])
              || (!_isnan<T>(bins[0]) && _isnan<T>(bins[n_bins-1]));
    }

    if (_isnan<S>(x)) {
        long long pos = (inc ? n_bins : 0);
        if (!side_is_right) {
            if (inc) {
                while (pos > 0 && _isnan<T>(bins[pos-1])) {
                    --pos;
                }
            } else {
                while (pos < n_bins && _isnan<T>(bins[pos])) {
                    ++pos;
                }
            }
        }
        y = pos;
        #ifdef __HIP_DEVICE_COMPILE__
        is_done = true;
        #else
        return;
        #endif
    }

    bool greater = false;
    if (side_is_right) {
        greater = inc && x >= bins[n_bins-1];
    } else {
        greater = (inc ? x > bins[n_bins-1] : x <= bins[n_bins-1]);
    }
    if (greater) {
        #ifdef __HIP_DEVICE_COMPILE__
        if (!is_done) {
          y = n_bins;
          is_done = true;
        }
        #else
        y = n_bins;
        return;
        #endif
    }

    long long left = 0;
    // In the case the bins is all NaNs, digitize
    // needs to place all the valid values to the right
    if (!inc) {
        while (_isnan<T>(bins[left]) && left < n_bins) {
            ++left;
        }
        if (left == n_bins) {
            #ifdef __HIP_DEVICE_COMPILE__
            if (!is_done) {
              y = n_bins;
              is_done = true;
            }
            #else
            y = n_bins;
            return;
            #endif
        }
        if (side_is_right
                && !_isnan<T>(bins[n_bins-1]) && !_isnan<S>(x)
                && bins[n_bins-1] > x) {
            #ifdef __HIP_DEVICE_COMPILE__
            if (!is_done) {
              y = n_bins;
              is_done = true;
            }
            #else
            y = n_bins;
            return;
            #endif
        }
    }

    long long right = n_bins-1;
    while (left < right) {
        long long m = left + (right - left) / 2;
        bool look_right = true;
        if (side_is_right) {
            look_right = (inc ? bins[m] <= x : bins[m] > x);
        } else {
            look_right = (inc ? bins[m] < x : bins[m] >= x);
        }
        if (look_right) {
            left = m + 1;
        } else {
            right = m;
        }
    }
    #ifdef __HIP_DEVICE_COMPILE__
    if (!is_done) {
      y = right;
      is_done = true;
    }
    #else
    y = right;
    #endif
    ''', preamble=_preamble)

def searchsorted(a, v, side='left', sorter=None):
    """Finds indices where elements should be inserted to maintain order.

    Find the indices into a sorted array ``a`` such that,
    if the corresponding elements in ``v`` were inserted before the indices,
    the order of ``a`` would be preserved.

    Args:
        a (cupy.ndarray): Input array. If ``sorter`` is ``None``, then
            it must be sorted in ascending order,
            otherwise ``sorter`` must be an array of indices that sort it.
        v (cupy.ndarray): Values to insert into ``a``.
        side : {'left', 'right'}
            If ``left``, return the index of the first suitable location found
            If ``right``, return the last such index.
            If there is no suitable index, return either 0 or length of ``a``.
        sorter : 1-D array_like
            Optional array of integer indices that sort array ``a`` into
            ascending order. They are typically the result of
            :func:`~cupy.argsort`.

    Returns:
        cupy.ndarray: Array of insertion points with the same shape as ``v``.

    .. note:: When a is not in ascending order, behavior is undefined.

    .. seealso:: :func:`numpy.searchsorted`

    """
    return _searchsorted(a, v, side, sorter, True)

def _searchsorted(a, v, side, sorter, assume_increasing):
    """`assume_increasing` is used in the kernel to
    skip monotonically increasing or decreasing verification
    inside the cuda kernel.
    """
    if not isinstance(a, cupy.ndarray):
        raise NotImplementedError('Only int or ndarray are supported for a')

    if not isinstance(v, cupy.ndarray):
        raise NotImplementedError('Only int or ndarray are supported for v')

    if a.ndim > 1:
        raise ValueError('object too deep for desired array')
    if a.ndim < 1:
        raise ValueError('object of too small depth for desired array')
    if a.size == 0:
        return cupy.zeros(v.shape, dtype=cupy.int64)

    a_iscomplex = a.dtype.kind == 'c'
    v_iscomplex = v.dtype.kind == 'c'

    if a_iscomplex and not v_iscomplex:
        v = v.astype(a.dtype)
    elif v_iscomplex and not a_iscomplex:
        a = a.astype(v.dtype)

    # Numpy does not check if the array is monotonic inside searchsorted
    # which leads to undefined behavior in such cases.
    if sorter is not None:
        if sorter.dtype.kind not in ('i', 'u'):
            raise TypeError('sorter must be of integer type')
        if sorter.size != a.size:
            raise ValueError('sorter.size must equal a.size')
        a = a.take(sorter)

    y = cupy.zeros(v.shape, dtype=cupy.int64)

    _searchsorted_kernel(v, a, a.size, side == 'right', assume_increasing, y)
    return y

# TODO(okuta): Implement extract
<commit_msg>clean up<commit_after>import cupy
from cupy import core
from cupy.core import fusion
from cupy import _util

from cupy.core import _routines_indexing as _indexing
from cupy.core import _routines_statistics as _statistics

def argmax(a, axis=None, dtype=None, out=None, keepdims=False):
    """Returns the indices of the maximum along an axis.

    Args:
        a (cupy.ndarray): Array to take argmax.
        axis (int): Along which axis to find the maximum. ``a`` is flattened by
            default.
        dtype: Data type specifier.
        out (cupy.ndarray): Output array.
        keepdims (bool): If ``True``, the axis ``axis`` is preserved as an axis
            of length one.

    Returns:
        cupy.ndarray: The indices of the maximum of ``a`` along an axis.

    .. note::
       ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
       not in NumPy.

    .. note::
       ``axis`` argument accepts a tuple of ints, but this is specific to
       CuPy. NumPy does not support it.

    .. seealso:: :func:`numpy.argmax`

    """
    # TODO(okuta): check type
    return a.argmax(axis=axis, dtype=dtype, out=out, keepdims=keepdims)

def nanargmax(a, axis=None, dtype=None, out=None, keepdims=False):
    """Return the indices of the maximum values in the specified axis ignoring
    NaNs. For all-NaN slice ``-1`` is returned.
    Subclass cannot be passed yet, subok=True still unsupported

    Args:
        a (cupy.ndarray): Array to take nanargmax.
        axis (int): Along which axis to find the maximum. ``a`` is flattened by
            default.

    Returns:
        cupy.ndarray: The indices of the maximum of ``a``
            along an axis ignoring NaN values.

    .. note:: For performance reasons, ``cupy.nanargmax`` returns
            ``out of range values`` for all-NaN slice
            whereas ``numpy.nanargmax`` raises ``ValueError``
    .. seealso:: :func:`numpy.nanargmax`
    """
    if a.dtype.kind in 'biu':
        return argmax(a, axis=axis)

    return _statistics._nanargmax(a, axis, dtype, out, keepdims)

def argmin(a, axis=None, dtype=None, out=None, keepdims=False):
    """Returns the indices of the minimum along an axis.

    Args:
        a (cupy.ndarray): Array to take argmin.
        axis (int): Along which axis to find the minimum. ``a`` is flattened by
            default.
        dtype: Data type specifier.
        out (cupy.ndarray): Output array.
        keepdims (bool): If ``True``, the axis ``axis`` is preserved as an axis
            of length one.

    Returns:
        cupy.ndarray: The indices of the minimum of ``a`` along an axis.

    .. note::
       ``dtype`` and ``keepdim`` arguments are specific to CuPy. They are
       not in NumPy.

    .. note::
       ``axis`` argument accepts a tuple of ints, but this is specific to
       CuPy. NumPy does not support it.

    .. seealso:: :func:`numpy.argmin`

    """
    # TODO(okuta): check type
    return a.argmin(axis=axis, dtype=dtype, out=out, keepdims=keepdims)

def nanargmin(a, axis=None, dtype=None, out=None, keepdims=False):
    """Return the indices of the minimum values in the specified axis ignoring
    NaNs. For all-NaN slice ``-1`` is returned.
    Subclass cannot be passed yet, subok=True still unsupported

    Args:
        a (cupy.ndarray): Array to take nanargmin.
        axis (int): Along which axis to find the minimum. ``a`` is flattened by
            default.

    Returns:
        cupy.ndarray: The indices of the minimum of ``a``
            along an axis ignoring NaN values.

    .. note:: For performance reasons, ``cupy.nanargmin`` returns
            ``out of range values`` for all-NaN slice
            whereas ``numpy.nanargmin`` raises ``ValueError``
    .. seealso:: :func:`numpy.nanargmin`
    """
    if a.dtype.kind in 'biu':
        return argmin(a, axis=axis)

    return _statistics._nanargmin(a, axis, dtype, out, keepdims)

def nonzero(a):
    """Return the indices of the elements that are non-zero.

    Returns a tuple of arrays, one for each dimension of a,
    containing the indices of the non-zero elements in that dimension.

    Args:
        a (cupy.ndarray): array

    Returns:
        tuple of arrays: Indices of elements that are non-zero.

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.nonzero`

    """
    _util.check_array(a, arg_name='a')
    return a.nonzero()

def flatnonzero(a):
    """Return indices that are non-zero in the flattened version of a.

    This is equivalent to a.ravel().nonzero()[0].

    Args:
        a (cupy.ndarray): input array

    Returns:
        cupy.ndarray: Output array,
        containing the indices of the elements of a.ravel() that are non-zero.

    .. warning::

        This function may synchronize the device.

    .. seealso:: :func:`numpy.flatnonzero`
    """
    _util.check_array(a, arg_name='a')
    return a.ravel().nonzero()[0]

_where_ufunc = core.create_ufunc(
    'cupy_where',
    ('???->?', '?bb->b', '?BB->B', '?hh->h', '?HH->H', '?ii->i', '?II->I',
     '?ll->l', '?LL->L', '?qq->q', '?QQ->Q', '?ee->e', '?ff->f',
     # On CUDA 6.5 these combinations don't work correctly (on CUDA >=7.0, it
     # works).
     # See issue #551.
     '?hd->d', '?Hd->d',
     '?dd->d', '?FF->F', '?DD->D'),
    'out0 = in0 ? in1 : in2')

def where(condition, x=None, y=None):
    """Return elements, either from x or y, depending on condition.

    If only condition is given, return ``condition.nonzero()``.

    Args:
        condition (cupy.ndarray): When True, take x, otherwise take y.
        x (cupy.ndarray): Values from which to choose on ``True``.
        y (cupy.ndarray): Values from which to choose on ``False``.

    Returns:
        cupy.ndarray: Each element of output contains elements of ``x`` when
            ``condition`` is ``True``, otherwise elements of ``y``. If only
            ``condition`` is given, return the tuple ``condition.nonzero()``,
            the indices where ``condition`` is True.

    .. warning::

        This function may synchronize the device if both ``x`` and ``y`` are
        omitted.

    .. seealso:: :func:`numpy.where`

    """

    missing = (x is None, y is None).count(True)

    if missing == 1:
        raise ValueError('Must provide both \'x\' and \'y\' or neither.')
    if missing == 2:
        return nonzero(condition)  # may synchronize

    if fusion._is_fusing():
        return fusion._call_ufunc(_where_ufunc, condition, x, y)
    return _where_ufunc(condition.astype('?'), x, y)

def argwhere(a):
    """Return the indices of the elements that are non-zero.

    Returns a (N, ndim) dimantional array containing the
    indices of the non-zero elements. Where `N` is number of
    non-zero elements and `ndim` is dimension of the given array.

    Args:
        a (cupy.ndarray): array

    Returns:
        cupy.ndarray: Indices of elements that are non-zero.

    .. seealso:: :func:`numpy.argwhere`

    """
    _util.check_array(a, arg_name='a')
    return _indexing._ndarray_argwhere(a)

# This is to allow using the same kernels for all dtypes, ints & floats
# as nan is a special case
_preamble = '''
template<typename T>
__device__ bool _isnan(T val) {
    return val != val;
}
'''

_hip_preamble = r'''
#ifdef __HIP_DEVICE_COMPILE__
  #define no_thread_divergence(do_work, to_return) \
    if (!is_done) {                                \
      do_work;                                     \
      is_done = true;                              \
    }
#else
  #define no_thread_divergence(do_work, to_return) \
    do_work;                                       \
    if (to_return) { return; }
#endif
'''

_searchsorted_kernel = core.ElementwiseKernel(
    'S x, raw T bins, int64 n_bins, bool side_is_right, '
    'bool assume_increassing',
    'int64 y',
    '''
    #ifdef __HIP_DEVICE_COMPILE__
    bool is_done = false;
    #endif

    // Array is assumed to be monotonically
    // increasing unless a check is requested with the
    // `assume_increassing = False` parameter.
    // `digitize` allows increasing and decreasing arrays.
    bool inc = true;
    if (!assume_increassing && n_bins >= 2) {
        // In the case all the bins are nan the array is considered
        // to be decreasing in numpy
        inc = (bins[0] <= bins[n_bins-1])
              || (!_isnan<T>(bins[0]) && _isnan<T>(bins[n_bins-1]));
    }

    if (_isnan<S>(x)) {
        long long pos = (inc ? n_bins : 0);
        if (!side_is_right) {
            if (inc) {
                while (pos > 0 && _isnan<T>(bins[pos-1])) {
                    --pos;
                }
            } else {
                while (pos < n_bins && _isnan<T>(bins[pos])) {
                    ++pos;
                }
            }
        }
        no_thread_divergence( y = pos , true )
    }

    bool greater = false;
    if (side_is_right) {
        greater = inc && x >= bins[n_bins-1];
    } else {
        greater = (inc ? x > bins[n_bins-1] : x <= bins[n_bins-1]);
    }
    if (greater) {
        no_thread_divergence( y = n_bins , true )
    }

    long long left = 0;
    // In the case the bins is all NaNs, digitize
    // needs to place all the valid values to the right
    if (!inc) {
        while (_isnan<T>(bins[left]) && left < n_bins) {
            ++left;
        }
        if (left == n_bins) {
            no_thread_divergence( y = n_bins , true )
        }
        if (side_is_right
                && !_isnan<T>(bins[n_bins-1]) && !_isnan<S>(x)
                && bins[n_bins-1] > x) {
            no_thread_divergence( y = n_bins , true )
        }
    }

    long long right = n_bins-1;
    while (left < right) {
        long long m = left + (right - left) / 2;
        bool look_right = true;
        if (side_is_right) {
            look_right = (inc ? bins[m] <= x : bins[m] > x);
        } else {
            look_right = (inc ? bins[m] < x : bins[m] >= x);
        }
        if (look_right) {
            left = m + 1;
        } else {
            right = m;
        }
    }
    no_thread_divergence( y = right , false )
    ''', preamble=_preamble+_hip_preamble)

def searchsorted(a, v, side='left', sorter=None):
    """Finds indices where elements should be inserted to maintain order.

    Find the indices into a sorted array ``a`` such that,
    if the corresponding elements in ``v`` were inserted before the indices,
    the order of ``a`` would be preserved.

    Args:
        a (cupy.ndarray): Input array. If ``sorter`` is ``None``, then
            it must be sorted in ascending order,
            otherwise ``sorter`` must be an array of indices that sort it.
        v (cupy.ndarray): Values to insert into ``a``.
        side : {'left', 'right'}
            If ``left``, return the index of the first suitable location found
            If ``right``, return the last such index.
            If there is no suitable index, return either 0 or length of ``a``.
        sorter : 1-D array_like
            Optional array of integer indices that sort array ``a`` into
            ascending order. They are typically the result of
            :func:`~cupy.argsort`.

    Returns:
        cupy.ndarray: Array of insertion points with the same shape as ``v``.

    .. note:: When a is not in ascending order, behavior is undefined.

    .. seealso:: :func:`numpy.searchsorted`

    """
    return _searchsorted(a, v, side, sorter, True)

def _searchsorted(a, v, side, sorter, assume_increasing):
    """`assume_increasing` is used in the kernel to
    skip monotonically increasing or decreasing verification
    inside the cuda kernel.
    """
    if not isinstance(a, cupy.ndarray):
        raise NotImplementedError('Only int or ndarray are supported for a')

    if not isinstance(v, cupy.ndarray):
        raise NotImplementedError('Only int or ndarray are supported for v')

    if a.ndim > 1:
        raise ValueError('object too deep for desired array')
    if a.ndim < 1:
        raise ValueError('object of too small depth for desired array')
    if a.size == 0:
        return cupy.zeros(v.shape, dtype=cupy.int64)

    a_iscomplex = a.dtype.kind == 'c'
    v_iscomplex = v.dtype.kind == 'c'

    if a_iscomplex and not v_iscomplex:
        v = v.astype(a.dtype)
    elif v_iscomplex and not a_iscomplex:
        a = a.astype(v.dtype)

    # Numpy does not check if the array is monotonic inside searchsorted
    # which leads to undefined behavior in such cases.
    if sorter is not None:
        if sorter.dtype.kind not in ('i', 'u'):
            raise TypeError('sorter must be of integer type')
        if sorter.size != a.size:
            raise ValueError('sorter.size must equal a.size')
        a = a.take(sorter)

    y = cupy.zeros(v.shape, dtype=cupy.int64)

    _searchsorted_kernel(v, a, a.size, side == 'right', assume_increasing, y)
    return y

# TODO(okuta): Implement extract
<|endoftext|>


<commit_before># For making Tools for scheduling app

import sys
import inspect

from .models import Reservation

# Function calls for pseudo-model

def get_tool_choices():
    tool_choices = []
    for name, obj in inspect.getmembers(sys.modules[__name__]):
        if inspect.isclass(obj):
            tool_choices.append((obj.get_slug(), obj.get_name()))
    return tuple(tool_choices)

def get_tool_list():
    tool_list = []
    for name, obj in inspect.getmembers(sys.modules[__name__]):
        if inspect.isclass(obj):
            tool_list.append(obj.get_slug())
    return tuple(tool_list)

# custom attributes for tools
tool = {'d180': {'max_reservations': 5}, 'd75': {'max_reservations': 3}}

def get_tool_info(slug):
    return tool[slug]['max_reservations']

# Tool definitions

class d180(object):

    @staticmethod
    def get_name():
        return 'D180'

    @staticmethod
    def get_slug():
        return 'd180'

class d75(object):

    @staticmethod
    def get_name():
        return 'D75'

    @staticmethod
    def get_slug():
        return 'd75'
<commit_msg>Fixed circular import<commit_after># For making Tools for scheduling app

import sys
import inspect

# Function calls for pseudo-model

def get_tool_choices():
    tool_choices = []
    for name, obj in inspect.getmembers(sys.modules[__name__]):
        if inspect.isclass(obj):
            tool_choices.append((obj.get_slug(), obj.get_name()))
    return tuple(tool_choices)

def get_tool_list():
    tool_list = []
    for name, obj in inspect.getmembers(sys.modules[__name__]):
        if inspect.isclass(obj):
            tool_list.append(obj.get_slug())
    return tuple(tool_list)

# custom attributes for tools
tool = {'d180': {'max_reservations': 5}, 'd75': {'max_reservations': 3}}

def get_tool_info(slug):
    return tool[slug]['max_reservations']

# Tool definitions

class d180(object):

    @staticmethod
    def get_name():
        return 'D180'

    @staticmethod
    def get_slug():
        return 'd180'

class d75(object):

    @staticmethod
    def get_name():
        return 'D75'

    @staticmethod
    def get_slug():
        return 'd75'
<|endoftext|>


<commit_before>__version__ = '0.9.0'
<commit_msg>back to dev<commit_after>__version__ = '0.10.0.dev'
<|endoftext|>


<commit_before>import boto3
from botocore.exceptions import ClientError
import logging

from ..instancemanager.instancemanager import _get_own_ip
from ..instancemanager.instancemanager import InstanceManager

logger = logging.getLogger()

class AWSTools(InstanceManager):
    """The AWSTools class provides an abstraction over boto3 and EC2 for the use with CSAOpt

    It is intended to be used as a context manager, disposing of instances in it's __exit__()
    call.

    Create IAM credentials:
        * new IAM user with programmatic access
        * assign to a (potentially new) group with AmazonEC2FullAccess
        * write down and store the access key and secret key

    Boto3 will check these environment variables for credentials:

    Note:
        If the AWS credentials are not provided in the config file, boto3 will look into
        the following environment variables:
        * AWS_ACCESS_KEY_ID
        * AWS_SECRET_ACCESS_KEY

    """
    def __init__(self, config, internal_conf):
        if config['aws.secret_key'] and config['aws.access_key']:
            self.ec2 = boto3.client(
                aws_access_key_id=config['aws.access_key'],
                aws_secret_access_key=config['aws.secrey_key']
            )
        else:
            # This will look for the env variables
            self.ec2 = boto3.client('ec2')
        self.region
        self.instances = []
        self.sec_group = None
        self.private_key_path = None
        self.worker_count = config['aws.worker_count']
        self.separate_queue_instance = config['aws.separate_queue_instance']

    def __enter__(self):
        """On enter, AWSTools prepares the AWS security group, key pair and spins up the required intances"""
        self._create_sec_group()
        self._create_key_pair()
        instances = self._start_worker_instances()
        if self.separate_queue_instance:
            instance = self._start_queue_instance()
            instances.append(instance)

        self.instances = instances

    def __exit__(self, exc_type, exc_value, traceback):
        """On exit, AWSTools terminates the started instances and removes security groups"""
        instance_ids = [instance.id for instance in self.instances]
        self.ec2.instances.filter(InstanceIds=instance_ids).stop()
        self.ec2.instances.filter(InstanceIds=instance_ids).terminate()

    def _start_instances(self, count=2, imageId='ami-1e299d7e', instanceType='t2.micro'):
        """Start a number of ec2 instances"""
        self.instances = self.ec2.create_instances(
                ImageId=imageId,
                MinCount=count,
                MaxCount=count,
                InstanceType=instanceType)

    def _terminate_instances(self):
        """Terminate all instances managed by AWSTools"""
        for instance in self.instances:
            instance = self.ec2.Instance(instance.id)
            response = instance.terminate()
            print(response)

    def get_running_instances(self):
        return self.instances

    def _create_key_pair(self, key_name):
        """Creates a key pair for communitcation with the EC2 instances. TODO: this might be unnecessary"""
        key = self.ec2.create_key_pair(KeyName=key_name)
        return key

    def _save_key_material(self, key):
        pass

    def _remove_key(self):
        pass

    def _remove_sec_group(self):
        """Removes the security group created by CSAOpt"""

        if self.security_group_id is not None:
            try:
                self.ec2.delete_security_group(GroupId=self.security_group_id)
                logger.debug('Security Group Deleted')
            except ClientError as e:
                logger.error('Could not remove security group: {}'.format(e))
        else:
            logger.warn('Cannot remove security group, because none was created. Skipping')

    def _create_sec_group(self, name):
        self.own_external_ip = _get_own_ip()
        response = self.ec2.describe_vpcs()
        vpc_id = response.get('Vpcs', [{}])[0].get('VpcId', '')

        try:
            response = self.ec2.create_security_group(
                GroupName=name,
                Description='Security Group for CSAOpt',
                VpcId=vpc_id)

            self.security_group_id = response['GroupId']

            data = self.ec2.authorize_security_group_ingress(
                GroupId=self.security_group_id,
                IpPermissions=[  # TODO: 80 and 22 are not needed here. Rather, these should be the zmq ports
                    {'IpProtocol': 'tcp',
                     'FromPort': 80,
                     'ToPort': 80,
                     'IpRanges': [{'CidrIp': '{}/0'.format(self.own_external_ip)}]},
                    {'IpProtocol': 'tcp',
                     'FromPort': 22,
                     'ToPort': 22,
                     'IpRanges': [{'CidrIp': '{}/0'.format(self.own_external_ip)}]}
                ])
            logger.debug('Authorized Security Group Ingress for CidrIp {} with result: {}'.format(
                self.own_external_ip,
                data
                ))
        except ClientError as e:
            logger.error('Could not create Security Group: {}', e)
<commit_msg>Switching to docker: no need to create key pairs<commit_after>import boto3
from botocore.exceptions import ClientError
import logging

from ..instancemanager.instancemanager import _get_own_ip
from ..instancemanager.instancemanager import InstanceManager

logger = logging.getLogger()

class AWSTools(InstanceManager):
    """The AWSTools class provides an abstraction over boto3 and EC2 for the use with CSAOpt

    It is intended to be used as a context manager, disposing of instances in it's __exit__()
    call.

    Create IAM credentials:
        * new IAM user with programmatic access
        * assign to a (potentially new) group with AmazonEC2FullAccess
        * write down and store the access key and secret key

    Boto3 will check these environment variables for credentials:

    Note:
        If the AWS credentials are not provided in the config file, boto3 will look into
        the following environment variables:
        * AWS_ACCESS_KEY_ID
        * AWS_SECRET_ACCESS_KEY

    """
    def __init__(self, config, internal_conf):
        if config['aws.secret_key'] and config['aws.access_key']:
            self.ec2 = boto3.client(
                aws_access_key_id=config['aws.access_key'],
                aws_secret_access_key=config['aws.secrey_key']
            )
        else:
            # This will look for the env variables
            self.ec2 = boto3.client('ec2')
        self.region
        self.instances = []
        self.sec_group = None
        self.private_key_path = None
        self.worker_count = config['aws.worker_count']
        self.separate_queue_instance = config['aws.separate_queue_instance']

    def __enter__(self):
        """On enter, AWSTools prepares the AWS security group, key pair and spins up the required intances"""
        self._create_sec_group()
        instances = self._start_worker_instances()
        if self.separate_queue_instance:
            instance = self._start_queue_instance()
            instances.append(instance)

        self.instances = instances

    def __exit__(self, exc_type, exc_value, traceback):
        """On exit, AWSTools terminates the started instances and removes security groups"""
        instance_ids = [instance.id for instance in self.instances]
        self.ec2.instances.filter(InstanceIds=instance_ids).stop()
        self.ec2.instances.filter(InstanceIds=instance_ids).terminate()

    def _start_instances(self, count=2, imageId='ami-1e299d7e', instanceType='t2.micro'):
        """Start a number of ec2 instances"""
        self.instances = self.ec2.create_instances(
                ImageId=imageId,
                MinCount=count,
                MaxCount=count,
                InstanceType=instanceType)

    def _terminate_instances(self):
        """Terminate all instances managed by AWSTools"""
        for instance in self.instances:
            instance = self.ec2.Instance(instance.id)
            response = instance.terminate()
            print(response)

    def get_running_instances(self):
        return self.instances

    def _remove_sec_group(self):
        """Removes the security group created by CSAOpt"""

        if self.security_group_id is not None:
            try:
                self.ec2.delete_security_group(GroupId=self.security_group_id)
                logger.debug('Security Group Deleted')
            except ClientError as e:
                logger.error('Could not remove security group: {}'.format(e))
        else:
            logger.warn('Cannot remove security group, because none was created. Skipping')

    def _create_sec_group(self, name):
        self.own_external_ip = _get_own_ip()
        response = self.ec2.describe_vpcs()
        vpc_id = response.get('Vpcs', [{}])[0].get('VpcId', '')

        try:
            response = self.ec2.create_security_group(
                GroupName=name,
                Description='Security Group for CSAOpt',
                VpcId=vpc_id)

            self.security_group_id = response['GroupId']

            data = self.ec2.authorize_security_group_ingress(
                GroupId=self.security_group_id,
                IpPermissions=[  # TODO: 80 and 22 are not needed here. Rather, these should be the zmq ports
                    {'IpProtocol': 'tcp',
                     'FromPort': 80,
                     'ToPort': 80,
                     'IpRanges': [{'CidrIp': '{}/0'.format(self.own_external_ip)}]},
                    {'IpProtocol': 'tcp',
                     'FromPort': 22,
                     'ToPort': 22,
                     'IpRanges': [{'CidrIp': '{}/0'.format(self.own_external_ip)}]}
                ])
            logger.debug('Authorized Security Group Ingress for CidrIp {} with result: {}'.format(
                self.own_external_ip,
                data
                ))
        except ClientError as e:
            logger.error('Could not create Security Group: {}', e)
<|endoftext|>


<commit_before>import os.path
import uuid
from contextlib import contextmanager
from csv import DictReader
from datetime import datetime, timedelta
from itertools import chain
from unittest.mock import patch

from testil import tempdir

from corehq.apps.auditcare.models import AccessAudit, NavigationEventAudit

from ..utils.export import (
    AuditWindowQuery,
    ForeignKeyAccessError,
    get_all_log_events,
    get_domain_first_access_times,
    get_foreign_names,
    get_sql_start_date,
    navigation_events_by_user,
    write_export_from_all_log_events,
)
from .testutils import AuditcareTest

class TestNavigationEventsQueries(AuditcareTest):

    maxDiff = None
    username = "test@test.com"

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        def iter_events(model, username, **kw):
            for event_date in cls.event_dates:
                yield model(user=username, event_date=event_date, **kw)

        cls.event_dates = [datetime(2021, 2, d, 3) for d in range(1, 29)]
        headers = {"REQUEST_METHOD": "GET"}
        NavigationEventAudit.objects.bulk_create(chain(
            iter_events(NavigationEventAudit, cls.username, headers=headers),
            iter_events(NavigationEventAudit, "other@test.com", headers=headers)
        ))
        AccessAudit.objects.bulk_create(chain(
            iter_events(AccessAudit, cls.username, access_type="o"),
            iter_events(AccessAudit, "other@test.com", access_type="i")
        ))

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()

    def test_navigation_events_query(self):
        events = list(navigation_events_by_user(self.username))
        self.assertEqual({e.user for e in events}, {self.username})
        expected_dates = self.event_dates
        self.assertEqual([e.event_date for e in events], expected_dates)
        self.assertEqual(len(events), 28)

    def test_navigation_events_date_range_query(self):
        start = datetime(2021, 2, 5)
        end = datetime(2021, 2, 15)
        events = list(navigation_events_by_user(self.username, start, end))
        self.assertEqual({e.user for e in events}, {self.username})
        self.assertEqual({e.event_date for e in events}, set(self.event_dates[4:15]))
        self.assertEqual(len(events), 11)

    def test_recent_all_log_events_query(self):
        start = datetime(2021, 2, 4)
        events = list(get_all_log_events(start))
        self.assertEqual(len(events), 100)
        self.assertEqual({e.user for e in events}, {self.username, "other@test.com"})
        self.assertEqual({e.doc_type for e in events}, {"AccessAudit", "NavigationEventAudit"})
        self.assertEqual({e.event_date for e in events}, set(self.event_dates[3:]))

    def test_all_log_events_date_range_query(self):
        start = datetime(2021, 2, 5)
        end = datetime(2021, 2, 15)
        events = list(get_all_log_events(start, end))
        self.assertEqual({e.user for e in events}, {self.username, "other@test.com"})
        self.assertEqual({e.doc_type for e in events}, {"AccessAudit", "NavigationEventAudit"})
        self.assertEqual({e.event_date for e in events}, set(self.event_dates[4:15]))
        self.assertEqual(len(events), 44)

    def test_all_log_events_query_returns_events_from_couch(self):
        start = datetime(2021, 2, 1)
        end = datetime(2021, 2, 1)
        events = list(get_all_log_events(start, end))
        self.assertEqual({e.user for e in events}, {self.username, "other@test.com"})
        self.assertEqual({e.doc_type for e in events}, {"AccessAudit", "NavigationEventAudit"})
        self.assertEqual({e.event_date for e in events}, {datetime(2021, 2, 1, 3)})
        self.assertEqual(len(events), 4)

    def test_write_export_from_all_log_events(self):
        def unpack(row):
            return {k: row[k] for k in ["Date", "Type", "User", "Description"]}
        start = datetime(2021, 2, 1)
        end = datetime(2021, 2, 1)
        with tempdir() as tmp:
            filename = os.path.join(tmp, "events.csv")
            with open(filename, 'w', encoding="utf-8") as csvfile:
                write_export_from_all_log_events(csvfile, start=start, end=end)
            with open(filename, encoding="utf-8") as fh:
                def key(row):
                    return row["Date"], row["Type"], row["User"]
                rows = DictReader(fh)
                items = sorted([unpack(r) for r in rows], key=key)
                expected_items = [
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'AccessAudit',
                        'User': 'other@test.com',
                        'Description': 'Login: other@test.com',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'AccessAudit',
                        'User': 'test@test.com',
                        'Description': 'Logout: test@test.com',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'NavigationEventAudit',
                        'User': 'other@test.com',
                        'Description': 'other@test.com',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'NavigationEventAudit',
                        'User': 'test@test.com',
                        'Description': 'test@test.com',
                    },
                ]
                self.assertEqual(items, expected_items)

    def test_query_window_size(self):
        # NOTE small window size ensures multiple queries per event date
        with patch_window_size(1):
            events = list(get_all_log_events(datetime(2021, 2, 4)))
        self.assertEqual(len(events), 100)

    def test_related_query_error(self):
        event = next(iter(navigation_events_by_user('other@test.com')))
        keys = get_foreign_names(NavigationEventAudit)
        self.assertIn("user_agent", keys)
        self.assertIn("user_agent_fk", keys)
        for key in keys:
            with self.assertRaises(ForeignKeyAccessError):
                getattr(event, key)

    def test_get_sql_start_date(self):
        self.assertEqual(get_sql_start_date(), datetime(2021, 2, 1, 3))

    def test_get_domain_first_access_times(self):

        def create_session_events(domain):
            login_event = dict(
                user=self.username,
                domain=domain,
                session_key=uuid.uuid4().hex,
                event_date=datetime.utcnow(),
            )
            for event_domain, minutes in [(None, -1),
                                          (domain, 0),
                                          (domain, 1)]:
                fields = login_event.copy()
                # save one event with domain/date changed by loop params:
                fields["domain"] = event_domain
                fields['event_date'] += timedelta(minutes=minutes)
                NavigationEventAudit(**fields).save()
                # update the fields and save another:
                #   - same session
                #   - different domain
                #   - 5 minutes ealier
                fields["domain"] = "not-queried"
                fields["event_date"] += timedelta(minutes=-5)
                NavigationEventAudit(**fields).save()
            return login_event

        domain = "delmar"
        login_events = []
        for x in range(2):
            login_event = create_session_events(domain)
            # rename `event_date` to `access_time` (name of aggregation field)
            login_event["access_time"] = login_event.pop("event_date")
            login_events.append(login_event)
            # remove the session key (not returned by the query)
            del login_event["session_key"]
        self.assertEqual(list(get_domain_first_access_times([domain])), login_events)

class TestNavigationEventsQueriesWithoutData(AuditcareTest):

    def test_get_all_log_events_returns_empty(self):
        start = end = datetime.utcnow()
        self.assertEqual(list(get_all_log_events(start, end)), [])

    def test_get_sql_start_date_returns_datetime(self):
        self.assertIsInstance(get_sql_start_date(), datetime)

@contextmanager
def patch_window_size(size):
    with patch.object(AuditWindowQuery.__init__, "__defaults__", (size,)):
        qry = AuditWindowQuery("ignored")
        assert qry.window_size == size, f"patch failed ({qry.window_size})"
        yield
<commit_msg>fix: tests<commit_after>import os.path
import uuid
from contextlib import contextmanager
from csv import DictReader
from datetime import datetime, timedelta
from itertools import chain
from unittest.mock import patch

from testil import tempdir

from corehq.apps.auditcare.models import AccessAudit, NavigationEventAudit
from corehq.apps.auditcare.utils.migration import get_sql_start_date

from ..utils.export import (
    AuditWindowQuery,
    ForeignKeyAccessError,
    get_all_log_events,
    get_domain_first_access_times,
    get_foreign_names,
    navigation_events_by_user,
    write_export_from_all_log_events,
)
from .testutils import AuditcareTest

class TestNavigationEventsQueries(AuditcareTest):

    maxDiff = None
    username = "test@test.com"

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        def iter_events(model, username, **kw):
            for event_date in cls.event_dates:
                yield model(user=username, event_date=event_date, **kw)

        cls.event_dates = [datetime(2021, 2, d, 3) for d in range(1, 29)]
        headers = {"REQUEST_METHOD": "GET"}
        NavigationEventAudit.objects.bulk_create(chain(
            iter_events(NavigationEventAudit, cls.username, headers=headers),
            iter_events(NavigationEventAudit, "other@test.com", headers=headers)
        ))
        AccessAudit.objects.bulk_create(chain(
            iter_events(AccessAudit, cls.username, access_type="o"),
            iter_events(AccessAudit, "other@test.com", access_type="i")
        ))

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()

    def test_navigation_events_query(self):
        events = list(navigation_events_by_user(self.username))
        self.assertEqual({e.user for e in events}, {self.username})
        expected_dates = self.event_dates
        self.assertEqual([e.event_date for e in events], expected_dates)
        self.assertEqual(len(events), 28)

    def test_navigation_events_date_range_query(self):
        start = datetime(2021, 2, 5)
        end = datetime(2021, 2, 15)
        events = list(navigation_events_by_user(self.username, start, end))
        self.assertEqual({e.user for e in events}, {self.username})
        self.assertEqual({e.event_date for e in events}, set(self.event_dates[4:15]))
        self.assertEqual(len(events), 11)

    def test_recent_all_log_events_query(self):
        start = datetime(2021, 2, 4)
        events = list(get_all_log_events(start))
        self.assertEqual(len(events), 100)
        self.assertEqual({e.user for e in events}, {self.username, "other@test.com"})
        self.assertEqual({e.doc_type for e in events}, {"AccessAudit", "NavigationEventAudit"})
        self.assertEqual({e.event_date for e in events}, set(self.event_dates[3:]))

    def test_all_log_events_date_range_query(self):
        start = datetime(2021, 2, 5)
        end = datetime(2021, 2, 15)
        events = list(get_all_log_events(start, end))
        self.assertEqual({e.user for e in events}, {self.username, "other@test.com"})
        self.assertEqual({e.doc_type for e in events}, {"AccessAudit", "NavigationEventAudit"})
        self.assertEqual({e.event_date for e in events}, set(self.event_dates[4:15]))
        self.assertEqual(len(events), 44)

    def test_all_log_events_query_returns_events_from_couch(self):
        start = datetime(2021, 2, 1)
        end = datetime(2021, 2, 1)
        events = list(get_all_log_events(start, end))
        self.assertEqual({e.user for e in events}, {self.username, "other@test.com"})
        self.assertEqual({e.doc_type for e in events}, {"AccessAudit", "NavigationEventAudit"})
        self.assertEqual({e.event_date for e in events}, {datetime(2021, 2, 1, 3)})
        self.assertEqual(len(events), 4)

    def test_write_export_from_all_log_events(self):
        def unpack(row):
            return {k: row[k] for k in ["Date", "Type", "User", "Description"]}
        start = datetime(2021, 2, 1)
        end = datetime(2021, 2, 1)
        with tempdir() as tmp:
            filename = os.path.join(tmp, "events.csv")
            with open(filename, 'w', encoding="utf-8") as csvfile:
                write_export_from_all_log_events(csvfile, start=start, end=end)
            with open(filename, encoding="utf-8") as fh:
                def key(row):
                    return row["Date"], row["Type"], row["User"]
                rows = DictReader(fh)
                items = sorted([unpack(r) for r in rows], key=key)
                expected_items = [
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'AccessAudit',
                        'User': 'other@test.com',
                        'Description': 'Login: other@test.com',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'AccessAudit',
                        'User': 'test@test.com',
                        'Description': 'Logout: test@test.com',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'NavigationEventAudit',
                        'User': 'other@test.com',
                        'Description': 'other@test.com',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'NavigationEventAudit',
                        'User': 'test@test.com',
                        'Description': 'test@test.com',
                    },
                ]
                self.assertEqual(items, expected_items)

    def test_query_window_size(self):
        # NOTE small window size ensures multiple queries per event date
        with patch_window_size(1):
            events = list(get_all_log_events(datetime(2021, 2, 4)))
        self.assertEqual(len(events), 100)

    def test_related_query_error(self):
        event = next(iter(navigation_events_by_user('other@test.com')))
        keys = get_foreign_names(NavigationEventAudit)
        self.assertIn("user_agent", keys)
        self.assertIn("user_agent_fk", keys)
        for key in keys:
            with self.assertRaises(ForeignKeyAccessError):
                getattr(event, key)

    def test_get_sql_start_date(self):
        self.assertEqual(get_sql_start_date(), datetime(2021, 2, 1, 3))

    def test_get_domain_first_access_times(self):

        def create_session_events(domain):
            login_event = dict(
                user=self.username,
                domain=domain,
                session_key=uuid.uuid4().hex,
                event_date=datetime.utcnow(),
            )
            for event_domain, minutes in [(None, -1),
                                          (domain, 0),
                                          (domain, 1)]:
                fields = login_event.copy()
                # save one event with domain/date changed by loop params:
                fields["domain"] = event_domain
                fields['event_date'] += timedelta(minutes=minutes)
                NavigationEventAudit(**fields).save()
                # update the fields and save another:
                #   - same session
                #   - different domain
                #   - 5 minutes ealier
                fields["domain"] = "not-queried"
                fields["event_date"] += timedelta(minutes=-5)
                NavigationEventAudit(**fields).save()
            return login_event

        domain = "delmar"
        login_events = []
        for x in range(2):
            login_event = create_session_events(domain)
            # rename `event_date` to `access_time` (name of aggregation field)
            login_event["access_time"] = login_event.pop("event_date")
            login_events.append(login_event)
            # remove the session key (not returned by the query)
            del login_event["session_key"]
        self.assertEqual(list(get_domain_first_access_times([domain])), login_events)

class TestNavigationEventsQueriesWithoutData(AuditcareTest):

    def test_get_all_log_events_returns_empty(self):
        start = end = datetime.utcnow()
        self.assertEqual(list(get_all_log_events(start, end)), [])

    def test_get_sql_start_date_returns_datetime(self):
        self.assertIsInstance(get_sql_start_date(), datetime)

@contextmanager
def patch_window_size(size):
    with patch.object(AuditWindowQuery.__init__, "__defaults__", (size,)):
        qry = AuditWindowQuery("ignored")
        assert qry.window_size == size, f"patch failed ({qry.window_size})"
        yield
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Display public IP address and online status.

Configuration parameters:
    cache_timeout: how often we refresh this module in seconds (default 30)
    expected: define expected values for format placeholders,
        and use `color_degraded` to show the output of this module
        if any of them does not match the actual value.
        This should be a dict eg {'country': 'France'}
        (default None)
    format: available placeholders are {ip} and {country},
        as well as any other key in JSON fetched from `url_geo`
        (default '{ip}')
    hide_when_offline: hide the module output when offline (default False)
    icon_off: what to display when offline (default '■')
    icon_on: what to display when online (default '●')
    mode: default mode to display is 'ip' or 'status' (click to toggle)
        (default 'ip')
    negative_cache_timeout: how often to check again when offline (default 2)
    timeout: how long before deciding we're offline (default 5)
    url_geo: IP to check for geo location (must output json)
        (default 'https://freegeoip.net/json/')

Format placeholders:
    {icon} display the icon
    {country} display the country
    {ip} display current ip address
    any other key in JSON fetched from `url_geo`

Color options:
    color_bad: Offline
    color_degraded: Output is unexpected (IP/country mismatch, etc.)
    color_good: Online

@author ultrabug

SAMPLE OUTPUT
{'full_text': '37.48.108.0'}

geo
{'full_text': '37.48.108.0 Russia'}

mode
{'color': '#00FF00', 'full_text': u'\u25cf'}
"""

URL_GEO_OLD_DEFAULT = 'http://ip-api.com/json'
URL_GEO_NEW_DEFAULT = 'https://freegeoip.net/json/'

class Py3status:
    """
    """
    # available configuration parameters
    cache_timeout = 30
    expected = None
    format = '{ip}'
    hide_when_offline = False
    icon_off = u'■'
    icon_on = u'●'
    mode = 'ip'
    negative_cache_timeout = 2
    timeout = 5
    url_geo = URL_GEO_NEW_DEFAULT

    class Meta:
        deprecated = {
            'remove': [
                {
                    'param': 'url',
                    'msg': 'obsolete parameter, use `url_geo` instead',
                },
            ],
            'rename': [
                {
                    'param': 'format_online',
                    'new': 'icon_on',
                    'msg': 'obsolete parameter, use `icon_on` instead',
                },
                {
                    'param': 'format_offline',
                    'new': 'icon_off',
                    'msg': 'obsolete parameter, use `icon_off` instead',
                },
            ],
        }

    def post_config_hook(self):
        if self.expected is None:
            self.expected = {}

        # Backwards compatibility
        self.substitutions = {}
        if self.url_geo == URL_GEO_NEW_DEFAULT:
            self.substitutions['country'] = 'country_name'
        elif self.url_geo == URL_GEO_OLD_DEFAULT:
            self.substitutions['ip'] = 'query'

    def on_click(self, event):
        """
        Toggle between display modes 'ip' and 'status'
        """
        if self.mode == 'ip':
            self.mode = 'status'
        else:
            self.mode = 'ip'

    def _get_my_ip_info(self):
        """
        """
        try:
            info = self.py3.request(self.url_geo, timeout=self.timeout).json()
            for old, new in self.substitutions.items():
                info[old] = info.get(new)
            return info
        except self.py3.RequestException:
            return None

    def whatismyip(self):
        """
        """
        info = self._get_my_ip_info()
        response = {
            'cached_until': self.py3.time_in(self.negative_cache_timeout)
        }

        if info is None and self.hide_when_offline:
            response['full_text'] = ''
        elif info is not None:
            info['icon'] = self.icon_on
            response['cached_until'] = self.py3.time_in(self.cache_timeout)
            response['color'] = self.py3.COLOR_GOOD
            if self.mode == 'ip':
                response['full_text'] = self.py3.safe_format(self.format, info)
                for key, val in self.expected.items():
                    if val != info.get(key):
                        response['color'] = self.py3.COLOR_DEGRADED
                        break
            else:
                response['full_text'] = self.icon_on
        else:
            response['full_text'] = self.icon_off
            response['color'] = self.py3.COLOR_BAD
        return response

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test
    module_test(Py3status)
<commit_msg>color_degraded support in status mode for whatismyip module<commit_after># -*- coding: utf-8 -*-
"""
Display public IP address and online status.

Configuration parameters:
    cache_timeout: how often we refresh this module in seconds (default 30)
    expected: define expected values for format placeholders,
        and use `color_degraded` to show the output of this module
        if any of them does not match the actual value.
        This should be a dict eg {'country': 'France'}
        (default None)
    format: available placeholders are {ip} and {country},
        as well as any other key in JSON fetched from `url_geo`
        (default '{ip}')
    hide_when_offline: hide the module output when offline (default False)
    icon_off: what to display when offline (default '■')
    icon_on: what to display when online (default '●')
    mode: default mode to display is 'ip' or 'status' (click to toggle)
        (default 'ip')
    negative_cache_timeout: how often to check again when offline (default 2)
    timeout: how long before deciding we're offline (default 5)
    url_geo: IP to check for geo location (must output json)
        (default 'https://freegeoip.net/json/')

Format placeholders:
    {icon} display the icon
    {country} display the country
    {ip} display current ip address
    any other key in JSON fetched from `url_geo`

Color options:
    color_bad: Offline
    color_degraded: Output is unexpected (IP/country mismatch, etc.)
    color_good: Online

@author ultrabug

SAMPLE OUTPUT
{'full_text': '37.48.108.0'}

geo
{'full_text': '37.48.108.0 Russia'}

mode
{'color': '#00FF00', 'full_text': u'\u25cf'}
"""

URL_GEO_OLD_DEFAULT = 'http://ip-api.com/json'
URL_GEO_NEW_DEFAULT = 'https://freegeoip.net/json/'

class Py3status:
    """
    """
    # available configuration parameters
    cache_timeout = 30
    expected = None
    format = '{ip}'
    hide_when_offline = False
    icon_off = u'■'
    icon_on = u'●'
    mode = 'ip'
    negative_cache_timeout = 2
    timeout = 5
    url_geo = URL_GEO_NEW_DEFAULT

    class Meta:
        deprecated = {
            'remove': [
                {
                    'param': 'url',
                    'msg': 'obsolete parameter, use `url_geo` instead',
                },
            ],
            'rename': [
                {
                    'param': 'format_online',
                    'new': 'icon_on',
                    'msg': 'obsolete parameter, use `icon_on` instead',
                },
                {
                    'param': 'format_offline',
                    'new': 'icon_off',
                    'msg': 'obsolete parameter, use `icon_off` instead',
                },
            ],
        }

    def post_config_hook(self):
        if self.expected is None:
            self.expected = {}

        # Backwards compatibility
        self.substitutions = {}
        if self.url_geo == URL_GEO_NEW_DEFAULT:
            self.substitutions['country'] = 'country_name'
        elif self.url_geo == URL_GEO_OLD_DEFAULT:
            self.substitutions['ip'] = 'query'

    def on_click(self, event):
        """
        Toggle between display modes 'ip' and 'status'
        """
        if self.mode == 'ip':
            self.mode = 'status'
        else:
            self.mode = 'ip'

    def _get_my_ip_info(self):
        """
        """
        try:
            info = self.py3.request(self.url_geo, timeout=self.timeout).json()
            for old, new in self.substitutions.items():
                info[old] = info.get(new)
            return info
        except self.py3.RequestException:
            return None

    def whatismyip(self):
        """
        """
        info = self._get_my_ip_info()
        response = {
            'cached_until': self.py3.time_in(self.negative_cache_timeout)
        }

        if info is None and self.hide_when_offline:
            response['full_text'] = ''
        elif info is not None:
            info['icon'] = self.icon_on
            response['cached_until'] = self.py3.time_in(self.cache_timeout)
            response['color'] = self.py3.COLOR_GOOD
            if self.mode == 'ip':
                response['full_text'] = self.py3.safe_format(self.format, info)
                for key, val in self.expected.items():
                    if val != info.get(key):
                        response['color'] = self.py3.COLOR_DEGRADED
                        break
            else:
                response['full_text'] = self.icon_on
                for key, val in self.expected.items():
                    if val != info.get(key):
                        response['color'] = self.py3.COLOR_DEGRADED
                        break
        else:
            response['full_text'] = self.icon_off
            response['color'] = self.py3.COLOR_BAD
        return response

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test
    module_test(Py3status)
<|endoftext|>


<commit_before>import numpy

from cupy.random import distributions
from cupy.random import generator
from cupy.random import sample

rand = sample.rand
randn = sample.randn
random_sample = sample.random_sample
random = random_sample
ranf = random_sample
sample = random_sample
bytes = numpy.random.bytes

lognormal = distributions.lognormal
normal = distributions.normal
standard_normal = distributions.standard_normal
uniform = distributions.uniform

RandomState = generator.RandomState
get_random_state = generator.get_random_state
seed = generator.seed
reset_states = generator.reset_states
<commit_msg>Expose cupy.random.sample<commit_after>import numpy

from cupy.random import distributions
from cupy.random import generator
from cupy.random import sample as sample_

rand = sample_.rand
randn = sample_.randn
random_sample = sample_.random_sample
random = random_sample
ranf = random_sample
sample = random_sample
bytes = numpy.random.bytes

lognormal = distributions.lognormal
normal = distributions.normal
standard_normal = distributions.standard_normal
uniform = distributions.uniform

RandomState = generator.RandomState
get_random_state = generator.get_random_state
seed = generator.seed
reset_states = generator.reset_states
<|endoftext|>