<commit_before>import logging
import random

import bson
from django.contrib.contenttypes.fields import (GenericForeignKey,
                                                GenericRelation)
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import MultipleObjectsReturned
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import models
from django.db.models import ForeignKey
from django.db.models.signals import post_save
from django.dispatch import receiver
from django_extensions.db.models import TimeStampedModel
from include import IncludeQuerySet

from osf.utils.caching import cached_property
from osf.exceptions import ValidationError
from osf.utils.fields import LowercaseCharField, NonNaiveDateTimeField

ALPHABET = '23456789abcdefghjkmnpqrstuvwxyz'

logger = logging.getLogger(__name__)

def generate_guid(length=5):
    while True:
        guid_id = ''.join(random.sample(ALPHABET, length))

        try:
            # is the guid in the blacklist
            BlackListGuid.objects.get(guid=guid_id)
        except BlackListGuid.DoesNotExist:
            # it's not, check and see if it's already in the database
            try:
                Guid.objects.get(_id=guid_id)
            except Guid.DoesNotExist:
                # valid and unique guid
                return guid_id

def generate_object_id():
    return str(bson.ObjectId())

class BaseModel(TimeStampedModel):
    migration_page_size = 50000

    objects = models.QuerySet.as_manager()

    class Meta:
        abstract = True

    def __unicode__(self):
        return '{}'.format(self.id)

    def to_storage(self, include_auto_now=True):
        local_django_fields = set([x.name for x in self._meta.concrete_fields if include_auto_now or not getattr(x, 'auto_now', False)])
        return {name: self.serializable_value(name) for name in local_django_fields}

    @classmethod
    def get_fk_field_names(cls):
        return [field.name for field in cls._meta.get_fields() if
                    field.is_relation and not field.auto_created and (field.many_to_one or field.one_to_one) and not isinstance(field, GenericForeignKey)]

    @classmethod
    def get_m2m_field_names(cls):
        return [field.attname or field.name for field in
                     cls._meta.get_fields() if
                     field.is_relation and field.many_to_many and not hasattr(field, 'field')]

    @classmethod
    def load(cls, data, select_for_update=False):
        try:
            if isinstance(data, basestring):
                # Some models (CitationStyle) have an _id that is not a bson
                # Looking up things by pk will never work with a basestring
                return cls.objects.get(_id=data) if not select_for_update else cls.objects.filter(_id=data).select_for_update().get()
            return cls.objects.get(pk=data) if not select_for_update else cls.objects.filter(pk=data).select_for_update().get()
        except cls.DoesNotExist:
            return None

    @classmethod
    def remove(cls, query=None):
        return cls.objects.filter(query).delete() if query else cls.objects.all().delete()

    @classmethod
    def remove_one(cls, obj):
        if obj.pk:
            return obj.delete()

    @property
    def _primary_name(self):
        return '_id'

    @property
    def _is_loaded(self):
        return bool(self.pk)

    def reload(self):
        return self.refresh_from_db()

    def refresh_from_db(self):
        super(BaseModel, self).refresh_from_db()
        # Django's refresh_from_db does not uncache GFKs
        for field in self._meta.private_fields:
            if hasattr(field, 'cache_attr') and field.cache_attr in self.__dict__:
                del self.__dict__[field.cache_attr]

    def clone(self):
        """Create a new, unsaved copy of this object."""
        copy = self.__class__.objects.get(pk=self.pk)
        copy.id = None

        # empty all the fks
        fk_field_names = [f.name for f in self._meta.model._meta.get_fields() if isinstance(f, (ForeignKey, GenericForeignKey))]
        for field_name in fk_field_names:
            setattr(copy, field_name, None)

        try:
            copy._id = bson.ObjectId()
        except AttributeError:
            pass
        return copy

    def save(self, *args, **kwargs):
        # Make Django validate on save (like modm)
        if not kwargs.get('force_insert') and not kwargs.get('force_update'):
            try:
                self.full_clean()
            except DjangoValidationError as err:
                raise ValidationError(*err.args)
        return super(BaseModel, self).save(*args, **kwargs)

# TODO: Rename to Identifier?
class Guid(BaseModel):
    """Stores either a short guid or long object_id for any model that inherits from BaseIDMixin.
    Each ID field (e.g. 'guid', 'object_id') MUST have an accompanying method, named with
    'initialize_<ID type>' (e.g. 'initialize_guid') that generates and sets the field.
    """
    primary_identifier_name = '_id'

    id = models.AutoField(primary_key=True)
    _id = LowercaseCharField(max_length=255, null=False, blank=False, default=generate_guid, db_index=True,
                           unique=True)
    referent = GenericForeignKey()
    content_type = models.ForeignKey(ContentType, null=True, blank=True, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField(null=True, blank=True)
    created = NonNaiveDateTimeField(db_index=True, auto_now_add=True)

    def __repr__(self):
        return '<id:{0}, referent:({1})>'.format(self._id, self.referent.__repr__())

    # Override load in order to load by GUID
    @classmethod
    def load(cls, data, select_for_update=False):
        try:
            return cls.objects.get(_id=data) if not select_for_update else cls.objects.filter(_id=data).select_for_update().get()
        except cls.DoesNotExist:
            return None

    class Meta:
        ordering = ['-created']
        get_latest_by = 'created'
        index_together = (
            ('content_type', 'object_id', 'created'),
        )

class BlackListGuid(BaseModel):
    id = models.AutoField(primary_key=True)
    guid = LowercaseCharField(max_length=255, unique=True, db_index=True)

    @property
    def _id(self):
        return self.guid

def generate_guid_instance():
    return Guid.objects.create().id

class PKIDStr(str):
    def __new__(self, _id, pk):
        return str.__new__(self, _id)

    def __init__(self, _id, pk):
        self.__pk = pk

    def __int__(self):
        return self.__pk

class BaseIDMixin(models.Model):
    class Meta:
        abstract = True

class ObjectIDMixin(BaseIDMixin):
    primary_identifier_name = '_id'

    _id = models.CharField(max_length=24, default=generate_object_id, unique=True, db_index=True)

    def __unicode__(self):
        return '_id: {}'.format(self._id)

    @classmethod
    def load(cls, q, select_for_update=False):
        try:
            return cls.objects.get(_id=q) if not select_for_update else cls.objects.filter(_id=q).select_for_update().get()
        except cls.DoesNotExist:
            # modm doesn't throw exceptions when loading things that don't exist
            return None

    class Meta:
        abstract = True

class InvalidGuid(Exception):
    pass

class OptionalGuidMixin(BaseIDMixin):
    """
    This makes it so that things can **optionally** have guids. Think files.
    Things that inherit from this must also inherit from ObjectIDMixin ... probably
    """
    __guid_min_length__ = 5

    guids = GenericRelation(Guid, related_name='referent', related_query_name='referents')
    content_type_pk = models.PositiveIntegerField(null=True, blank=True)

    def __unicode__(self):
        return '{}'.format(self.get_guid() or self.id)

    def get_guid(self, create=False):
        if not self.pk:
            logger.warn('Implicitly saving object before creating guid')
            self.save()
        if create:
            try:
                guid, created = Guid.objects.get_or_create(
                    object_id=self.pk,
                    content_type_id=ContentType.objects.get_for_model(self).pk
                )
            except MultipleObjectsReturned:
                # lol, hacks
                pass
            else:
                return guid
        return self.guids.first()

    class Meta:
        abstract = True

class GuidMixinQuerySet(IncludeQuerySet):

    def _filter_or_exclude(self, negate, *args, **kwargs):
        return super(GuidMixinQuerySet, self)._filter_or_exclude(negate, *args, **kwargs).include('guids')

    def all(self):
        return super(GuidMixinQuerySet, self).all().include('guids')

    def count(self):
        return super(GuidMixinQuerySet, self.include(None)).count()

class GuidMixin(BaseIDMixin):
    __guid_min_length__ = 5

    guids = GenericRelation(Guid, related_name='referent', related_query_name='referents')
    content_type_pk = models.PositiveIntegerField(null=True, blank=True)

    objects = GuidMixinQuerySet.as_manager()
    # TODO: use pre-delete signal to disable delete cascade

    def __unicode__(self):
        return '{}'.format(self._id)

    @cached_property
    def _id(self):
        try:
            guid = self.guids.all()[0]
        except IndexError:
            return None
        if guid:
            return guid._id
        return None

    @_id.setter
    def _id(self, value):
        # TODO do we really want to allow this?
        guid, created = Guid.objects.get_or_create(_id=value)
        if created:
            guid.object_id = self.pk
            guid.content_type = ContentType.objects.get_for_model(self)
            guid.save()
        elif guid.content_type == ContentType.objects.get_for_model(self) and guid.object_id == self.pk:
            # TODO should this up the created for the guid until now so that it appears as the first guid
            # for this object?
            return
        else:
            raise InvalidGuid('Cannot indirectly repoint an existing guid, please use the Guid model')

    _primary_key = _id

    @classmethod
    def load(cls, q, select_for_update=False):
        # Minor optimization--no need to query if q is None or ''
        if not q:
            return None
        try:
            # guids___id__isnull=False forces an INNER JOIN
            if select_for_update:
                return cls.objects.filter(guids___id__isnull=False, guids___id=q).select_for_update().first()
            return cls.objects.filter(guids___id__isnull=False, guids___id=q).first()
        except cls.DoesNotExist:
            return None

    @property
    def deep_url(self):
        return None

    class Meta:
        abstract = True

@receiver(post_save)
def ensure_guid(sender, instance, created, **kwargs):
    if not issubclass(sender, GuidMixin):
        return False
    existing_guids = Guid.objects.filter(object_id=instance.pk, content_type=ContentType.objects.get_for_model(instance))
    has_cached_guids = hasattr(instance, '_prefetched_objects_cache') and 'guids' in instance._prefetched_objects_cache
    if not existing_guids.exists():
        # Clear query cache of instance.guids
        if has_cached_guids:
            del instance._prefetched_objects_cache['guids']
        Guid.objects.create(object_id=instance.pk, content_type=ContentType.objects.get_for_model(instance),
                            _id=generate_guid(instance.__guid_min_length__))
<commit_msg>Remove remove and remove_one methods from BaseModel.<commit_after>import logging
import random

import bson
from django.contrib.contenttypes.fields import (GenericForeignKey,
                                                GenericRelation)
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import MultipleObjectsReturned
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import models
from django.db.models import ForeignKey
from django.db.models.signals import post_save
from django.dispatch import receiver
from django_extensions.db.models import TimeStampedModel
from include import IncludeQuerySet

from osf.utils.caching import cached_property
from osf.exceptions import ValidationError
from osf.utils.fields import LowercaseCharField, NonNaiveDateTimeField

ALPHABET = '23456789abcdefghjkmnpqrstuvwxyz'

logger = logging.getLogger(__name__)

def generate_guid(length=5):
    while True:
        guid_id = ''.join(random.sample(ALPHABET, length))

        try:
            # is the guid in the blacklist
            BlackListGuid.objects.get(guid=guid_id)
        except BlackListGuid.DoesNotExist:
            # it's not, check and see if it's already in the database
            try:
                Guid.objects.get(_id=guid_id)
            except Guid.DoesNotExist:
                # valid and unique guid
                return guid_id

def generate_object_id():
    return str(bson.ObjectId())

class BaseModel(TimeStampedModel):
    migration_page_size = 50000

    objects = models.QuerySet.as_manager()

    class Meta:
        abstract = True

    def __unicode__(self):
        return '{}'.format(self.id)

    def to_storage(self, include_auto_now=True):
        local_django_fields = set([x.name for x in self._meta.concrete_fields if include_auto_now or not getattr(x, 'auto_now', False)])
        return {name: self.serializable_value(name) for name in local_django_fields}

    @classmethod
    def get_fk_field_names(cls):
        return [field.name for field in cls._meta.get_fields() if
                    field.is_relation and not field.auto_created and (field.many_to_one or field.one_to_one) and not isinstance(field, GenericForeignKey)]

    @classmethod
    def get_m2m_field_names(cls):
        return [field.attname or field.name for field in
                     cls._meta.get_fields() if
                     field.is_relation and field.many_to_many and not hasattr(field, 'field')]

    @classmethod
    def load(cls, data, select_for_update=False):
        try:
            if isinstance(data, basestring):
                # Some models (CitationStyle) have an _id that is not a bson
                # Looking up things by pk will never work with a basestring
                return cls.objects.get(_id=data) if not select_for_update else cls.objects.filter(_id=data).select_for_update().get()
            return cls.objects.get(pk=data) if not select_for_update else cls.objects.filter(pk=data).select_for_update().get()
        except cls.DoesNotExist:
            return None

    @property
    def _primary_name(self):
        return '_id'

    @property
    def _is_loaded(self):
        return bool(self.pk)

    def reload(self):
        return self.refresh_from_db()

    def refresh_from_db(self):
        super(BaseModel, self).refresh_from_db()
        # Django's refresh_from_db does not uncache GFKs
        for field in self._meta.private_fields:
            if hasattr(field, 'cache_attr') and field.cache_attr in self.__dict__:
                del self.__dict__[field.cache_attr]

    def clone(self):
        """Create a new, unsaved copy of this object."""
        copy = self.__class__.objects.get(pk=self.pk)
        copy.id = None

        # empty all the fks
        fk_field_names = [f.name for f in self._meta.model._meta.get_fields() if isinstance(f, (ForeignKey, GenericForeignKey))]
        for field_name in fk_field_names:
            setattr(copy, field_name, None)

        try:
            copy._id = bson.ObjectId()
        except AttributeError:
            pass
        return copy

    def save(self, *args, **kwargs):
        # Make Django validate on save (like modm)
        if not kwargs.get('force_insert') and not kwargs.get('force_update'):
            try:
                self.full_clean()
            except DjangoValidationError as err:
                raise ValidationError(*err.args)
        return super(BaseModel, self).save(*args, **kwargs)

# TODO: Rename to Identifier?
class Guid(BaseModel):
    """Stores either a short guid or long object_id for any model that inherits from BaseIDMixin.
    Each ID field (e.g. 'guid', 'object_id') MUST have an accompanying method, named with
    'initialize_<ID type>' (e.g. 'initialize_guid') that generates and sets the field.
    """
    primary_identifier_name = '_id'

    id = models.AutoField(primary_key=True)
    _id = LowercaseCharField(max_length=255, null=False, blank=False, default=generate_guid, db_index=True,
                           unique=True)
    referent = GenericForeignKey()
    content_type = models.ForeignKey(ContentType, null=True, blank=True, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField(null=True, blank=True)
    created = NonNaiveDateTimeField(db_index=True, auto_now_add=True)

    def __repr__(self):
        return '<id:{0}, referent:({1})>'.format(self._id, self.referent.__repr__())

    # Override load in order to load by GUID
    @classmethod
    def load(cls, data, select_for_update=False):
        try:
            return cls.objects.get(_id=data) if not select_for_update else cls.objects.filter(_id=data).select_for_update().get()
        except cls.DoesNotExist:
            return None

    class Meta:
        ordering = ['-created']
        get_latest_by = 'created'
        index_together = (
            ('content_type', 'object_id', 'created'),
        )

class BlackListGuid(BaseModel):
    id = models.AutoField(primary_key=True)
    guid = LowercaseCharField(max_length=255, unique=True, db_index=True)

    @property
    def _id(self):
        return self.guid

def generate_guid_instance():
    return Guid.objects.create().id

class PKIDStr(str):
    def __new__(self, _id, pk):
        return str.__new__(self, _id)

    def __init__(self, _id, pk):
        self.__pk = pk

    def __int__(self):
        return self.__pk

class BaseIDMixin(models.Model):
    class Meta:
        abstract = True

class ObjectIDMixin(BaseIDMixin):
    primary_identifier_name = '_id'

    _id = models.CharField(max_length=24, default=generate_object_id, unique=True, db_index=True)

    def __unicode__(self):
        return '_id: {}'.format(self._id)

    @classmethod
    def load(cls, q, select_for_update=False):
        try:
            return cls.objects.get(_id=q) if not select_for_update else cls.objects.filter(_id=q).select_for_update().get()
        except cls.DoesNotExist:
            # modm doesn't throw exceptions when loading things that don't exist
            return None

    class Meta:
        abstract = True

class InvalidGuid(Exception):
    pass

class OptionalGuidMixin(BaseIDMixin):
    """
    This makes it so that things can **optionally** have guids. Think files.
    Things that inherit from this must also inherit from ObjectIDMixin ... probably
    """
    __guid_min_length__ = 5

    guids = GenericRelation(Guid, related_name='referent', related_query_name='referents')
    content_type_pk = models.PositiveIntegerField(null=True, blank=True)

    def __unicode__(self):
        return '{}'.format(self.get_guid() or self.id)

    def get_guid(self, create=False):
        if not self.pk:
            logger.warn('Implicitly saving object before creating guid')
            self.save()
        if create:
            try:
                guid, created = Guid.objects.get_or_create(
                    object_id=self.pk,
                    content_type_id=ContentType.objects.get_for_model(self).pk
                )
            except MultipleObjectsReturned:
                # lol, hacks
                pass
            else:
                return guid
        return self.guids.first()

    class Meta:
        abstract = True

class GuidMixinQuerySet(IncludeQuerySet):

    def _filter_or_exclude(self, negate, *args, **kwargs):
        return super(GuidMixinQuerySet, self)._filter_or_exclude(negate, *args, **kwargs).include('guids')

    def all(self):
        return super(GuidMixinQuerySet, self).all().include('guids')

    def count(self):
        return super(GuidMixinQuerySet, self.include(None)).count()

class GuidMixin(BaseIDMixin):
    __guid_min_length__ = 5

    guids = GenericRelation(Guid, related_name='referent', related_query_name='referents')
    content_type_pk = models.PositiveIntegerField(null=True, blank=True)

    objects = GuidMixinQuerySet.as_manager()
    # TODO: use pre-delete signal to disable delete cascade

    def __unicode__(self):
        return '{}'.format(self._id)

    @cached_property
    def _id(self):
        try:
            guid = self.guids.all()[0]
        except IndexError:
            return None
        if guid:
            return guid._id
        return None

    @_id.setter
    def _id(self, value):
        # TODO do we really want to allow this?
        guid, created = Guid.objects.get_or_create(_id=value)
        if created:
            guid.object_id = self.pk
            guid.content_type = ContentType.objects.get_for_model(self)
            guid.save()
        elif guid.content_type == ContentType.objects.get_for_model(self) and guid.object_id == self.pk:
            # TODO should this up the created for the guid until now so that it appears as the first guid
            # for this object?
            return
        else:
            raise InvalidGuid('Cannot indirectly repoint an existing guid, please use the Guid model')

    _primary_key = _id

    @classmethod
    def load(cls, q, select_for_update=False):
        # Minor optimization--no need to query if q is None or ''
        if not q:
            return None
        try:
            # guids___id__isnull=False forces an INNER JOIN
            if select_for_update:
                return cls.objects.filter(guids___id__isnull=False, guids___id=q).select_for_update().first()
            return cls.objects.filter(guids___id__isnull=False, guids___id=q).first()
        except cls.DoesNotExist:
            return None

    @property
    def deep_url(self):
        return None

    class Meta:
        abstract = True

@receiver(post_save)
def ensure_guid(sender, instance, created, **kwargs):
    if not issubclass(sender, GuidMixin):
        return False
    existing_guids = Guid.objects.filter(object_id=instance.pk, content_type=ContentType.objects.get_for_model(instance))
    has_cached_guids = hasattr(instance, '_prefetched_objects_cache') and 'guids' in instance._prefetched_objects_cache
    if not existing_guids.exists():
        # Clear query cache of instance.guids
        if has_cached_guids:
            del instance._prefetched_objects_cache['guids']
        Guid.objects.create(object_id=instance.pk, content_type=ContentType.objects.get_for_model(instance),
                            _id=generate_guid(instance.__guid_min_length__))
<|endoftext|>


<commit_before>

def ae_db_serializer(cls):
    from google.appengine.api import users
    from google.appengine.ext import blobstore, db

    @classmethod
    def restler_collection_types(cls, obj):
        if isinstance(obj, db.Query):
            return True
        else:
            return False

    @classmethod
    def restler_encoder(cls, obj):
        if isinstance(obj, db.GeoPt):
            return "%s %s" % (obj.lat, obj.lon)
        if isinstance(obj, db.IM):
            return "%s %s" % (obj.protocol, obj.address)
        if isinstance(obj, users.User):
            return obj.user_id() or obj.email()
        if isinstance(obj, blobstore.BlobInfo):
            return str(obj.key())  # TODO is this correct?

    @classmethod
    def restler_kind(cls, model):
        return model.kind().lower()

    @classmethod
    def restler_properties(cls, model):
        return list(model.properties().iterkeys())

    cls.restler_collection_types = restler_collection_types
    cls.restler_encoder = restler_encoder
    cls.restler_kind = restler_kind
    cls.restler_properties = restler_properties

    return cls

def ae_ndb_serializer(cls):
    from google.appengine.api import users
    from google.appengine.ext import ndb

    @classmethod
    def restler_collection_types(cls, obj):
        if isinstance(obj, ndb.query.Query):
            return True
        else:
            return False

    @classmethod
    def restler_encoder(cls, obj):
        if isinstance(obj, ndb.GeoPt):
            return "%s %s" % (obj.lat, obj.lon)
        if isinstance(obj, users.User):
            return obj.user_id() or obj.email()

    @classmethod
    def restler_kind(cls, model):
        try:
            return model.__class__.__name__.lower()
        except:
            # TODO When is this the case?
            return model.__name__.lower()

    @classmethod
    def restler_properties(cls, model):
        return list(model._properties.iterkeys())

    cls.restler_collection_types = restler_collection_types
    cls.restler_encoder = restler_encoder
    cls.restler_kind = restler_kind
    cls.restler_properties = restler_properties

    return cls
<commit_msg>extracted common code to ae_common_encoder<commit_after>

def ae_common_encoder(obj):
    from google.appengine.api import users
    if isinstance(obj, users.User):
            return obj.user_id() or obj.email()

def ae_db_serializer(cls):
    # from google.appengine.api import users
    from google.appengine.ext import blobstore, db

    @classmethod
    def restler_collection_types(cls, obj):
        if isinstance(obj, db.Query):
            return True
        else:
            return False

    @classmethod
    def restler_encoder(cls, obj):
        if isinstance(obj, db.GeoPt):
            return "%s %s" % (obj.lat, obj.lon)
        if isinstance(obj, db.IM):
            return "%s %s" % (obj.protocol, obj.address)
        if isinstance(obj, blobstore.BlobInfo):
            return str(obj.key())  # TODO is this correct?
        if ae_common_encoder(obj):
            return ae_common_encoder(obj)

    @classmethod
    def restler_kind(cls, model):
        return model.kind().lower()

    @classmethod
    def restler_properties(cls, model):
        return list(model.properties().iterkeys())

    cls.restler_collection_types = restler_collection_types
    cls.restler_encoder = restler_encoder
    cls.restler_kind = restler_kind
    cls.restler_properties = restler_properties

    return cls

def ae_ndb_serializer(cls):
    from google.appengine.ext import ndb

    @classmethod
    def restler_collection_types(cls, obj):
        if isinstance(obj, ndb.query.Query):
            return True
        else:
            return False

    @classmethod
    def restler_encoder(cls, obj):
        if isinstance(obj, ndb.GeoPt):
            return "%s %s" % (obj.lat, obj.lon)
        if ae_common_encoder(obj):
            return ae_common_encoder(obj)

    @classmethod
    def restler_kind(cls, model):
        try:
            return model.__class__.__name__.lower()
        except:
            # TODO When is this the case?
            return model.__name__.lower()

    @classmethod
    def restler_properties(cls, model):
        return list(model._properties.iterkeys())

    cls.restler_collection_types = restler_collection_types
    cls.restler_encoder = restler_encoder
    cls.restler_kind = restler_kind
    cls.restler_properties = restler_properties

    return cls
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import time
import os
import pika
import subprocess
import urllib2
import json

RWCY_URL = "http://arewecompatibleyet.com/data/masterbugtable.js"
RWCY_PREFIX =\
    """/* This file is generated by preproc/buildlists.py - do not edit */
var masterBugTable ="""

BASEPATH = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
BROWSER_CMD = BASEPATH + "/xvfb-run.sh -w 0 python " + BASEPATH + "/browser.py %s"

browsers = {}
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='mozcompat')
channel.basic_qos(prefetch_count=1)

i = 0

def callback(channel, method, properties, body):
    global i
    i += 1
    print "RECEIVED", i, body
    if not body in browsers:
        browsers[body] = subprocess.Popen(BROWSER_CMD % (body), shell=True)
    while not is_clean():
        time.sleep(3)

def is_clean():
    values = [(key, value) for key, value in browsers.items()]
    for key, value in values:
        if value.poll() is not None:
            del browsers[key]
    return len(browsers) < 5

if len(sys.argv) == 2 and sys.argv[1] == "listen":
    channel.basic_consume(callback, queue='mozcompat', no_ack=True)
    channel.start_consuming()

elif (len(sys.argv) == 2 or len(sys.argv) == 3) and sys.argv[1] == "rwcy":
    response = urllib2.urlopen(RWCY_URL)
    js_str = response.read()[len(RWCY_PREFIX):].strip()
    js = json.loads(js_str)
    # the "masterbugtable" JSON data structure: the hostIndex has one property for each host which has either open or closed bugs in bugzilla
    # In addition, the lists object contains local top lists - for example js["lists"]["no70"]["data"] is an array of (roughly) the top 70 sites in Norway
    # If we want to do a test run of *all* URLs in the system, including local lists, we should go through all these lists and add properties to masterbugtable's hostIndex
    for key in js["lists"].keys():
        for site in js["lists"][key]["data"]:
            if not (site in js["hostIndex"]):
                js["hostIndex"][site] = 1
    start_from = 0
    if len(sys.argv) == 3:
        start_from = int(sys.argv[2])
        print 'rwcy lists from index %i' % start_from
    counter = -1
    print "%i sites to run through, starting from %i" % (len(js["hostIndex"].keys()), start_from)
    for key in js["hostIndex"].keys():
        counter += 1
        if key[-1] == "." or counter < start_from:
            continue
        channel.basic_publish(exchange='', routing_key='mozcompat', body=key)

elif len(sys.argv) == 3:
    if sys.argv[1] == "push":
        channel.basic_publish(exchange='', routing_key='mozcompat',
                              body=sys.argv[2])
    elif sys.argv[1] == "pushall":
        f = open(sys.argv[2], "r")
        sites = f.readlines()
        for site in sites:
            channel.basic_publish(exchange='', routing_key='mozcompat',
                                  body=site)

else:
    print "WRONG BLA BLA"
<commit_msg>support running a country-specific list from arewecompatibleyet.com (rather than all lists)<commit_after>#!/usr/bin/env python
import sys
import time
import os
import pika
import subprocess
import urllib2
import json

RWCY_URL = "http://arewecompatibleyet.com/data/masterbugtable.js"
RWCY_PREFIX =\
    """/* This file is generated by preproc/buildlists.py - do not edit */
var masterBugTable ="""

BASEPATH = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
BROWSER_CMD = BASEPATH + "/xvfb-run.sh -w 0 python " + BASEPATH + "/browser.py %s"

browsers = {}
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='mozcompat')
channel.basic_qos(prefetch_count=1)

i = 0

def callback(channel, method, properties, body):
    global i
    i += 1
    print "RECEIVED", i, body
    if not body in browsers:
        browsers[body] = subprocess.Popen(BROWSER_CMD % (body), shell=True)
    while not is_clean():
        time.sleep(3)

def is_clean():
    values = [(key, value) for key, value in browsers.items()]
    for key, value in values:
        if value.poll() is not None:
            del browsers[key]
    return len(browsers) < 5

if len(sys.argv) == 2 and sys.argv[1] == "listen":
    channel.basic_consume(callback, queue='mozcompat', no_ack=True)
    channel.start_consuming()

elif len(sys.argv) in [2,3,4] and (sys.argv[1] == "rwcy" or sys.argv[1] == "rwcylist"):
    # support command lines like
    # webcompat.py rwcy
    # webcompat.py rwcy 1050
    # webcompat.py rwcylist jp50
    # webcompat.py rwcylist jp50 35
    if sys.argv[1] == "rwcylist" and len(sys.argv) == 2:
            print('ERROR: list name missing')
            quit()
    response = urllib2.urlopen(RWCY_URL)
    js_str = response.read()[len(RWCY_PREFIX):].strip()
    js = json.loads(js_str)
    # the "masterbugtable" JSON data structure: the hostIndex has one property for each host which has either open or closed bugs in bugzilla
    # In addition, the lists object contains local top lists - for example js["lists"]["no70"]["data"] is an array of (roughly) the top 70 sites in Norway
    # If we want to do a test run of *all* URLs in the system, including local lists, we should go through all these lists and add properties to masterbugtable's hostIndex
    if sys.argv[1] == "rwcy":
            for key in js["lists"].keys():
                for site in js["lists"][key]["data"]:
                    if not (site in js["hostIndex"]):
                        js["hostIndex"][site] = 1
    start_from = 0
    if sys.argv[1] == "rwcy" and len(sys.argv) == 3:
        start_from = int(sys.argv[2])
        print 'rwcy lists from index %i' % start_from
    elif sys.argv[1]  == "rwcylist" and len(sys.argv) == 4:
        start_from = int(sys.argv[3])
        print 'rwcy lists from index %i' % start_from
    counter = -1
    the_list = js["hostIndex"].keys() if sys.argv[1] == "rwcy" else js['lists'][sys.argv[2]]['data']
    print "%i sites to run through, starting from %i" % (len(the_list), start_from)
    for key in the_list:
        counter += 1
        if key[-1] == "." or counter < start_from:
            continue
        channel.basic_publish(exchange='', routing_key='mozcompat', body=key)

elif len(sys.argv) == 3:
    if sys.argv[1] == "push":
        channel.basic_publish(exchange='', routing_key='mozcompat',
                              body=sys.argv[2])
    elif sys.argv[1] == "pushall":
        f = open(sys.argv[2], "r")
        sites = f.readlines()
        for site in sites:
            channel.basic_publish(exchange='', routing_key='mozcompat',
                                  body=site)

else:
    print "WRONG BLA BLA"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
"""import getopt
import configparser"""

class sim_datacenter:
  """A datacenter that can contain racks"""
  def __init__(self, name):
    self.name = name
    self.racks = []
    self.rack_max = None

  """Set maximum number of racks in DC"""
  def set_rack_max(self, rack_max):
    self.rack_max = rack_max

  def add_rack(self, rack):
    if (not isinstance(rack, sim_rack)):
      return rack.name+" is not a rack!"
    if ( self.rack_max == None or len(racks) < self.rack_max ):
      self.racks.append(rack)
    else:
      return self.name+" is full, can't add rack!"

  def find_suitable_host(self, kind, vcpu, ram):
    suitable_objects = []
    for rack in self.racks:
      for server in rack.servers:
        if kind in server.capability:
          usage = server.get_server_usage()
          if usage["vcpu"] + vcpu <= server.vcpu_max_capacity and usage["ram"] + ram <= server.ram_max_capacity:
            suitable_objects.append(server)
        if server.vms:
          for vm in server.vms:
            if kind in vm.capability:
              print("TODO")
    return suitable_objects

  def __str__(self):
    output = "-Datacenter "+self.name+"\n"
    output += "  Datacenter size: "+str(self.rack_max)+"\n"
    output += "  Racks in this datacenter: \n"
    for rack in self.racks:
      output += str(rack)
    return output

class sim_rack:
  """A rack that can contain servers"""
  def __init__(self, name):
    self.name = name
    self.servers = []
    self.rack_size = 42

  """Set maximum number of servers units in rack"""
  def set_rack_size(self, rack_size):
    self.rack_size = rack_size

  def get_rack_usage(self):
    rack_usage = 0
    for server in self.servers:
      rack_usage += server.server_size
    return rack_usage

  def add_server(self, server):
    if (not isinstance(server, sim_server)):
      return server.name+" is not a server!"
    rack_usage = self.get_rack_usage()
    if ( self.rack_size == None or rack_usage < self.rack_size):
      self.servers.append(server)
    else:
      return self.name+" is full, can't add server!"

  def __str__(self):
    output = "    +"+self.name+"\n"
    output += "      Rack usage: "+str(self.get_rack_usage())+"/"+str(self.rack_size)+"U\n"
    output += "      Servers in this rack: \n"
    for server in self.servers:
      output += str(server)
    return output

class sim_server:
  """A 2U server that may run containers or virtual machines or both"""
  def __init__(self, name):
    self.name = name
    self.capability = []
    self.server_size = 2
    self.vcpu_max_capacity = None
    self.ram_max_capacity = None
    self.vms = None
    self.containers = None

  """Set the ability to run VMs or containers"""
  def set_server_capability(self, capability):
    self.capability = capability

  """Set the vCPU limit on server"""
  def set_vcpu_max_capacity(self, vcpu_max_capacity):
    self.vcpu_max_capacity = vcpu_max_capacity

  """Set the RAM limit on server"""
  def set_ram_max_capacity(self, ram_max_capacity):
    self.ram_max_capacity = ram_max_capacity

  def get_server_usage(self):
    server_usage = {"vcpu" : 0, "ram" : 0}
    if self.vms or self.containers:
      for logical_object in self.vms + self.containers:
        server_usage["vcpu"] += logical_object.vcpu_alloc
        server_usage["ram"] += logical_object.gigabytes_ram_alloc
    return server_usage

  def check_server_capability(self, kind, vcpu, ram):
    if kind in capability:
      return True

  def __str__(self):
    output ="        *"+self.name+"\n"
    output += "          Server size : "+str(self.server_size)+"U\n"
    if self.vcpu_max_capacity:
      output += "          Server vCPU usage: "+str(self.get_server_usage()["vcpu"])+"/"+str(self.vcpu_max_capacity)+" vCPU\n"
    if self.ram_max_capacity:
      output += "          Server RAM usage: "+str(self.get_server_usage()["ram"])+"/"+str(self.ram_max_capacity)+" GB RAM\n"
    if "vm" in self.capability:
      output += "          Can run VMs\n"
    if (self.vms):
      for vm in self.vms:
        output += str(vm)+"\n"
    if "container" in self.capability:
      output += "          Can run containers\n"
    if (self.containers):
      for container in self.containers:
        output += str(container)+"\n"
    return output

class sim_logical_object:
  """Allocate vm or container in a specified DC"""
  def add_logical_object_in_dc(self, dc):
    host = dc.find_suitable_host(self.kind, self.vcpu_alloc, self.gigabytes_ram_alloc)

  """Force VM or container allocation on a specified server"""
  def add_logical_object_on_server(self, server):
    print("TODO")

  def print_name_cpu_ram(self):
    output ="          %"+self.name+"\n"
    output +="           "+self.vcpu_alloc+"\n"
    output +="           "+self.gigabytes_ram_alloc+"\n"

  def __init__(self, name, vcpu_alloc, gigabytes_ram_alloc):
    self.name = name
    self.vcpu_alloc = vcpu_alloc
    self.gigabytes_ram_alloc = gigabytes_ram_alloc

class sim_vm(sim_logical_object):
  """A VM on a server that can execute VMs"""
  kind = "vm"

  """Set the ability to run VMs or containers"""
  def set_vm_capability(self, capability):
    self.capability = capability

  def __str__(self):
    print_name_cpu_ram()
    if (self.containers):
      for container in self.containers:
        output += str(container)+"\n"
    return output

class sim_container(sim_logical_object):
  """A container on a server that can execute containers"""
  kind = "container"

  """Force container allocation on a specified VM"""
  def add_container_on_vm(self, vm):
    print("TODO")

  def __str__(self):
    print_name_cpu_ram()
    return output

def init_infrastructure():
  dc1 = sim_datacenter("dc1")
  dc2 = sim_datacenter("dc2")

  rack1 = sim_rack("dc1_rack1")
  dc1.add_rack(rack1)
  rack2 = sim_rack("dc2_rack2")
  dc2.add_rack(rack2)

  srv1 = sim_server("dc1_rack1_srv1")
  srv1.set_server_capability(["vm","container"])
  """Server with 2 sockets, 14 cores each socket, 4 vCPU each core"""
  srv1.set_vcpu_max_capacity(2*14*4)
  """Server with 12 sticks of 16GB of RAM"""
  srv1.set_ram_max_capacity(12*16)
  rack1.add_server(srv1)

  srv2 = sim_server("dc2_rack2_srv2")
  srv2.set_server_capability(["vm"])
  rack2.add_server(srv2)

  srv3 = sim_server("dc1_rack1_srv3")
  srv3.set_server_capability(["vm"])
  """Server with 1 sockets, 1 cores each socket, 4 vCPU each core"""
  srv3.set_vcpu_max_capacity(1*1*4)
  """Server with 1 stick of 16GB of RAM"""
  srv3.set_ram_max_capacity(16)
  rack1.add_server(srv3)

  vm1 = sim_vm("dc1_vm1", 8, 16)
  vm1.add_logical_object_in_dc(dc1)
  vm1.set_vm_capability(["container"])

  cont1 = sim_container("dc1_cont1", 1, 1)
  cont1.add_logical_object_in_dc(dc1)

  print(dc1)
  print(dc2)

def usage():
  return ''

def main():
  """#default, can be overriden
  config_file_path = 'example.cfg'

  try:
    opts, args = getopt.getopt(sys.argv[1:], 'c:h', ['config=', 'help'])
  except getopt.GetoptError:
    usage()
    sys.exit(2)

  for o, a in opts:
    if o in ('-c', '--config'):
      config_file_path = a
    if o in ('-h', '--help'):
      usage()

  config = configparser.RawConfigParser()
  config.read(config_file_path)
  test = [config.get('Test', 'Test_output')]"""

  init_infrastructure()

  sys.exit()

if __name__ == '__main__':
  main()
<commit_msg>WIP : registering guest in host<commit_after># -*- coding: utf-8 -*-
import sys
import random
"""import getopt
import configparser"""

class sim_datacenter:
  """A datacenter that can contain racks"""
  def __init__(self, name):
    self.name = name
    self.racks = []
    self.rack_max = None

  """Set maximum number of racks in DC"""
  def set_rack_max(self, rack_max):
    self.rack_max = rack_max

  def add_rack(self, rack):
    if (not isinstance(rack, sim_rack)):
      return rack.name+" is not a rack!"
    if ( self.rack_max == None or len(racks) < self.rack_max ):
      self.racks.append(rack)
    else:
      return self.name+" is full, can't add rack!"

  def find_suitable_host(self, kind, vcpu, ram):
    suitable_objects = []
    for rack in self.racks:
      for server in rack.servers:
        if kind in server.capability:
          usage = server.get_server_usage()
          if usage["vcpu"] + vcpu <= server.vcpu_max_capacity and usage["ram"] + ram <= server.ram_max_capacity:
            suitable_objects.append(server)
        if server.vms:
          for vm in server.vms:
            if kind in vm.capability:
              print("TODO")
    return random.choice(suitable_objects)

  def __str__(self):
    output = "-Datacenter "+self.name+"\n"
    output += "  Datacenter size: "+str(self.rack_max)+"\n"
    output += "  Racks in this datacenter: \n"
    for rack in self.racks:
      output += str(rack)
    return output

class sim_rack:
  """A rack that can contain servers"""
  def __init__(self, name):
    self.name = name
    self.servers = []
    self.rack_size = 42

  """Set maximum number of servers units in rack"""
  def set_rack_size(self, rack_size):
    self.rack_size = rack_size

  def get_rack_usage(self):
    rack_usage = 0
    for server in self.servers:
      rack_usage += server.server_size
    return rack_usage

  def add_server(self, server):
    if (not isinstance(server, sim_server)):
      return server.name+" is not a server!"
    rack_usage = self.get_rack_usage()
    if ( self.rack_size == None or rack_usage < self.rack_size):
      self.servers.append(server)
    else:
      return self.name+" is full, can't add server!"

  def __str__(self):
    output = "    +"+self.name+"\n"
    output += "      Rack usage: "+str(self.get_rack_usage())+"/"+str(self.rack_size)+"U\n"
    output += "      Servers in this rack: \n"
    for server in self.servers:
      output += str(server)
    return output

class sim_server:
  """A 2U server that may run containers or virtual machines or both"""
  def __init__(self, name):
    self.name = name
    self.capability = []
    self.server_size = 2
    self.vcpu_max_capacity = None
    self.ram_max_capacity = None
    self.vms = None
    self.containers = None

  """Set the ability to run VMs or containers"""
  def set_server_capability(self, capability):
    self.capability = capability

  """Set the vCPU limit on server"""
  def set_vcpu_max_capacity(self, vcpu_max_capacity):
    self.vcpu_max_capacity = vcpu_max_capacity

  """Set the RAM limit on server"""
  def set_ram_max_capacity(self, ram_max_capacity):
    self.ram_max_capacity = ram_max_capacity

  def get_server_usage(self):
    server_usage = {"vcpu" : 0, "ram" : 0}
    if self.vms or self.containers:
      for logical_object in self.vms + self.containers:
        server_usage["vcpu"] += logical_object.vcpu_alloc
        server_usage["ram"] += logical_object.gigabytes_ram_alloc
    return server_usage

  def check_server_capability(self, kind, vcpu, ram):
    if kind in capability:
      return True

  def register_logical_object_to_host(self, guest):
    print("TODO")

  def __str__(self):
    output ="        *"+self.name+"\n"
    output += "          Server size : "+str(self.server_size)+"U\n"
    if self.vcpu_max_capacity:
      output += "          Server vCPU usage: "+str(self.get_server_usage()["vcpu"])+"/"+str(self.vcpu_max_capacity)+" vCPU\n"
    if self.ram_max_capacity:
      output += "          Server RAM usage: "+str(self.get_server_usage()["ram"])+"/"+str(self.ram_max_capacity)+" GB RAM\n"
    if "vm" in self.capability:
      output += "          Can run VMs\n"
    if (self.vms):
      for vm in self.vms:
        output += str(vm)+"\n"
    if "container" in self.capability:
      output += "          Can run containers\n"
    if (self.containers):
      for container in self.containers:
        output += str(container)+"\n"
    return output

class sim_logical_object:
  """Allocate vm or container in a specified DC"""
  def add_logical_object_in_dc(self, dc):
    host = dc.find_suitable_host(self.kind, self.vcpu_alloc, self.gigabytes_ram_alloc)
    host.register_logical_object_to_host(self)

  def register_logical_object_to_host(self, guest):
    print("TODO")

  """Force VM or container allocation on a specified server"""
  def add_logical_object_on_server(self, server):
    print(guest)

  def print_name_cpu_ram(self):
    output ="          %"+self.name+"\n"
    output +="           "+str(self.vcpu_alloc)+"\n"
    output +="           "+str(self.gigabytes_ram_alloc)+"\n"

  def __init__(self, name, vcpu_alloc, gigabytes_ram_alloc):
    self.name = name
    self.vcpu_alloc = vcpu_alloc
    self.gigabytes_ram_alloc = gigabytes_ram_alloc

class sim_vm(sim_logical_object):
  """A VM on a server that can execute VMs"""
  kind = "vm"

  """Set the ability to run VMs or containers"""
  def set_vm_capability(self, capability):
    self.capability = capability

  def __str__(self):
    output = ""
    self.print_name_cpu_ram()
    if (self.containers):
      for container in self.containers:
        output += str(container)+"\n"
    return output

  def __init__(self, name, vcpu_alloc, gigabytes_ram_alloc):
    self.name = name
    self.vcpu_alloc = vcpu_alloc
    self.gigabytes_ram_alloc = gigabytes_ram_alloc
    self.containers = []

class sim_container(sim_logical_object):
  """A container on a server that can execute containers"""
  kind = "container"

  """Force container allocation on a specified VM"""
  def add_container_on_vm(self, vm):
    print("TODO")

  def __str__(self):
    self.print_name_cpu_ram()
    return output

def init_infrastructure():
  dc1 = sim_datacenter("dc1")
  dc2 = sim_datacenter("dc2")

  rack1 = sim_rack("dc1_rack1")
  dc1.add_rack(rack1)
  rack2 = sim_rack("dc2_rack2")
  dc2.add_rack(rack2)

  srv1 = sim_server("dc1_rack1_srv1")
  srv1.set_server_capability(["vm","container"])
  """Server with 2 sockets, 14 cores each socket, 4 vCPU each core"""
  srv1.set_vcpu_max_capacity(2*14*4)
  """Server with 12 sticks of 16GB of RAM"""
  srv1.set_ram_max_capacity(12*16)
  rack1.add_server(srv1)

  srv2 = sim_server("dc2_rack2_srv2")
  srv2.set_server_capability(["vm"])
  rack2.add_server(srv2)

  srv3 = sim_server("dc1_rack1_srv3")
  srv3.set_server_capability(["vm"])
  """Server with 1 sockets, 1 cores each socket, 4 vCPU each core"""
  srv3.set_vcpu_max_capacity(1*1*4)
  """Server with 1 stick of 16GB of RAM"""
  srv3.set_ram_max_capacity(16)
  rack1.add_server(srv3)

  vm1 = sim_vm("dc1_vm1", 8, 16)
  vm1.add_logical_object_in_dc(dc1)
  vm1.set_vm_capability(["container"])

  """cont1 = sim_container("dc1_cont1", 1, 1)
  cont1.add_logical_object_in_dc(dc1)"""

  print(dc1)
  print(dc2)

def usage():
  return ''

def main():
  """#default, can be overriden
  config_file_path = 'example.cfg'

  try:
    opts, args = getopt.getopt(sys.argv[1:], 'c:h', ['config=', 'help'])
  except getopt.GetoptError:
    usage()
    sys.exit(2)

  for o, a in opts:
    if o in ('-c', '--config'):
      config_file_path = a
    if o in ('-h', '--help'):
      usage()

  config = configparser.RawConfigParser()
  config.read(config_file_path)
  test = [config.get('Test', 'Test_output')]"""

  init_infrastructure()

  sys.exit()

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
pgoapi - Pokemon Go API
Copyright (c) 2016 tjado <https://github.com/tejado>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.

Author: tjado <https://github.com/tejado>
"""

import os
import re
import json
import requests
import argparse
import time
import ssl
import logging
import sys

if sys.version_info >= (2, 7, 9):
    ssl._create_default_https_context = ssl._create_unverified_context

from bot import PokemonGoBot

def init_config():
    parser = argparse.ArgumentParser()
    config_file = "config.json"

    # If config file exists, load variables from json
    load   = {}
    if os.path.isfile(config_file):
        with open(config_file) as data:
            load.update(json.load(data))

    # Read passed in Arguments
    required = lambda x: not x in load
    parser.add_argument("-a", "--auth_service", help="Auth Service ('ptc' or 'google')",
        required=required("auth_service"))
    parser.add_argument("-u", "--username", help="Username", required=required("username"))
    parser.add_argument("-p", "--password", help="Password", required=required("password"))
    parser.add_argument("-l", "--location", help="Location", required=required("location"))
    parser.add_argument("-s", "--spinstop", help="SpinPokeStop",action='store_true')
    parser.add_argument("-w", "--walk", help="Walk instead of teleport with given speed (meters per second, e.g. 2.5)", type=float, default=0)
    parser.add_argument("-c", "--cp",help="Set CP less than to transfer(DEFAULT 100)",type=int,default=100)
    parser.add_argument("-k", "--gmapkey",help="Set Google Maps API KEY",type=str,default=None)
    parser.add_argument("-d", "--debug", help="Debug Mode", action='store_true')
    parser.add_argument("-t", "--test", help="Only parse the specified location", action='store_true')
    parser.set_defaults(DEBUG=False, TEST=False)
    config = parser.parse_args()

    # Passed in arguments shoud trump
    for key in config.__dict__:
        if key in load and config.__dict__[key] == None:
            config.__dict__[key] = load[key]

    if config.auth_service not in ['ptc', 'google']:
      log.error("Invalid Auth service specified! ('ptc' or 'google')")
      return None

    return config

def main():
    # log settings
    # log format
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(module)10s] [%(levelname)5s] %(message)s')
    # log level for http request class
    logging.getLogger("requests").setLevel(logging.WARNING)
    # log level for main pgoapi class
    logging.getLogger("pgoapi").setLevel(logging.INFO)
    # log level for internal pgoapi class
    logging.getLogger("rpc_api").setLevel(logging.INFO)

    config = init_config()
    if not config:
        return

    if config.debug:
        logging.getLogger("requests").setLevel(logging.DEBUG)
        logging.getLogger("pgoapi").setLevel(logging.DEBUG)
        logging.getLogger("rpc_api").setLevel(logging.DEBUG)

    bot = PokemonGoBot(config)
    bot.start()

    while(True):
        try:
            bot.take_step()
        except:
            print('Exception in bot, have a rest')
            time.sleep(2)

if __name__ == '__main__':
    main()
<commit_msg>Douglas Camata @douglascamata 17:29 catching all exception is really bad<commit_after>#!/usr/bin/env python
"""
pgoapi - Pokemon Go API
Copyright (c) 2016 tjado <https://github.com/tejado>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.

Author: tjado <https://github.com/tejado>
"""

import os
import re
import json
import requests
import argparse
import time
import ssl
import logging
import sys

if sys.version_info >= (2, 7, 9):
    ssl._create_default_https_context = ssl._create_unverified_context

from bot import PokemonGoBot

def init_config():
    parser = argparse.ArgumentParser()
    config_file = "config.json"

    # If config file exists, load variables from json
    load   = {}
    if os.path.isfile(config_file):
        with open(config_file) as data:
            load.update(json.load(data))

    # Read passed in Arguments
    required = lambda x: not x in load
    parser.add_argument("-a", "--auth_service", help="Auth Service ('ptc' or 'google')",
        required=required("auth_service"))
    parser.add_argument("-u", "--username", help="Username", required=required("username"))
    parser.add_argument("-p", "--password", help="Password", required=required("password"))
    parser.add_argument("-l", "--location", help="Location", required=required("location"))
    parser.add_argument("-s", "--spinstop", help="SpinPokeStop",action='store_true')
    parser.add_argument("-w", "--walk", help="Walk instead of teleport with given speed (meters per second, e.g. 2.5)", type=float, default=0)
    parser.add_argument("-c", "--cp",help="Set CP less than to transfer(DEFAULT 100)",type=int,default=100)
    parser.add_argument("-k", "--gmapkey",help="Set Google Maps API KEY",type=str,default=None)
    parser.add_argument("-d", "--debug", help="Debug Mode", action='store_true')
    parser.add_argument("-t", "--test", help="Only parse the specified location", action='store_true')
    parser.set_defaults(DEBUG=False, TEST=False)
    config = parser.parse_args()

    # Passed in arguments shoud trump
    for key in config.__dict__:
        if key in load and config.__dict__[key] == None:
            config.__dict__[key] = load[key]

    if config.auth_service not in ['ptc', 'google']:
      log.error("Invalid Auth service specified! ('ptc' or 'google')")
      return None

    return config

def main():
    # log settings
    # log format
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(module)10s] [%(levelname)5s] %(message)s')
    # log level for http request class
    logging.getLogger("requests").setLevel(logging.WARNING)
    # log level for main pgoapi class
    logging.getLogger("pgoapi").setLevel(logging.INFO)
    # log level for internal pgoapi class
    logging.getLogger("rpc_api").setLevel(logging.INFO)

    config = init_config()
    if not config:
        return

    if config.debug:
        logging.getLogger("requests").setLevel(logging.DEBUG)
        logging.getLogger("pgoapi").setLevel(logging.DEBUG)
        logging.getLogger("rpc_api").setLevel(logging.DEBUG)

    bot = PokemonGoBot(config)
    bot.start()

    while(True):
        bot.take_step()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from operator import and_, or_

from django.db.models import Q as DjangoQ
from django.db.models import FieldDoesNotExist

from modularodm import Q as MODMQ
from modularodm.query import query, QueryGroup

class BaseQ(object):
    def __or__(self, other):
        return OrQ(self, other)

    def __and__(self, other):
        return AndQ(self, other)

class CompoundQ(BaseQ, query.QueryGroup):
    @property
    def nodes(self):
        return self.__queries

    def __init__(self, *queries):
        self.__queries = queries

    def __repr__(self):
        return '<{0}({1})>'.format(
            self.__class__.__name__,
            ', '.join(repr(node) for node in self.nodes)
        )

    @classmethod
    def from_modm_query(cls, query, model_cls=None):
        op_function = and_ if query.operator == 'and' else or_
        return reduce(op_function, (Q.from_modm_query(node, model_cls) for node in query.nodes))

class AndQ(CompoundQ):
    operator = 'and'

    def __and__(self, other):
        return AndQ(other, *self.nodes)

    def to_django_query(self):
        return reduce(lambda acc, val: acc & val, (q.to_django_query() for q in self.nodes))

class OrQ(CompoundQ):
    operator = 'or'

    def __or__(self, other):
        return OrQ(other, *self.nodes)

    def to_django_query(self):
        return reduce(lambda acc, val: acc | val, (q.to_django_query() for q in self.nodes))

class Q(BaseQ, query.RawQuery):
    QUERY_MAP = {'eq': 'exact'}

    @classmethod
    def from_modm_query(cls, query, model_cls=None):
        from django.contrib.contenttypes.models import ContentType

        if isinstance(query, QueryGroup):
            compound_cls = AndQ if query.operator == 'and' else OrQ
            return compound_cls.from_modm_query(query, model_cls=model_cls)
        elif isinstance(query, MODMQ):
            attribute = query.attribute
            if attribute == 'referent':
                # if it's a referent they must have passed an instance
                return cls('object_id', 'eq', query.argument.id) & cls('content_type', 'eq',
                                                                       ContentType.objects.get_for_model(
                                                                           query.argument))
            if model_cls:
                field_aliases = getattr(model_cls, 'FIELD_ALIASES', {})
                attribute = field_aliases.get(attribute, attribute)
                field = _get_field(model_cls, attribute)
                internal_type = _get_internal_type(field)
                # Mongo compatibility fix: an 'eq' query on array fields
                # behaves like 'contains' for postgres ArrayFields
                if (
                    internal_type == 'ArrayField' and
                    query.operator == 'eq'
                ):
                    return cls(attribute, 'contains', [query.argument])
                # Queries like Q('tags', 'eq', []) should be translated to
                # Q('tags', 'isnull', True)
                elif internal_type == 'ManyToManyField' and query.argument in (list(), tuple()):
                    is_null = query.operator == 'eq'
                    return cls(attribute, 'isnull', is_null)
            return cls(attribute, query.operator, query.argument)
        elif isinstance(query, cls):
            if query.attribute == 'referent':
                # if it's a referent they must have passed an instance
                return cls('object_id', 'eq', query.argument.pk) & cls('content_type', 'eq',
                                                                       ContentType.objects.get_for_model(
                                                                           query.argument))
            if model_cls:
                field_aliases = getattr(model_cls, 'FIELD_ALIASES', {})
                if query.attribute in field_aliases:
                    return cls(field_aliases[query.attribute], query.operator, query.argument)
            return query
        else:
            raise ValueError(
                'from_modm_query must receive either a modularodm.Q, modularodm.query.QueryGroup, '
                'or osf.modm_compat.Q object'
            )

    @property
    def operator(self):
        return self.__op

    @property
    def attribute(self):
        return self.__key

    @property
    def argument(self):
        return self.__val

    @property
    def op(self):
        if self.__val is None:
            return 'isnull'
        return self.QUERY_MAP.get(self.__op, self.__op)

    @property
    def key(self):
        return self.__key

    @property
    def val(self):
        if self.__val is None:
            return True if self.__op == 'eq' else False
        return self.__val

    def __init__(self, key, op, val):
        self.__op = op
        self.__key = key
        self.__val = val

    def to_django_query(self):
        if self.op == 'ne':
            return ~DjangoQ(**{'__'.join(self.key.split('.')): self.val})
        return DjangoQ(**{'__'.join(self.key.split('.') + [self.op]): self.val})

    def __repr__(self):
        return '<Q({}, {}, {})>'.format(self.key, self.op, self.val)

def _get_field(model_cls, field_name):
    try:
        return model_cls._meta.get_field(field_name)
    except FieldDoesNotExist:
        return None

def _get_internal_type(field):
    # NOTE: GenericForeignKey does not implement get_internal_type
    if hasattr(field, 'get_internal_type'):
        return field.get_internal_type()
    else:
        return None

def to_django_query(query, model_cls=None):
    """Translate a modular-odm Q or QueryGroup to a Django query.
    """
    return Q.from_modm_query(query, model_cls=model_cls).to_django_query()
<commit_msg>Add temporary fix for modm_compat to account for between stages<commit_after># -*- coding: utf-8 -*-
from operator import and_, or_

from django.db.models import Q as DjangoQ
from django.db.models import FieldDoesNotExist

from modularodm import Q as MODMQ
from modularodm.query import query, QueryGroup

class BaseQ(object):
    def __or__(self, other):
        return OrQ(self, other)

    def __and__(self, other):
        return AndQ(self, other)

class CompoundQ(BaseQ, query.QueryGroup):
    @property
    def nodes(self):
        return self.__queries

    def __init__(self, *queries):
        self.__queries = queries

    def __repr__(self):
        return '<{0}({1})>'.format(
            self.__class__.__name__,
            ', '.join(repr(node) for node in self.nodes)
        )

    @classmethod
    def from_modm_query(cls, query, model_cls=None):
        op_function = and_ if query.operator == 'and' else or_
        return reduce(op_function, (Q.from_modm_query(node, model_cls) for node in query.nodes))

class AndQ(CompoundQ):
    operator = 'and'

    def __and__(self, other):
        return AndQ(other, *self.nodes)

    def to_django_query(self):
        return reduce(lambda acc, val: acc & val, (q.to_django_query() for q in self.nodes))

class OrQ(CompoundQ):
    operator = 'or'

    def __or__(self, other):
        return OrQ(other, *self.nodes)

    def to_django_query(self):
        return reduce(lambda acc, val: acc | val, (q.to_django_query() for q in self.nodes))

class Q(BaseQ, query.RawQuery):
    QUERY_MAP = {'eq': 'exact'}

    @classmethod
    def from_modm_query(cls, query, model_cls=None):
        from django.contrib.contenttypes.models import ContentType

        if isinstance(query, QueryGroup):
            compound_cls = AndQ if query.operator == 'and' else OrQ
            return compound_cls.from_modm_query(query, model_cls=model_cls)
        elif isinstance(query, MODMQ):
            attribute = query.attribute
            if attribute == 'referent':
                # if it's a referent they must have passed an instance
                return cls('object_id', 'eq', query.argument.id) & cls('content_type', 'eq',
                                                                       ContentType.objects.get_for_model(
                                                                           query.argument))
            if model_cls:
                field_aliases = getattr(model_cls, 'FIELD_ALIASES', {})
                attribute = field_aliases.get(attribute, attribute)
                field = _get_field(model_cls, attribute)
                internal_type = _get_internal_type(field)
                # Mongo compatibility fix: an 'eq' query on array fields
                # behaves like 'contains' for postgres ArrayFields
                if (
                    internal_type == 'ArrayField' and
                    query.operator == 'eq'
                ):
                    return cls(attribute, 'contains', [query.argument])
                # Queries like Q('tags', 'eq', []) should be translated to
                # Q('tags', 'isnull', True)
                elif internal_type == 'ManyToManyField' and query.argument in (list(), tuple()):
                    is_null = query.operator == 'eq'
                    return cls(attribute, 'isnull', is_null)
            return cls(attribute, query.operator, query.argument)
        elif isinstance(query, cls):
            if query.attribute == 'referent':
                # if it's a referent they must have passed an instance
                return cls('object_id', 'eq', query.argument.pk) & cls('content_type', 'eq',
                                                                       ContentType.objects.get_for_model(
                                                                           query.argument))
            if model_cls:
                field_aliases = getattr(model_cls, 'FIELD_ALIASES', {})
                if query.attribute in field_aliases:
                    return cls(field_aliases[query.attribute], query.operator, query.argument)
            return query
        else:
            raise ValueError(
                'from_modm_query must receive either a modularodm.Q, modularodm.query.QueryGroup, '
                'or osf.modm_compat.Q object'
            )

    @property
    def operator(self):
        return self.__op

    @property
    def attribute(self):
        return self.__key

    @property
    def argument(self):
        return self.__val

    @property
    def op(self):
        if self.__val is None:
            return 'isnull'
        return self.QUERY_MAP.get(self.__op, self.__op)

    @property
    def key(self):
        return self.__key

    @property
    def val(self):
        if self.__val is None:
            return True if self.__op == 'eq' else False
        return self.__val

    def __init__(self, key, op, val):
        self.__op = op
        self.__key = key
        self.__val = val

    def to_django_query(self):
        if self.op == 'ne':
            return ~DjangoQ(**{'__'.join(self.key.split('.')): self.val})
        return DjangoQ(**{'__'.join(self.key.split('.') + [self.op]): self.val})

    def __repr__(self):
        return '<Q({}, {}, {})>'.format(self.key, self.op, self.val)

def _get_field(model_cls, field_name):
    try:
        return model_cls._meta.get_field(field_name)
    except FieldDoesNotExist:
        return None

def _get_internal_type(field):
    # NOTE: GenericForeignKey does not implement get_internal_type
    if hasattr(field, 'get_internal_type'):
        return field.get_internal_type()
    else:
        return None

def to_django_query(query, model_cls=None):
    """Translate a modular-odm Q or QueryGroup to a Django query.
    """
    # temporary measure as queries are converted away from modm
    if isinstance(query, DjangoQ):
        return query
    return Q.from_modm_query(query, model_cls=model_cls).to_django_query()
<|endoftext|>


<commit_before># Authors: Nicolas Tresegnie <nicolas.tresegnie@gmail.com>
# License: BSD 3 clause

import warnings

import numpy as np
import numpy.ma as ma
from scipy import sparse
from scipy import stats

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_array
from ..utils.fixes import astype
from ..utils.sparsefuncs import _get_median
from ..utils.validation import check_is_fitted
from ..utils.validation import FLOAT_DTYPES

from ..externals import six

zip = six.moves.zip
map = six.moves.map

__all__ = [
    'Imputer',
]

def _get_mask(X, value_to_mask):
    """Compute the boolean mask X == missing_values."""
    if value_to_mask == "NaN" or np.isnan(value_to_mask):
        return np.isnan(X)
    else:
        return X == value_to_mask

def _most_frequent(array, extra_value, n_repeat):
    """Compute the most frequent value in a 1d array extended with
       [extra_value] * n_repeat, where extra_value is assumed to be not part
       of the array."""
    # Compute the most frequent value in array only
    if array.size > 0:
        mode = stats.mode(array)
        most_frequent_value = mode[0][0]
        most_frequent_count = mode[1][0]
    else:
        most_frequent_value = 0
        most_frequent_count = 0

    # Compare to array + [extra_value] * n_repeat
    if most_frequent_count == 0 and n_repeat == 0:
        return np.nan
    elif most_frequent_count < n_repeat:
        return extra_value
    elif most_frequent_count > n_repeat:
        return most_frequent_value
    elif most_frequent_count == n_repeat:
        # Ties the breaks. Copy the behaviour of scipy.stats.mode
        if most_frequent_value < extra_value:
            return most_frequent_value
        else:
            return extra_value

class Imputer(BaseEstimator, TransformerMixin):
    """Imputation transformer for completing missing values.

    Read more in the :ref:`User Guide <imputation>`.

    Parameters
    ----------
    missing_values : integer or "NaN", optional (default="NaN")
        The placeholder for the missing values. All occurrences of
        `missing_values` will be imputed. For missing values encoded as np.nan,
        use the string value "NaN".

    strategy : string, optional (default="mean")
        The imputation strategy.

        - If "mean", then replace missing values using the mean along
          the axis.
        - If "median", then replace missing values using the median along
          the axis.
        - If "most_frequent", then replace missing using the most frequent
          value along the axis.

    axis : integer, optional (default=0)
        The axis along which to impute.

        - If `axis=0`, then impute along columns.
        - If `axis=1`, then impute along rows.

    verbose : integer, optional (default=0)
        Controls the verbosity of the imputer.

    copy : boolean, optional (default=True)
        If True, a copy of X will be created. If False, imputation will
        be done in-place whenever possible. Note that, in the following cases,
        a new copy will always be made, even if `copy=False`:

        - If X is not an array of floating values;
        - If X is sparse and `missing_values=0`;
        - If `axis=0` and X is encoded as a CSR matrix;
        - If `axis=1` and X is encoded as a CSC matrix.

    Attributes
    ----------
    statistics_ : array of shape (n_features,)
        The imputation fill value for each feature if axis == 0.

    Notes
    -----
    - When ``axis=0``, columns which only contained missing values at `fit`
      are discarded upon `transform`.
    - When ``axis=1``, an exception is raised if there are rows for which it is
      not possible to fill in the missing values (e.g., because they only
      contain missing values).
    """
    def __init__(self, missing_values="NaN", strategy="mean",
                 axis=0, verbose=0, copy=True):
        self.missing_values = missing_values
        self.strategy = strategy
        self.axis = axis
        self.verbose = verbose
        self.copy = copy

    def fit(self, X, y=None):
        """Fit the imputer on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Input data, where ``n_samples`` is the number of samples and
            ``n_features`` is the number of features.

        Returns
        -------
        self : object
            Returns self.
        """
        # Check parameters
        allowed_strategies = ["mean", "median", "most_frequent"]
        if self.strategy not in allowed_strategies:
            raise ValueError("Can only use these strategies: {0} "
                             " got strategy={1}".format(allowed_strategies,
                                                        self.strategy))

        if self.axis not in [0, 1]:
            raise ValueError("Can only impute missing values on axis 0 and 1, "
                             " got axis={0}".format(self.axis))

        # Since two different arrays can be provided in fit(X) and
        # transform(X), the imputation data will be computed in transform()
        # when the imputation is done per sample (i.e., when axis=1).
        if self.axis == 0:
            X = check_array(X, accept_sparse='csc', dtype=np.float64,
                            force_all_finite=False)

            if sparse.issparse(X):
                self.statistics_ = self._sparse_fit(X,
                                                    self.strategy,
                                                    self.missing_values,
                                                    self.axis)
            else:
                self.statistics_ = self._dense_fit(X,
                                                   self.strategy,
                                                   self.missing_values,
                                                   self.axis)

        return self

    def _sparse_fit(self, X, strategy, missing_values, axis):
        """Fit the transformer on sparse data."""
        # Imputation is done "by column", so if we want to do it
        # by row we only need to convert the matrix to csr format.
        if axis == 1:
            X = X.tocsr()
        else:
            X = X.tocsc()

        # Count the zeros
        if missing_values == 0:
            n_zeros_axis = np.zeros(X.shape[not axis], dtype=int)
        else:
            n_zeros_axis = X.shape[axis] - np.diff(X.indptr)

        # Mean
        if strategy == "mean":
            if missing_values != 0:
                n_non_missing = n_zeros_axis

                # Mask the missing elements
                mask_missing_values = _get_mask(X.data, missing_values)
                mask_valids = np.logical_not(mask_missing_values)

                # Sum only the valid elements
                new_data = X.data.copy()
                new_data[mask_missing_values] = 0
                X = sparse.csc_matrix((new_data, X.indices, X.indptr),
                                      copy=False)
                sums = X.sum(axis=0)

                # Count the elements != 0
                mask_non_zeros = sparse.csc_matrix(
                    (mask_valids.astype(np.float64),
                     X.indices,
                     X.indptr), copy=False)
                s = mask_non_zeros.sum(axis=0)
                n_non_missing = np.add(n_non_missing, s)

            else:
                sums = X.sum(axis=axis)
                n_non_missing = np.diff(X.indptr)

            # Ignore the error, columns with a np.nan statistics_
            # are not an error at this point. These columns will
            # be removed in transform
            with np.errstate(all="ignore"):
                return np.ravel(sums) / np.ravel(n_non_missing)

        # Median + Most frequent
        else:
            # Remove the missing values, for each column
            columns_all = np.hsplit(X.data, X.indptr[1:-1])
            mask_missing_values = _get_mask(X.data, missing_values)
            mask_valids = np.hsplit(np.logical_not(mask_missing_values),
                                    X.indptr[1:-1])

            # astype necessary for bug in numpy.hsplit before v1.9
            columns = [col[astype(mask, bool, copy=False)]
                       for col, mask in zip(columns_all, mask_valids)]

            # Median
            if strategy == "median":
                median = np.empty(len(columns))
                for i, column in enumerate(columns):
                    median[i] = _get_median(column, n_zeros_axis[i])

                return median

            # Most frequent
            elif strategy == "most_frequent":
                most_frequent = np.empty(len(columns))

                for i, column in enumerate(columns):
                    most_frequent[i] = _most_frequent(column,
                                                      0,
                                                      n_zeros_axis[i])

                return most_frequent

    def _dense_fit(self, X, strategy, missing_values, axis):
        """Fit the transformer on dense data."""
        X = check_array(X, force_all_finite=False)
        mask = _get_mask(X, missing_values)
        masked_X = ma.masked_array(X, mask=mask)

        # Mean
        if strategy == "mean":
            mean_masked = np.ma.mean(masked_X, axis=axis)
            # Avoid the warning "Warning: converting a masked element to nan."
            mean = np.ma.getdata(mean_masked)
            mean[np.ma.getmask(mean_masked)] = np.nan

            return mean

        # Median
        elif strategy == "median":
            if tuple(int(v) for v in np.__version__.split('.')[:2]) < (1, 5):
                # In old versions of numpy, calling a median on an array
                # containing nans returns nan. This is different is
                # recent versions of numpy, which we want to mimic
                masked_X.mask = np.logical_or(masked_X.mask,
                                              np.isnan(X))
            median_masked = np.ma.median(masked_X, axis=axis)
            # Avoid the warning "Warning: converting a masked element to nan."
            median = np.ma.getdata(median_masked)
            median[np.ma.getmaskarray(median_masked)] = np.nan

            return median

        # Most frequent
        elif strategy == "most_frequent":
            # scipy.stats.mstats.mode cannot be used because it will no work
            # properly if the first element is masked and if its frequency
            # is equal to the frequency of the most frequent valid element
            # See https://github.com/scipy/scipy/issues/2636

            # To be able access the elements by columns
            if axis == 0:
                X = X.transpose()
                mask = mask.transpose()

            most_frequent = np.empty(X.shape[0])

            for i, (row, row_mask) in enumerate(zip(X[:], mask[:])):
                row_mask = np.logical_not(row_mask).astype(np.bool)
                row = row[row_mask]
                most_frequent[i] = _most_frequent(row, np.nan, 0)

            return most_frequent

    def transform(self, X):
        """Impute all missing values in X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            The input data to complete.
        """
        if self.axis == 0:
            check_is_fitted(self, 'statistics_')

        # Since two different arrays can be provided in fit(X) and
        # transform(X), the imputation data need to be recomputed
        # when the imputation is done per sample
        if self.axis == 1:
            X = check_array(X, accept_sparse='csr', dtype=FLOAT_DTYPES,
                            force_all_finite=False, copy=self.copy)

            if sparse.issparse(X):
                statistics = self._sparse_fit(X,
                                              self.strategy,
                                              self.missing_values,
                                              self.axis)

            else:
                statistics = self._dense_fit(X,
                                             self.strategy,
                                             self.missing_values,
                                             self.axis)
        else:
            X = check_array(X, accept_sparse='csc', dtype=FLOAT_DTYPES,
                            force_all_finite=False, copy=self.copy)
            statistics = self.statistics_

        # Delete the invalid rows/columns
        invalid_mask = np.isnan(statistics)
        valid_mask = np.logical_not(invalid_mask)
        valid_statistics = statistics[valid_mask]
        valid_statistics_indexes = np.where(valid_mask)[0]
        missing = np.arange(X.shape[not self.axis])[invalid_mask]

        if self.axis == 0 and invalid_mask.any():
            if self.verbose:
                warnings.warn("Deleting features without "
                              "observed values: %s" % missing)
            X = X[:, valid_statistics_indexes]
        elif self.axis == 1 and invalid_mask.any():
            raise ValueError("Some rows only contain "
                             "missing values: %s" % missing)

        # Do actual imputation
        if sparse.issparse(X) and self.missing_values != 0:
            mask = _get_mask(X.data, self.missing_values)
            indexes = np.repeat(np.arange(len(X.indptr) - 1, dtype=np.int),
                                np.diff(X.indptr))[mask]

            X.data[mask] = astype(valid_statistics[indexes], X.dtype,
                                  copy=False)
        else:
            if sparse.issparse(X):
                X = X.toarray()

            mask = _get_mask(X, self.missing_values)
            n_missing = np.sum(mask, axis=self.axis)
            values = np.repeat(valid_statistics, n_missing)

            if self.axis == 0:
                coordinates = np.where(mask.transpose())[::-1]
            else:
                coordinates = mask

            X[coordinates] = values

        return X
<commit_msg>[MRG] Fail imputer early when number of features are not the same in fit and transform (#7374)<commit_after># Authors: Nicolas Tresegnie <nicolas.tresegnie@gmail.com>
# License: BSD 3 clause

import warnings

import numpy as np
import numpy.ma as ma
from scipy import sparse
from scipy import stats

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_array
from ..utils.fixes import astype
from ..utils.sparsefuncs import _get_median
from ..utils.validation import check_is_fitted
from ..utils.validation import FLOAT_DTYPES

from ..externals import six

zip = six.moves.zip
map = six.moves.map

__all__ = [
    'Imputer',
]

def _get_mask(X, value_to_mask):
    """Compute the boolean mask X == missing_values."""
    if value_to_mask == "NaN" or np.isnan(value_to_mask):
        return np.isnan(X)
    else:
        return X == value_to_mask

def _most_frequent(array, extra_value, n_repeat):
    """Compute the most frequent value in a 1d array extended with
       [extra_value] * n_repeat, where extra_value is assumed to be not part
       of the array."""
    # Compute the most frequent value in array only
    if array.size > 0:
        mode = stats.mode(array)
        most_frequent_value = mode[0][0]
        most_frequent_count = mode[1][0]
    else:
        most_frequent_value = 0
        most_frequent_count = 0

    # Compare to array + [extra_value] * n_repeat
    if most_frequent_count == 0 and n_repeat == 0:
        return np.nan
    elif most_frequent_count < n_repeat:
        return extra_value
    elif most_frequent_count > n_repeat:
        return most_frequent_value
    elif most_frequent_count == n_repeat:
        # Ties the breaks. Copy the behaviour of scipy.stats.mode
        if most_frequent_value < extra_value:
            return most_frequent_value
        else:
            return extra_value

class Imputer(BaseEstimator, TransformerMixin):
    """Imputation transformer for completing missing values.

    Read more in the :ref:`User Guide <imputation>`.

    Parameters
    ----------
    missing_values : integer or "NaN", optional (default="NaN")
        The placeholder for the missing values. All occurrences of
        `missing_values` will be imputed. For missing values encoded as np.nan,
        use the string value "NaN".

    strategy : string, optional (default="mean")
        The imputation strategy.

        - If "mean", then replace missing values using the mean along
          the axis.
        - If "median", then replace missing values using the median along
          the axis.
        - If "most_frequent", then replace missing using the most frequent
          value along the axis.

    axis : integer, optional (default=0)
        The axis along which to impute.

        - If `axis=0`, then impute along columns.
        - If `axis=1`, then impute along rows.

    verbose : integer, optional (default=0)
        Controls the verbosity of the imputer.

    copy : boolean, optional (default=True)
        If True, a copy of X will be created. If False, imputation will
        be done in-place whenever possible. Note that, in the following cases,
        a new copy will always be made, even if `copy=False`:

        - If X is not an array of floating values;
        - If X is sparse and `missing_values=0`;
        - If `axis=0` and X is encoded as a CSR matrix;
        - If `axis=1` and X is encoded as a CSC matrix.

    Attributes
    ----------
    statistics_ : array of shape (n_features,)
        The imputation fill value for each feature if axis == 0.

    Notes
    -----
    - When ``axis=0``, columns which only contained missing values at `fit`
      are discarded upon `transform`.
    - When ``axis=1``, an exception is raised if there are rows for which it is
      not possible to fill in the missing values (e.g., because they only
      contain missing values).
    """
    def __init__(self, missing_values="NaN", strategy="mean",
                 axis=0, verbose=0, copy=True):
        self.missing_values = missing_values
        self.strategy = strategy
        self.axis = axis
        self.verbose = verbose
        self.copy = copy

    def fit(self, X, y=None):
        """Fit the imputer on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Input data, where ``n_samples`` is the number of samples and
            ``n_features`` is the number of features.

        Returns
        -------
        self : object
            Returns self.
        """
        # Check parameters
        allowed_strategies = ["mean", "median", "most_frequent"]
        if self.strategy not in allowed_strategies:
            raise ValueError("Can only use these strategies: {0} "
                             " got strategy={1}".format(allowed_strategies,
                                                        self.strategy))

        if self.axis not in [0, 1]:
            raise ValueError("Can only impute missing values on axis 0 and 1, "
                             " got axis={0}".format(self.axis))

        # Since two different arrays can be provided in fit(X) and
        # transform(X), the imputation data will be computed in transform()
        # when the imputation is done per sample (i.e., when axis=1).
        if self.axis == 0:
            X = check_array(X, accept_sparse='csc', dtype=np.float64,
                            force_all_finite=False)

            if sparse.issparse(X):
                self.statistics_ = self._sparse_fit(X,
                                                    self.strategy,
                                                    self.missing_values,
                                                    self.axis)
            else:
                self.statistics_ = self._dense_fit(X,
                                                   self.strategy,
                                                   self.missing_values,
                                                   self.axis)

        return self

    def _sparse_fit(self, X, strategy, missing_values, axis):
        """Fit the transformer on sparse data."""
        # Imputation is done "by column", so if we want to do it
        # by row we only need to convert the matrix to csr format.
        if axis == 1:
            X = X.tocsr()
        else:
            X = X.tocsc()

        # Count the zeros
        if missing_values == 0:
            n_zeros_axis = np.zeros(X.shape[not axis], dtype=int)
        else:
            n_zeros_axis = X.shape[axis] - np.diff(X.indptr)

        # Mean
        if strategy == "mean":
            if missing_values != 0:
                n_non_missing = n_zeros_axis

                # Mask the missing elements
                mask_missing_values = _get_mask(X.data, missing_values)
                mask_valids = np.logical_not(mask_missing_values)

                # Sum only the valid elements
                new_data = X.data.copy()
                new_data[mask_missing_values] = 0
                X = sparse.csc_matrix((new_data, X.indices, X.indptr),
                                      copy=False)
                sums = X.sum(axis=0)

                # Count the elements != 0
                mask_non_zeros = sparse.csc_matrix(
                    (mask_valids.astype(np.float64),
                     X.indices,
                     X.indptr), copy=False)
                s = mask_non_zeros.sum(axis=0)
                n_non_missing = np.add(n_non_missing, s)

            else:
                sums = X.sum(axis=axis)
                n_non_missing = np.diff(X.indptr)

            # Ignore the error, columns with a np.nan statistics_
            # are not an error at this point. These columns will
            # be removed in transform
            with np.errstate(all="ignore"):
                return np.ravel(sums) / np.ravel(n_non_missing)

        # Median + Most frequent
        else:
            # Remove the missing values, for each column
            columns_all = np.hsplit(X.data, X.indptr[1:-1])
            mask_missing_values = _get_mask(X.data, missing_values)
            mask_valids = np.hsplit(np.logical_not(mask_missing_values),
                                    X.indptr[1:-1])

            # astype necessary for bug in numpy.hsplit before v1.9
            columns = [col[astype(mask, bool, copy=False)]
                       for col, mask in zip(columns_all, mask_valids)]

            # Median
            if strategy == "median":
                median = np.empty(len(columns))
                for i, column in enumerate(columns):
                    median[i] = _get_median(column, n_zeros_axis[i])

                return median

            # Most frequent
            elif strategy == "most_frequent":
                most_frequent = np.empty(len(columns))

                for i, column in enumerate(columns):
                    most_frequent[i] = _most_frequent(column,
                                                      0,
                                                      n_zeros_axis[i])

                return most_frequent

    def _dense_fit(self, X, strategy, missing_values, axis):
        """Fit the transformer on dense data."""
        X = check_array(X, force_all_finite=False)
        mask = _get_mask(X, missing_values)
        masked_X = ma.masked_array(X, mask=mask)

        # Mean
        if strategy == "mean":
            mean_masked = np.ma.mean(masked_X, axis=axis)
            # Avoid the warning "Warning: converting a masked element to nan."
            mean = np.ma.getdata(mean_masked)
            mean[np.ma.getmask(mean_masked)] = np.nan

            return mean

        # Median
        elif strategy == "median":
            if tuple(int(v) for v in np.__version__.split('.')[:2]) < (1, 5):
                # In old versions of numpy, calling a median on an array
                # containing nans returns nan. This is different is
                # recent versions of numpy, which we want to mimic
                masked_X.mask = np.logical_or(masked_X.mask,
                                              np.isnan(X))
            median_masked = np.ma.median(masked_X, axis=axis)
            # Avoid the warning "Warning: converting a masked element to nan."
            median = np.ma.getdata(median_masked)
            median[np.ma.getmaskarray(median_masked)] = np.nan

            return median

        # Most frequent
        elif strategy == "most_frequent":
            # scipy.stats.mstats.mode cannot be used because it will no work
            # properly if the first element is masked and if its frequency
            # is equal to the frequency of the most frequent valid element
            # See https://github.com/scipy/scipy/issues/2636

            # To be able access the elements by columns
            if axis == 0:
                X = X.transpose()
                mask = mask.transpose()

            most_frequent = np.empty(X.shape[0])

            for i, (row, row_mask) in enumerate(zip(X[:], mask[:])):
                row_mask = np.logical_not(row_mask).astype(np.bool)
                row = row[row_mask]
                most_frequent[i] = _most_frequent(row, np.nan, 0)

            return most_frequent

    def transform(self, X):
        """Impute all missing values in X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            The input data to complete.
        """
        if self.axis == 0:
            check_is_fitted(self, 'statistics_')
            X = check_array(X, accept_sparse='csc', dtype=FLOAT_DTYPES,
                            force_all_finite=False, copy=self.copy)
            statistics = self.statistics_
            if X.shape[1] != statistics.shape[0]:
                raise ValueError("X has %d features per sample, expected %d"
                                 % (X.shape[1], self.statistics_.shape[0]))

        # Since two different arrays can be provided in fit(X) and
        # transform(X), the imputation data need to be recomputed
        # when the imputation is done per sample
        else:
            X = check_array(X, accept_sparse='csr', dtype=FLOAT_DTYPES,
                            force_all_finite=False, copy=self.copy)

            if sparse.issparse(X):
                statistics = self._sparse_fit(X,
                                              self.strategy,
                                              self.missing_values,
                                              self.axis)

            else:
                statistics = self._dense_fit(X,
                                             self.strategy,
                                             self.missing_values,
                                             self.axis)

        # Delete the invalid rows/columns
        invalid_mask = np.isnan(statistics)
        valid_mask = np.logical_not(invalid_mask)
        valid_statistics = statistics[valid_mask]
        valid_statistics_indexes = np.where(valid_mask)[0]
        missing = np.arange(X.shape[not self.axis])[invalid_mask]

        if self.axis == 0 and invalid_mask.any():
            if self.verbose:
                warnings.warn("Deleting features without "
                              "observed values: %s" % missing)
            X = X[:, valid_statistics_indexes]
        elif self.axis == 1 and invalid_mask.any():
            raise ValueError("Some rows only contain "
                             "missing values: %s" % missing)

        # Do actual imputation
        if sparse.issparse(X) and self.missing_values != 0:
            mask = _get_mask(X.data, self.missing_values)
            indexes = np.repeat(np.arange(len(X.indptr) - 1, dtype=np.int),
                                np.diff(X.indptr))[mask]

            X.data[mask] = astype(valid_statistics[indexes], X.dtype,
                                  copy=False)
        else:
            if sparse.issparse(X):
                X = X.toarray()

            mask = _get_mask(X, self.missing_values)
            n_missing = np.sum(mask, axis=self.axis)
            values = np.repeat(valid_statistics, n_missing)

            if self.axis == 0:
                coordinates = np.where(mask.transpose())[::-1]
            else:
                coordinates = mask

            X[coordinates] = values

        return X
<|endoftext|>