<commit_before>import os
import sys
from django.db import models
from django.forms.models import model_to_dict
from django.core.urlresolvers import reverse
from django.forms.models import model_to_dict
from django.utils import timezone
from django.core.exceptions import PermissionDenied
import model_policy
from model_autodeletion import ephemeral_models

try:
    # This is a no-op if observer_disabled is set to 1 in the config file
    from observer import *
except:
    print >> sys.stderr, "import of observer failed! printing traceback and disabling observer:"
    import traceback
    traceback.print_exc()

    # guard against something failing
    def notify_observer(*args, **kwargs):
        pass

# This manager will be inherited by all subclasses because
# the core model is abstract.
class PlCoreBaseDeletionManager(models.Manager):
    def get_queryset(self):
        parent=super(PlCoreBaseDeletionManager, self)
        if hasattr(parent, "get_queryset"):
            return parent.get_queryset().filter(deleted=True)
        else:
            return parent.get_query_set().filter(deleted=True)

    # deprecated in django 1.7 in favor of get_queryset().
    def get_query_set(self):
        return self.get_queryset()

# This manager will be inherited by all subclasses because
# the core model is abstract.
class PlCoreBaseManager(models.Manager):
    def get_queryset(self):
        parent=super(PlCoreBaseManager, self)
        if hasattr(parent, "get_queryset"):
            return parent.get_queryset().filter(deleted=False)
        else:
            return parent.get_query_set().filter(deleted=False)

    # deprecated in django 1.7 in favor of get_queryset().
    def get_query_set(self):
        return self.get_queryset()

class DiffModelMixIn:
    # Provides useful methods for computing which objects in a model have
    # changed. Make sure to do self._initial = self._dict in the __init__
    # method.

    # This is broken out of PlCoreBase into a Mixin so the User model can
    # also make use of it.

    @property
    def _dict(self):
        return model_to_dict(self, fields=[field.name for field in
                             self._meta.fields])

    @property
    def diff(self):
        d1 = self._initial
        d2 = self._dict
        diffs = [(k, (v, d2[k])) for k, v in d1.items() if v != d2[k]]
        return dict(diffs)

    @property
    def has_changed(self):
        return bool(self.diff)

    @property
    def changed_fields(self):
        return self.diff.keys()

    def has_field_changed(self, field_name):
        return field_name in self.diff.keys()

    def get_field_diff(self, field_name):
        return self.diff.get(field_name, None)

class PlCoreBase(models.Model): # , DiffModelMixIn):
    objects = PlCoreBaseManager()
    deleted_objects = PlCoreBaseDeletionManager()

    # ---- copy stuff from DiffModelMixin ----

    # XXX Django fails miserably when trying to create initial migrations when
    #    DiffModelMixin is used. So, until we figure out what's wrong,
    #    just copied the guts of DiffModelMixIn here.

    @property
    def _dict(self):
        return model_to_dict(self, fields=[field.name for field in
                             self._meta.fields])

    @property
    def diff(self):
        d1 = self._initial
        d2 = self._dict
        diffs = [(k, (v, d2[k])) for k, v in d1.items() if v != d2[k]]
        return dict(diffs)

    @property
    def has_changed(self):
        return bool(self.diff)

    @property
    def changed_fields(self):
        return self.diff.keys()

    def has_field_changed(self, field_name):
        return field_name in self.diff.keys()

    def get_field_diff(self, field_name):
        return self.diff.get(field_name, None)
    # ---- end copy stuff from DiffModelMixin ----

    # default values for created and updated are only there to keep evolution
    # from failing.
    created = models.DateTimeField(auto_now_add=True, default=timezone.now)
    updated = models.DateTimeField(auto_now=True, default=timezone.now)
    enacted = models.DateTimeField(null=True, default=None)
    backend_status = models.CharField(max_length=140,
                                      default="Provisioning in progress")
    deleted = models.BooleanField(default=False)

    class Meta:
        # Changing abstract to False would require the managers of subclasses of
        # PlCoreBase to be customized individually.
        abstract = True
        app_label = "core"

    def __init__(self, *args, **kwargs):
        super(PlCoreBase, self).__init__(*args, **kwargs)
        self._initial = self._dict # for DiffModelMixIn
        self.silent = False

    def can_update(self, user):
        if user.is_readonly:
            return False
        if user.is_admin:
            return True

        return False

    def delete(self, *args, **kwds):
        # so we have something to give the observer
        purge = kwds.get('purge',False)
        if purge:
            del kwds['purge']
        silent = kwds.get('silent',False)
        if silent:
            del kwds['silent']
        try:
            purge = purge or observer_disabled
        except NameError:
            pass

        if (purge):
            super(PlCoreBase, self).delete(*args, **kwds)
        else:
            self.deleted = True
            self.enacted=None
            self.save(update_fields=['enacted','deleted'], silent=silent)

    def save(self, *args, **kwargs):
        # let the user specify silence as either a kwarg or an instance varible
        silent = self.silent
        if "silent" in kwargs:
            silent=silent or kwargs.pop("silent")

        super(PlCoreBase, self).save(*args, **kwargs)

        # This is a no-op if observer_disabled is set
        if not silent:
            notify_observer()

        self._initial = self._dict

    def save_by_user(self, user, *args, **kwds):
        if not self.can_update(user):
            if getattr(self, "_cant_update_fieldName", None) is not None:
                raise PermissionDenied("You do not have permission to update field %s on object %s" % (self._cant_update_fieldName, self.__class__.__name__))
            else:
                raise PermissionDenied("You do not have permission to update %s objects" % self.__class__.__name__)

        self.save(*args, **kwds)

    def delete_by_user(self, user, *args, **kwds):
        if not self.can_update(user):
            raise PermissionDenied("You do not have permission to delete %s objects" % self.__class__.__name__)
        self.delete(*args, **kwds)

    @classmethod
    def select_by_user(cls, user):
        # This should be overridden by descendant classes that want to perform
        # filtering of visible objects by user.
        return cls.objects.all()

    @classmethod
    def is_ephemeral(cls):
	return cls in ephemeral_models

<commit_msg>add getValidators function<commit_after>import os
import sys
from django.db import models
from django.forms.models import model_to_dict
from django.core.urlresolvers import reverse
from django.forms.models import model_to_dict
from django.utils import timezone
from django.core.exceptions import PermissionDenied
import model_policy
from model_autodeletion import ephemeral_models

try:
    # This is a no-op if observer_disabled is set to 1 in the config file
    from observer import *
except:
    print >> sys.stderr, "import of observer failed! printing traceback and disabling observer:"
    import traceback
    traceback.print_exc()

    # guard against something failing
    def notify_observer(*args, **kwargs):
        pass

# This manager will be inherited by all subclasses because
# the core model is abstract.
class PlCoreBaseDeletionManager(models.Manager):
    def get_queryset(self):
        parent=super(PlCoreBaseDeletionManager, self)
        if hasattr(parent, "get_queryset"):
            return parent.get_queryset().filter(deleted=True)
        else:
            return parent.get_query_set().filter(deleted=True)

    # deprecated in django 1.7 in favor of get_queryset().
    def get_query_set(self):
        return self.get_queryset()

# This manager will be inherited by all subclasses because
# the core model is abstract.
class PlCoreBaseManager(models.Manager):
    def get_queryset(self):
        parent=super(PlCoreBaseManager, self)
        if hasattr(parent, "get_queryset"):
            return parent.get_queryset().filter(deleted=False)
        else:
            return parent.get_query_set().filter(deleted=False)

    # deprecated in django 1.7 in favor of get_queryset().
    def get_query_set(self):
        return self.get_queryset()

class DiffModelMixIn:
    # Provides useful methods for computing which objects in a model have
    # changed. Make sure to do self._initial = self._dict in the __init__
    # method.

    # This is broken out of PlCoreBase into a Mixin so the User model can
    # also make use of it.

    @property
    def _dict(self):
        return model_to_dict(self, fields=[field.name for field in
                             self._meta.fields])

    @property
    def diff(self):
        d1 = self._initial
        d2 = self._dict
        diffs = [(k, (v, d2[k])) for k, v in d1.items() if v != d2[k]]
        return dict(diffs)

    @property
    def has_changed(self):
        return bool(self.diff)

    @property
    def changed_fields(self):
        return self.diff.keys()

    def has_field_changed(self, field_name):
        return field_name in self.diff.keys()

    def get_field_diff(self, field_name):
        return self.diff.get(field_name, None)

class PlCoreBase(models.Model): # , DiffModelMixIn):
    objects = PlCoreBaseManager()
    deleted_objects = PlCoreBaseDeletionManager()

    # ---- copy stuff from DiffModelMixin ----

    # XXX Django fails miserably when trying to create initial migrations when
    #    DiffModelMixin is used. So, until we figure out what's wrong,
    #    just copied the guts of DiffModelMixIn here.

    @property
    def _dict(self):
        return model_to_dict(self, fields=[field.name for field in
                             self._meta.fields])

    @property
    def diff(self):
        d1 = self._initial
        d2 = self._dict
        diffs = [(k, (v, d2[k])) for k, v in d1.items() if v != d2[k]]
        return dict(diffs)

    @property
    def has_changed(self):
        return bool(self.diff)

    @property
    def changed_fields(self):
        return self.diff.keys()

    def has_field_changed(self, field_name):
        return field_name in self.diff.keys()

    def get_field_diff(self, field_name):
        return self.diff.get(field_name, None)
    # ---- end copy stuff from DiffModelMixin ----

    # default values for created and updated are only there to keep evolution
    # from failing.
    created = models.DateTimeField(auto_now_add=True, default=timezone.now)
    updated = models.DateTimeField(auto_now=True, default=timezone.now)
    enacted = models.DateTimeField(null=True, default=None)
    backend_status = models.CharField(max_length=140,
                                      default="Provisioning in progress")
    deleted = models.BooleanField(default=False)

    class Meta:
        # Changing abstract to False would require the managers of subclasses of
        # PlCoreBase to be customized individually.
        abstract = True
        app_label = "core"

    def __init__(self, *args, **kwargs):
        super(PlCoreBase, self).__init__(*args, **kwargs)
        self._initial = self._dict # for DiffModelMixIn
        self.silent = False

    def can_update(self, user):
        if user.is_readonly:
            return False
        if user.is_admin:
            return True

        return False

    def delete(self, *args, **kwds):
        # so we have something to give the observer
        purge = kwds.get('purge',False)
        if purge:
            del kwds['purge']
        silent = kwds.get('silent',False)
        if silent:
            del kwds['silent']
        try:
            purge = purge or observer_disabled
        except NameError:
            pass

        if (purge):
            super(PlCoreBase, self).delete(*args, **kwds)
        else:
            self.deleted = True
            self.enacted=None
            self.save(update_fields=['enacted','deleted'], silent=silent)

    def save(self, *args, **kwargs):
        # let the user specify silence as either a kwarg or an instance varible
        silent = self.silent
        if "silent" in kwargs:
            silent=silent or kwargs.pop("silent")

        super(PlCoreBase, self).save(*args, **kwargs)

        # This is a no-op if observer_disabled is set
        if not silent:
            notify_observer()

        self._initial = self._dict

    def save_by_user(self, user, *args, **kwds):
        if not self.can_update(user):
            if getattr(self, "_cant_update_fieldName", None) is not None:
                raise PermissionDenied("You do not have permission to update field %s on object %s" % (self._cant_update_fieldName, self.__class__.__name__))
            else:
                raise PermissionDenied("You do not have permission to update %s objects" % self.__class__.__name__)

        self.save(*args, **kwds)

    def delete_by_user(self, user, *args, **kwds):
        if not self.can_update(user):
            raise PermissionDenied("You do not have permission to delete %s objects" % self.__class__.__name__)
        self.delete(*args, **kwds)

    @classmethod
    def select_by_user(cls, user):
        # This should be overridden by descendant classes that want to perform
        # filtering of visible objects by user.
        return cls.objects.all()

    @classmethod
    def is_ephemeral(cls):
	return cls in ephemeral_models

    def getValidators(self):
        """ primarily for REST API, return a dictionary of field names mapped
            to lists of the type of validations that need to be applied to
            those fields.
        """
        validators = {}
        for field in self._meta.fields:
            l = []
            if field.blank==False:
                l.append("notBlank")
            validators[field.name] = l
        return validators

<|endoftext|>


<commit_before>#
# Calculates the co-ordinates along great arcs between two specified points
# which are assumed to be on disk.
#
from __future__ import absolute_import, division, print_function

import numpy as np
from astropy.coordinates import SkyCoord
import sunpy.coordinates

__all__ = ['great_arc']

def great_arc(start_point, end_point, center=None, number_points=100):
    """
    Calculates a user-specified number of points on an arc of a great circle
    connecting two points.

    Parameters
    ----------
    start_point : `~astropy.coordinates.SkyCoord`
        Start point.

    end_point : `~astropy.coordinates.SkyCoord`
        End point.

    center : `~astropy.coordinates.SkyCoord`
        Center of the sphere.

    number_points : int
        Number of points along the great arc.

    Returns
    -------
    arc : `~astropy.coordinates.SkyCoord`
        Co-ordinates along the great arc in the co-ordinate frame of the
        start point.

    Example
    -------
    >>> from astropy.coordinates import SkyCoord
    >>> import astropy.units as u
    >>> import sunpy.coordinates
    >>> from sunpy.coordinates import great_arc
    >>> import sunpy.map
    >>> from sunpy.data.sample import AIA_171_IMAGE
    >>> m = sunpy.map.Map(AIA_171_IMAGE)
    >>> a = SkyCoord(600*u.arcsec, -600*u.arcsec, frame=m.coordinate_frame)
    >>> b = SkyCoord(-100*u.arcsec, 800*u.arcsec, frame=m.coordinate_frame)
    >>> v = great_arc(a, b)
    """
    start_point_unit = start_point.transform_to('heliocentric').cartesian.xyz.unit
    if center is None:
        c = SkyCoord(0*start_point_unit, 0*start_point_unit, 0*start_point_unit, frame='heliocentric')
    a_cartesian = start_point.transform_to('heliocentric').cartesian.xyz.to(start_point_unit).value
    b_cartesian = end_point.transform_to('heliocentric').cartesian.xyz.to(start_point_unit).value
    c_cartesian = c.transform_to('heliocentric').cartesian.xyz.to(start_point_unit).value

    # Calculate the points along the great arc.
    v_cartesian = _calculate_great_arc(a_cartesian, b_cartesian, c_cartesian, number_points)*start_point_unit

    # Transform the great arc back into the input frame.
    return SkyCoord(v_cartesian[:, 0], v_cartesian[:, 1], v_cartesian[:, 2],
                    frame='heliocentric', observer=start_point.observer).transform_to(start_point.frame)

def _calculate_great_arc(start, end, center, number_points):
    """
    Calculate a user-specified number of points on a great arc between a start
    and end point on a sphere where the start and end points are assumed to be
    x,y,z Cartesian triples on a sphere relative to a center.  See the
    references below for a description of the algorithm

    Parameters
    ----------
    start : `~numpy.ndarray`
        Start point expressed as a Cartesian xyz triple.

    end : `~numpy.ndarray`
        End point expressed as a Cartesian xyz triple.

    center : `~numpy.ndarray`
        Center of the sphere expressed as a Cartesian xyz triple

    number_points : int
        Number of points along the great arc.

    Returns
    -------
    arc : `~numpy.ndarray`
        Co-ordinates along the great arc expressed as Cartesian xyz triples.
        The shape of the array is (number_points, 3).

    References
    ----------
    [1] https://www.mathworks.com/matlabcentral/newsreader/view_thread/277881
    [2] https://en.wikipedia.org/wiki/Great-circle_distance#Vector_version

    """
    # Vector from center to first point
    v1 = start - center

    # Distance of the first point from the center
    r = np.linalg.norm(v1)

    # Vector from center to second point
    v2 = end - center

    # The v3 lies in plane of v1 & v2 and is orthogonal to v1
    v3 = np.cross(np.cross(v1, v2), v1)

    # Ensure that the vector has length r
    v3 = r * v3 / np.linalg.norm(v3)

    # Range through the inner angle between v1 and v2
    inner_angles = np.linspace(0, np.arctan2(np.linalg.norm(np.cross(v1, v2)),
                                             np.dot(v1, v2)), num=number_points).reshape(number_points, 1)

    # Calculate the Cartesian locations from the first to second points
    return v1[np.newaxis, :] * np.cos(inner_angles) + \
           v3[np.newaxis, :] * np.sin(inner_angles) + center
<commit_msg>removed unnecessary import<commit_after>#
# Calculates the co-ordinates along great arcs between two specified points
# which are assumed to be on disk.
#
from __future__ import absolute_import, division, print_function

import numpy as np
from astropy.coordinates import SkyCoord

__all__ = ['great_arc']

def great_arc(start_point, end_point, center=None, number_points=100):
    """
    Calculates a user-specified number of points on an arc of a great circle
    connecting two points.

    Parameters
    ----------
    start_point : `~astropy.coordinates.SkyCoord`
        Start point.

    end_point : `~astropy.coordinates.SkyCoord`
        End point.

    center : `~astropy.coordinates.SkyCoord`
        Center of the sphere.

    number_points : int
        Number of points along the great arc.

    Returns
    -------
    arc : `~astropy.coordinates.SkyCoord`
        Co-ordinates along the great arc in the co-ordinate frame of the
        start point.

    Example
    -------
    >>> from astropy.coordinates import SkyCoord
    >>> import astropy.units as u
    >>> import sunpy.coordinates
    >>> from sunpy.coordinates import great_arc
    >>> import sunpy.map
    >>> from sunpy.data.sample import AIA_171_IMAGE
    >>> m = sunpy.map.Map(AIA_171_IMAGE)
    >>> a = SkyCoord(600*u.arcsec, -600*u.arcsec, frame=m.coordinate_frame)
    >>> b = SkyCoord(-100*u.arcsec, 800*u.arcsec, frame=m.coordinate_frame)
    >>> v = great_arc(a, b)
    """
    start_point_unit = start_point.transform_to('heliocentric').cartesian.xyz.unit
    if center is None:
        c = SkyCoord(0*start_point_unit, 0*start_point_unit, 0*start_point_unit, frame='heliocentric')
    a_cartesian = start_point.transform_to('heliocentric').cartesian.xyz.to(start_point_unit).value
    b_cartesian = end_point.transform_to('heliocentric').cartesian.xyz.to(start_point_unit).value
    c_cartesian = c.transform_to('heliocentric').cartesian.xyz.to(start_point_unit).value

    # Calculate the points along the great arc.
    v_cartesian = _calculate_great_arc(a_cartesian, b_cartesian, c_cartesian, number_points)*start_point_unit

    # Transform the great arc back into the input frame.
    return SkyCoord(v_cartesian[:, 0], v_cartesian[:, 1], v_cartesian[:, 2],
                    frame='heliocentric', observer=start_point.observer).transform_to(start_point.frame)

def _calculate_great_arc(start, end, center, number_points):
    """
    Calculate a user-specified number of points on a great arc between a start
    and end point on a sphere where the start and end points are assumed to be
    x,y,z Cartesian triples on a sphere relative to a center.  See the
    references below for a description of the algorithm

    Parameters
    ----------
    start : `~numpy.ndarray`
        Start point expressed as a Cartesian xyz triple.

    end : `~numpy.ndarray`
        End point expressed as a Cartesian xyz triple.

    center : `~numpy.ndarray`
        Center of the sphere expressed as a Cartesian xyz triple

    number_points : int
        Number of points along the great arc.

    Returns
    -------
    arc : `~numpy.ndarray`
        Co-ordinates along the great arc expressed as Cartesian xyz triples.
        The shape of the array is (number_points, 3).

    References
    ----------
    [1] https://www.mathworks.com/matlabcentral/newsreader/view_thread/277881
    [2] https://en.wikipedia.org/wiki/Great-circle_distance#Vector_version

    """
    # Vector from center to first point
    v1 = start - center

    # Distance of the first point from the center
    r = np.linalg.norm(v1)

    # Vector from center to second point
    v2 = end - center

    # The v3 lies in plane of v1 & v2 and is orthogonal to v1
    v3 = np.cross(np.cross(v1, v2), v1)

    # Ensure that the vector has length r
    v3 = r * v3 / np.linalg.norm(v3)

    # Range through the inner angle between v1 and v2
    inner_angles = np.linspace(0, np.arctan2(np.linalg.norm(np.cross(v1, v2)),
                                             np.dot(v1, v2)), num=number_points).reshape(number_points, 1)

    # Calculate the Cartesian locations from the first to second points
    return v1[np.newaxis, :] * np.cos(inner_angles) + \
           v3[np.newaxis, :] * np.sin(inner_angles) + center
<|endoftext|>


<commit_before># This file is for cross validation

from sklearn.cross_validation import KFold
import numpy as np

def cross_validation(data, target, method, n_folds = 5):
	"""
	This function estimates the test error associated with a given statistical learning
	method in order to evaluate its performance.

	Parameters
    ----------
    data: array
    	predictor variable

    target: array
    	response variable

    classfication_method: function
    	input the classfication method

    n_folds: float
    	 randomly dividing the set of observations into k groups
    	 the default value is 5

    Returns
    -------
    cv_scores: float

    """

    # Divide the set of observations in to k groups
    cv = KFold(n = len(data), n_folds)

    cv_scores = []

    # Seperate training set and test set, and then estimate the test error
    for train , test in cv
    	method.fit(data[train], target[train])
    	predicted = method.predict(data[test])
    	cv_scores.append(np.mean(predicted == target[test]))

    return cv_scores

<commit_msg>Delete CV.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import socket
from Queue import Queue
import time

import logging
logger = logging.getLogger('m3_common')

from ice import ICE

class m3_common(object):
    TITLE = "Generic M3 Programmer"

    @staticmethod
    def configure_root_logger():
        try:
            os.environ['ICE_DEBUG']
            logging.basicConfig(level=logging.DEBUG)
        except KeyError:
            logging.basicConfig(level=logging.INFO, format="%(message)s")

    @staticmethod
    def printing_sleep(seconds):
        if seconds < 1:
            time.sleep(seconds)
            return
        while (seconds > 0):
            sys.stdout.write("\rSleeping %d s" % (int(seconds)) + ' '*20)
            sys.stdout.flush()
            time.sleep(min(1, seconds))
            seconds -= 1
        sys.stdout.write('\r' + ' '*80 + '\r')
        sys.stdout.flush()

    @staticmethod
    def default_value(prompt, default, extra=None, invert=False):
        if invert and (extra is None):
            raise RuntimeError, "invert & !extra ?"
        if extra:
            r = raw_input(prompt + ' [' + default + extra + ']: ')
        else:
            r = raw_input(prompt + ' [' + default + ']: ')
        if len(r) == 0:
            if invert:
                return extra
            return default
        else:
            return r

    @staticmethod
    def do_default(prompt, fn, else_fn=None):
        y = m3_common.default_value(prompt, 'Y', '/n')
        if y[0] not in ('n', 'N'):
            fn()
        else:
            if else_fn:
                else_fn()

    @staticmethod
    def dont_do_default(prompt, fn, else_fn=None):
        resp = m3_common.default_value(prompt, 'y/', 'N', invert=True)
        if resp[0] in ('y', 'Y'):
            fn()
        else:
            if else_fn:
                else_fn()

    @staticmethod
    def build_injection_message(hexencoded, run_after=False):
        chip_id_mask = 0                # [0:3] Chip ID Mask
        reset = 0                       #   [4] Reset Request
        chip_id_coding = 0              #   [5] Chip ID coding
        is_i2c = 0                      #   [6] Indicates transmission is I2C message [addr+data]
        run_after = not not run_after   #   [7] Run code after programming?
        # Byte 0: Control
        control = chip_id_mask | (reset << 4) | (chip_id_coding << 5) | (is_i2c << 6) | (run_after << 7)

        # Byte 1,2: Chip ID
        chip_id = 0

        # Byte 3,4: Memory Address
        mem_addr = 0

        # Byte 5,6: Program Lengh
        length = len(hexencoded) >> 3   # hex exapnded -> bytes, /2
        length = socket.htons(length)

        # Byte 7: bit-wise XOR parity of header
        header_parity = 0
        for byte in (
                control,
                (chip_id >> 8) & 0xff,
                chip_id & 0xff,
                (mem_addr >> 8) & 0xff,
                mem_addr & 0xff,
                (length >> 8) & 0xff,
                length & 0xff,
                ):
            header_parity ^= byte

        # Byte 8: bit-wise XOR parity of data
        data_parity = 0
        for byte in [hexencoded[x:x+2] for x in xrange(0, len(hexencoded), 2)]:
            b = int(byte, 16)
            data_parity ^= b

        # Bytes 9+: Data

        # Assemble message:
        message = "%02X%04X%04X%04X%02X%02X%s" % (
                control,
                chip_id,
                mem_addr,
                length,
                header_parity,
                data_parity,
                hexencoded)

        return message

    def __init__(self):
        try:
            self.print_banner()
            self.parse_args()
            self.read_binfile()
            self.ice = ICE()
            self.callback_q = Queue()
            self.install_handler()
            self.ice.connect(self.serial_path)
        except NameError:
            logger.error("Abstract element missing.")
            raise

    def install_handler(self):
        self.ice.msg_handler[self.MSG_TYPE] = self.callback_helper

    def callback_helper(self, msg_type, event_id, length, msg):
        logger.debug("Callback: msg len " + str(len(msg)))
        if len(msg) == 0:
            logger.debug("Ignore msg of len 0")
            return
        callback_q.put(msg)

    def print_banner(self):
        logger.info("-" * 80)
        logger.info(" -- " + self.TITLE)
        logger.info("")

    def parse_args(self):
        if len(sys.argv) not in (3,):
            logger.info("USAGE: %s BINFILE SERAIL_DEVICE\n" % (sys.argv[0]))
            logger.info("")
            sys.exit(2)

        self.binfile = sys.argv[1]
        self.serial_path = sys.argv[2]

    def read_binfile(self):
        def guess_type_is_hex(binfile):
            for line in open(binfile):
                for c in line.strip():
                    c = ord(c)
                    if c < 0x20 or c > 0x7a:
                        return False
            return True

        if guess_type_is_hex(self.binfile):
            logger.info("Guessing hex-encoded stream for NI setup")
            logger.info("  ** This means one byte (two hex characters) per line")
            logger.info("  ** and these are the first two characters on each line.")
            logger.info("  ** If it needs to parse something more complex, let me know.")
            binfd = open(self.binfile, 'r')
            self.hexencoded = ""
            for line in binfd:
                self.hexencoded += line[0:2].upper()
        else:
            logger.info("Guessing compiled binary")
            binfd = open(self.binfile, 'rb')
            self.hexencoded = binfd.read().encode("hex").upper()

        if (len(self.hexencoded) % 4 == 0) and (len(self.hexencoded) % 8 != 0):
            # Image is halfword-aligned. Some tools generate these, but our system
            # assumes things are word-aligned. We pad an extra nop to the end to fix
            logger.info("Padding halfword aligned image with NOP to make word-aligned.")
            self.hexencoded += '46C0' # nop; (mov r8, r8)

        if (len(self.hexencoded) % 8) != 0:
            logger.warn("Binfile is not word-aligned. This is not a valid image")
            sys.exit(3)
        else:
            logger.info("Binfile is %d bytes long\n" % (len(self.hexencoded) / 2))

    def power_on(self):
        logger.info("Turning all M3 power rails on")
        self.ice.power_set_voltage(0,0.6)
        self.ice.power_set_voltage(1,1.2)
        self.ice.power_set_voltage(2,3.8)
        logger.info("Turning 3.8 on")
        self.ice.power_set_onoff(2,True)
        m3_common.printing_sleep(1.0)
        logger.info("Turning 1.2 on")
        self.ice.power_set_onoff(1,True)
        m3_common.printing_sleep(1.0)
        logger.info("Turning 0.6 on")
        self.ice.power_set_onoff(0,True)
        m3_common.printing_sleep(1.0)
        logger.info("Waiting 8 seconds for power rails to settle")
        m3_common.printing_sleep(8.0)

    def reset_m3(self):
        logger.info("M3 0.6V => OFF (reset controller)")
        self.ice.power_set_onoff(0,False)
        m3_common.printing_sleep(2.0)
        logger.info("M3 0.6V => ON")
        self.ice.power_set_onoff(0,True)
        m3_common.printing_sleep(2.0)

    def validate_bin(self): #, hexencoded, offset=0):
        raise NotImplementedError("Need to update for MBus. Let me know if needed.")
        logger.info("Configuring ICE to ACK adress 1001 100x")
        ice.i2c_set_address("1001100x") # 0x98

        logger.info("Running Validation sequence:")
        logger.info("\t DMA read at address 0x%x, length %d" % (offset, len(hexencoded)/2))
        logger.info("\t<Receive I2C message for DMA data>")
        logger.info("\tCompare received data and validate it was programmed correctly")

        length = len(hexencoded)/8
        offset = offset
        data = 0x80000000 | (length << 16) | offset
        dma_read_req = "%08X" % (socket.htonl(data))
        logger.debug("Sending: " + dma_read_req)
        ice.i2c_send(0xaa, dma_read_req.decode('hex'))

        logger.info("Chip Program Dump Response:")
        chip_bin = validate_q.get(True, ice.ONEYEAR)
        logger.debug("Raw chip bin response len " + str(len(chip_bin)))
        chip_bin = chip_bin.encode('hex')
        logger.debug("Chip bin len %d val: %s" % (len(chip_bin), chip_bin))

        #1,2-addr ...
        chip_bin = chip_bin[2:]

        # Consistent capitalization
        chip_bin = chip_bin.upper()
        hexencoded = hexencoded.upper()

        for b in range(len(hexencoded)):
            try:
                if hexencoded[b] != chip_bin[b]:
                    logger.warn("ERR: Mismatch at half-byte" + str(b))
                    logger.warn("Expected:" + hexencoded[b])
                    logger.warn("Got:" + chip_bin[b])
                    return False
            except IndexError:
                logger.warn("ERR: Length mismatch")
                logger.warn("Expected %d bytes" % (len(hexencoded)/2))
                logger.warn("Got %d bytes" % (len(chip_bin)/2))
                logger.warn("All prior bytes validated correctly")
                return False

        logger.info("Programming validated successfully")
        return True

    def hang_for_messages(self):
        logger.info("Script is waiting to print any MBus messages.")
        logger.info("To quit, press Ctrl-C")
        try:
            time.sleep(1000)
        except KeyboardInterrupt:
            logger.info("Exiting.")

class goc_programmer(m3_common):
    TITLE = "GOC Programmer"
    SLOW_FREQ_IN_HZ = 0.625

    def set_slow_frequency(self):
        self.ice.goc_set_frequency(self.SLOW_FREQ_IN_HZ)

    def wake_chip(self):
        passcode_string = "7394"
        logger.info("Sending passcode to GOC")
        logger.debug("Sending:" + passcode_string)
        self.ice.goc_send(passcode_string.decode('hex'))
        self.printing_sleep(4.0)

    def set_fast_frequency(self):
        self.ice.goc_set_frequency(8*self.SLOW_FREQ_IN_HZ)

    def send_goc_message(self, message):
        logger.info("Sending GOC message")
        logger.debug("Sending: " + message)
        self.ice.goc_send(message.decode('hex'))
        self.printing_sleep(1.0)

        logger.info("Sending extra blink to end transaction")
        extra = "80"
        logger.debug("Sending: " + extra)
        self.ice.goc_send(extra.decode('hex'))

    def validate_bin(self):
        raise NotImplementedError("If you need this, let me know")

    def DMA_start_interrupt(self):
        logger.info("Sending 0x88 0x00000000")
        self.send("88".decode('hex'), "00000000".decode('hex'))
<commit_msg>Have m3_common auto-detect ICE serial port<commit_after>#!/usr/bin/env python

import os
import sys
import socket
from Queue import Queue
import argparse
import time

import logging
logger = logging.getLogger('m3_common')

from ice import ICE

# Do this after ICE since ICE prints a nice help if pyserial is missing
import serial.tools.list_ports

class m3_common(object):
    TITLE = "Generic M3 Programmer"

    @staticmethod
    def configure_root_logger():
        try:
            os.environ['ICE_DEBUG']
            logging.basicConfig(level=logging.DEBUG)
        except KeyError:
            logging.basicConfig(level=logging.INFO, format="%(message)s")

    @staticmethod
    def printing_sleep(seconds):
        if seconds < 1:
            time.sleep(seconds)
            return
        while (seconds > 0):
            sys.stdout.write("\rSleeping %d s" % (int(seconds)) + ' '*20)
            sys.stdout.flush()
            time.sleep(min(1, seconds))
            seconds -= 1
        sys.stdout.write('\r' + ' '*80 + '\r')
        sys.stdout.flush()

    @staticmethod
    def default_value(prompt, default, extra=None, invert=False):
        if invert and (extra is None):
            raise RuntimeError, "invert & !extra ?"
        if extra:
            r = raw_input(prompt + ' [' + default + extra + ']: ')
        else:
            r = raw_input(prompt + ' [' + default + ']: ')
        if len(r) == 0:
            if invert:
                return extra
            return default
        else:
            return r

    @staticmethod
    def do_default(prompt, fn, else_fn=None):
        y = m3_common.default_value(prompt, 'Y', '/n')
        if y[0] not in ('n', 'N'):
            fn()
        else:
            if else_fn:
                else_fn()

    @staticmethod
    def dont_do_default(prompt, fn, else_fn=None):
        resp = m3_common.default_value(prompt, 'y/', 'N', invert=True)
        if resp[0] in ('y', 'Y'):
            fn()
        else:
            if else_fn:
                else_fn()

    @staticmethod
    def build_injection_message(hexencoded, run_after=False):
        chip_id_mask = 0                # [0:3] Chip ID Mask
        reset = 0                       #   [4] Reset Request
        chip_id_coding = 0              #   [5] Chip ID coding
        is_i2c = 0                      #   [6] Indicates transmission is I2C message [addr+data]
        run_after = not not run_after   #   [7] Run code after programming?
        # Byte 0: Control
        control = chip_id_mask | (reset << 4) | (chip_id_coding << 5) | (is_i2c << 6) | (run_after << 7)

        # Byte 1,2: Chip ID
        chip_id = 0

        # Byte 3,4: Memory Address
        mem_addr = 0

        # Byte 5,6: Program Lengh
        length = len(hexencoded) >> 3   # hex exapnded -> bytes, /2
        length = socket.htons(length)

        # Byte 7: bit-wise XOR parity of header
        header_parity = 0
        for byte in (
                control,
                (chip_id >> 8) & 0xff,
                chip_id & 0xff,
                (mem_addr >> 8) & 0xff,
                mem_addr & 0xff,
                (length >> 8) & 0xff,
                length & 0xff,
                ):
            header_parity ^= byte

        # Byte 8: bit-wise XOR parity of data
        data_parity = 0
        for byte in [hexencoded[x:x+2] for x in xrange(0, len(hexencoded), 2)]:
            b = int(byte, 16)
            data_parity ^= b

        # Bytes 9+: Data

        # Assemble message:
        message = "%02X%04X%04X%04X%02X%02X%s" % (
                control,
                chip_id,
                mem_addr,
                length,
                header_parity,
                data_parity,
                hexencoded)

        return message

    def __init__(self):
        try:
            self.print_banner()
            self.parse_args()
            self.read_binfile()
            self.ice = ICE()
            self.callback_q = Queue()
            self.install_handler()
            self.ice.connect(self.serial_path)
        except NameError:
            logger.error("Abstract element missing.")
            raise

    def install_handler(self):
        self.ice.msg_handler[self.MSG_TYPE] = self.callback_helper

    def callback_helper(self, msg_type, event_id, length, msg):
        logger.debug("Callback: msg len " + str(len(msg)))
        if len(msg) == 0:
            logger.debug("Ignore msg of len 0")
            return
        callback_q.put(msg)

    def print_banner(self):
        logger.info("-" * 80)
        logger.info(" -- " + self.TITLE)
        logger.info("")

    def add_parse_args(self):
        self.parser.add_argument("BINFILE", help="Program to flash")
        self.parser.add_argument("SERIAL", help="Path to ICE serial device", nargs='?')

    def parse_args(self):
        self.parser = argparse.ArgumentParser()
        self.add_parse_args()

        self.args = self.parser.parse_args()
        if self.args.SERIAL is None:
            self.serial_path = self.guess_serial()
        else:
            self.serial_path = self.args.SERIAL

    def guess_serial(self):
        candidates = []
        for s in serial.tools.list_ports.comports():
            s = s[0]
            if 'bluetooth' in s.lower():
                continue
            candidates.append(s)
        if len(candidates) == 0:
            # In many cases when debugging, we'll be using the fake_ice at '/tmp/com1'
            if os.path.exists('/tmp/com1'):
                candidates.append('/tmp/com1')
        if len(candidates) == 0:
            logger.error("Could not find the serial port ICE is attached to.\n")
            self.parser.print_help()
            sys.exit(1)
        elif len(candidates) == 1:
            logger.info("Guessing ICE is at: " + candidates[0])
            return candidates[0]
        else:
            def pick_serial():
                logger.info("Multiple possible serial ports found:")
                for i in xrange(len(candidates)):
                    logger.info("\t[{}] {}".format(i, candidates[i]))
                try:
                    resp = raw_input("Choose a serial port (Ctrl-C to quit): ").strip()
                except KeyboardInterrupt:
                    sys.exit(1)
                try:
                    return candidates[int(resp)]
                except:
                    logger.info("Please choose one of the available serial ports.")
                    return pick_serial()
            return pick_serial()

    def read_binfile(self):
        def guess_type_is_hex(binfile):
            for line in open(binfile):
                for c in line.strip():
                    c = ord(c)
                    if c < 0x20 or c > 0x7a:
                        return False
            return True

        if guess_type_is_hex(self.args.BINFILE):
            logger.info("Guessing hex-encoded stream for NI setup")
            logger.info("  ** This means one byte (two hex characters) per line")
            logger.info("  ** and these are the first two characters on each line.")
            logger.info("  ** If it needs to parse something more complex, let me know.")
            binfd = open(self.args.BINFILE, 'r')
            self.hexencoded = ""
            for line in binfd:
                self.hexencoded += line[0:2].upper()
        else:
            logger.info("Guessing compiled binary")
            binfd = open(self.args.BINFILE, 'rb')
            self.hexencoded = binfd.read().encode("hex").upper()

        if (len(self.hexencoded) % 4 == 0) and (len(self.hexencoded) % 8 != 0):
            # Image is halfword-aligned. Some tools generate these, but our system
            # assumes things are word-aligned. We pad an extra nop to the end to fix
            logger.info("Padding halfword aligned image with NOP to make word-aligned.")
            self.hexencoded += '46C0' # nop; (mov r8, r8)

        if (len(self.hexencoded) % 8) != 0:
            logger.warn("Binfile is not word-aligned. This is not a valid image")
            sys.exit(3)
        else:
            logger.info("Binfile is %d bytes long\n" % (len(self.hexencoded) / 2))

    def power_on(self):
        logger.info("Turning all M3 power rails on")
        self.ice.power_set_voltage(0,0.6)
        self.ice.power_set_voltage(1,1.2)
        self.ice.power_set_voltage(2,3.8)
        logger.info("Turning 3.8 on")
        self.ice.power_set_onoff(2,True)
        m3_common.printing_sleep(1.0)
        logger.info("Turning 1.2 on")
        self.ice.power_set_onoff(1,True)
        m3_common.printing_sleep(1.0)
        logger.info("Turning 0.6 on")
        self.ice.power_set_onoff(0,True)
        m3_common.printing_sleep(1.0)
        logger.info("Waiting 8 seconds for power rails to settle")
        m3_common.printing_sleep(8.0)

    def reset_m3(self):
        logger.info("M3 0.6V => OFF (reset controller)")
        self.ice.power_set_onoff(0,False)
        m3_common.printing_sleep(2.0)
        logger.info("M3 0.6V => ON")
        self.ice.power_set_onoff(0,True)
        m3_common.printing_sleep(2.0)

    def validate_bin(self): #, hexencoded, offset=0):
        raise NotImplementedError("Need to update for MBus. Let me know if needed.")
        logger.info("Configuring ICE to ACK adress 1001 100x")
        ice.i2c_set_address("1001100x") # 0x98

        logger.info("Running Validation sequence:")
        logger.info("\t DMA read at address 0x%x, length %d" % (offset, len(hexencoded)/2))
        logger.info("\t<Receive I2C message for DMA data>")
        logger.info("\tCompare received data and validate it was programmed correctly")

        length = len(hexencoded)/8
        offset = offset
        data = 0x80000000 | (length << 16) | offset
        dma_read_req = "%08X" % (socket.htonl(data))
        logger.debug("Sending: " + dma_read_req)
        ice.i2c_send(0xaa, dma_read_req.decode('hex'))

        logger.info("Chip Program Dump Response:")
        chip_bin = validate_q.get(True, ice.ONEYEAR)
        logger.debug("Raw chip bin response len " + str(len(chip_bin)))
        chip_bin = chip_bin.encode('hex')
        logger.debug("Chip bin len %d val: %s" % (len(chip_bin), chip_bin))

        #1,2-addr ...
        chip_bin = chip_bin[2:]

        # Consistent capitalization
        chip_bin = chip_bin.upper()
        hexencoded = hexencoded.upper()

        for b in range(len(hexencoded)):
            try:
                if hexencoded[b] != chip_bin[b]:
                    logger.warn("ERR: Mismatch at half-byte" + str(b))
                    logger.warn("Expected:" + hexencoded[b])
                    logger.warn("Got:" + chip_bin[b])
                    return False
            except IndexError:
                logger.warn("ERR: Length mismatch")
                logger.warn("Expected %d bytes" % (len(hexencoded)/2))
                logger.warn("Got %d bytes" % (len(chip_bin)/2))
                logger.warn("All prior bytes validated correctly")
                return False

        logger.info("Programming validated successfully")
        return True

    def hang_for_messages(self):
        logger.info("Script is waiting to print any MBus messages.")
        logger.info("To quit, press Ctrl-C")
        try:
            time.sleep(1000)
        except KeyboardInterrupt:
            logger.info("Exiting.")

class goc_programmer(m3_common):
    TITLE = "GOC Programmer"
    SLOW_FREQ_IN_HZ = 0.625

    def set_slow_frequency(self):
        self.ice.goc_set_frequency(self.SLOW_FREQ_IN_HZ)

    def wake_chip(self):
        passcode_string = "7394"
        logger.info("Sending passcode to GOC")
        logger.debug("Sending:" + passcode_string)
        self.ice.goc_send(passcode_string.decode('hex'))
        self.printing_sleep(4.0)

    def set_fast_frequency(self):
        self.ice.goc_set_frequency(8*self.SLOW_FREQ_IN_HZ)

    def send_goc_message(self, message):
        logger.info("Sending GOC message")
        logger.debug("Sending: " + message)
        self.ice.goc_send(message.decode('hex'))
        self.printing_sleep(1.0)

        logger.info("Sending extra blink to end transaction")
        extra = "80"
        logger.debug("Sending: " + extra)
        self.ice.goc_send(extra.decode('hex'))

    def validate_bin(self):
        raise NotImplementedError("If you need this, let me know")

    def DMA_start_interrupt(self):
        logger.info("Sending 0x88 0x00000000")
        self.send("88".decode('hex'), "00000000".decode('hex'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys, os, shutil, subprocess, random, threading, time, glob

if len(sys.argv) < 7:
    print("usage: " + sys.argv[0] + " sim_type static_Bz" + \
          " c13_percentage max_cluster_size log10_samples task_num [seed]")
    exit(1)

# process inputs
sim_type = sys.argv[1]
static_Bz = sys.argv[2]
c13_percentage = sys.argv[3]
max_cluster_size = sys.argv[4]
log10_samples = sys.argv[5]
task_num = int(sys.argv[6])
assert task_num > 1
seed_text = " ".join(sys.argv[7:])

# identify some directories and names
work_dir = os.path.dirname(os.path.realpath(__file__))
sim_file = "simulate.exe"
out_file = "data/fidelities-{}.txt".format("-".join([sim_type, static_Bz, c13_percentage,
                                                     max_cluster_size, log10_samples]))

if "TMPDIR" not in os.environ:
    print("please set the TMPDIR environment variable")
    exit(1)
job_dir = os.environ["TMPDIR"] + "/" + os.environ["USER"] + "/" + "-".join(sys.argv[1:])

# move into job directory
os.makedirs(job_dir)
shutil.copy2(work_dir+"/"+sim_file, job_dir+"/"+sim_file)
os.chdir(job_dir)

# main code
commands = ["./"+sim_file, "--no_output", "--" + sim_type,
            "--static_Bz", static_Bz,
            "--c13_percentage", c13_percentage,
            "--max_cluster_size", max_cluster_size]

unsigned_long_long_max = 2**64-1
lock = threading.RLock()

def run_sample(s):
    random.seed("".join(sys.argv[1:-1]) + str(s))
    seed = ["--seed", str(random.randint(0,unsigned_long_long_max))]
    process = subprocess.Popen(commands + seed,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = process.communicate()
    with lock:
        with open(out_file,"a") as f:
            f.write(" ".join(commands + seed)+"\n\n")
            f.write(out.decode("utf-8")+"\n")
            f.write(err.decode("utf-8")+"\n")
            f.write("-"*90 + "\n\n")

samples = int(10**float(log10_samples))
for s in range(samples):
    print("{} / {}".format(s,samples))
    t = threading.Thread(target=run_sample,args=[s])
    while threading.active_count() >= task_num:
        time.sleep(1)
    t.start()

# copy results back into the working directory
for f in glob.glob("data/*"):
    shutil.copy2(f, work_dir+"/"+f)

print("----- done -----")
<commit_msg>print job directory in fidelity-sweep.py<commit_after>#!/usr/bin/env python
import sys, os, shutil, subprocess, random, threading, time, glob

if len(sys.argv) < 7:
    print("usage: " + sys.argv[0] + " sim_type static_Bz" + \
          " c13_percentage max_cluster_size log10_samples task_num [seed]")
    exit(1)

# process inputs
sim_type = sys.argv[1]
static_Bz = sys.argv[2]
c13_percentage = sys.argv[3]
max_cluster_size = sys.argv[4]
log10_samples = sys.argv[5]
task_num = int(sys.argv[6])
assert task_num > 1
seed_text = " ".join(sys.argv[7:])

# identify some directories and names
work_dir = os.path.dirname(os.path.realpath(__file__))
sim_file = "simulate.exe"
out_file = "data/fidelities-{}.txt".format("-".join([sim_type, static_Bz, c13_percentage,
                                                     max_cluster_size, log10_samples]))

if "TMPDIR" not in os.environ:
    print("please set the TMPDIR environment variable")
    exit(1)
job_dir = os.environ["TMPDIR"] + "/" + os.environ["USER"] + "/" + "-".join(sys.argv[1:])
print("job directory: " + job_dir)

# move into job directory
os.makedirs(job_dir)
shutil.copy2(work_dir+"/"+sim_file, job_dir+"/"+sim_file)
os.chdir(job_dir)

# main code
commands = ["./"+sim_file, "--no_output", "--" + sim_type,
            "--static_Bz", static_Bz,
            "--c13_percentage", c13_percentage,
            "--max_cluster_size", max_cluster_size]

unsigned_long_long_max = 2**64-1
lock = threading.RLock()

def run_sample(s):
    random.seed("".join(sys.argv[1:-1]) + str(s))
    seed = ["--seed", str(random.randint(0,unsigned_long_long_max))]
    process = subprocess.Popen(commands + seed,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = process.communicate()
    with lock:
        with open(out_file,"a") as f:
            f.write(" ".join(commands + seed)+"\n\n")
            f.write(out.decode("utf-8")+"\n")
            f.write(err.decode("utf-8")+"\n")
            f.write("-"*90 + "\n\n")

samples = int(10**float(log10_samples))
for s in range(samples):
    print("{} / {}".format(s,samples))
    t = threading.Thread(target=run_sample,args=[s])
    while threading.active_count() >= task_num:
        time.sleep(1)
    t.start()

# copy results back into the working directory
for f in glob.glob("data/*"):
    shutil.copy2(f, work_dir+"/"+f)

print("----- done -----")
<|endoftext|>


<commit_before>from __future__ import unicode_literals

# imports - third-party imports
import requests

# imports - compatibility imports
import six

# imports - standard imports
from   collections.abc import Sequence, MutableSequence, Mapping, MutableMapping
if six.PY2:
	from urlparse import urlparse 	  # PY2
else:
	from urllib.parse import urlparse # PY3
import json

# imports - module imports
from   frappe.model.document import Document
from   frappe.exceptions import DuplicateEntryError
from   frappe import _dict
import frappe

session = frappe.session

def get_user_doc(user = None):
	if isinstance(user, Document):
		return user

	user = user or session.user
	user = frappe.get_doc('User', user)

	return user

def squashify(what):
	if isinstance(what, Sequence) and len(what) == 1:
		return what[0]

	return what

def safe_json_loads(*args):
	results = [ ]

	for arg in args:
		try:
			arg = json.loads(arg)
		except Exception as e:
			pass

		results.append(arg)

	return squashify(results)

def filter_dict(what, keys, ignore = False):
	copy = dict()

	if keys:
		for k in keys:
			if k not in what and not ignore:
				raise KeyError('{key} not in dict.'.format(key = k))
			else:
				copy.update({
					k: what[k]
				})
	else:
		copy = what.copy()

	return copy

def get_if_empty(a, b):
	if not a:
		a = b
	return a

def listify(arg):
	if not isinstance(arg, list):
		arg = [arg]
	return arg

def dictify(arg):
	if isinstance(arg, MutableSequence):
		for i, a in enumerate(arg):
			arg[i] = dictify(a)
	elif isinstance(arg, MutableMapping):
		arg = _dict(arg)

	return arg

def check_url(what, raise_err = False):
	if not urlparse(what).scheme:
		if raise_err:
			raise ValueError('{what} not a valid URL.')
		else:
			return False

	return True

def create_test_user(module):
	try:
		test_user = frappe.new_doc('User')
		test_user.first_name = '{module}'.format(module = module)
		test_user.email      = 'testuser.{module}@example.com'.format(module = module)
		test_user.save()
	except DuplicateEntryError:
		frappe.log('Test User Chat Profile exists.')

def get_emojis():
	redis  = frappe.cache()
	emojis = redis.hget('frappe_emojis', 'emojis')

	if not emojis:
		resp  = requests.get('http://git.io/frappe-emoji')
		if resp.ok:
			emojis = resp.json()
			redis.hset('frappe_emojis', 'emojis', emojis)

	return dictify(emojis)
<commit_msg>style: Optimize imports, fixed flake8 issues<commit_after>from __future__ import unicode_literals

# imports - standard imports
import json
from collections.abc import MutableMapping, MutableSequence, Sequence

# imports - third-party imports
import requests
from urllib.parse import urlparse

# imports - module imports
import frappe
from frappe.exceptions import DuplicateEntryError
from frappe.model.document import Document

session = frappe.session

def get_user_doc(user = None):
	if isinstance(user, Document):
		return user

	user = user or session.user
	user = frappe.get_doc('User', user)

	return user

def squashify(what):
	if isinstance(what, Sequence) and len(what) == 1:
		return what[0]

	return what

def safe_json_loads(*args):
	results = []

	for arg in args:
		try:
			arg = json.loads(arg)
		except Exception:
			pass

		results.append(arg)

	return squashify(results)

def filter_dict(what, keys, ignore = False):
	copy = dict()

	if keys:
		for k in keys:
			if k not in what and not ignore:
				raise KeyError('{key} not in dict.'.format(key = k))
			else:
				copy.update({
					k: what[k]
				})
	else:
		copy = what.copy()

	return copy

def get_if_empty(a, b):
	if not a:
		a = b
	return a

def listify(arg):
	if not isinstance(arg, list):
		arg = [arg]
	return arg

def dictify(arg):
	if isinstance(arg, MutableSequence):
		for i, a in enumerate(arg):
			arg[i] = dictify(a)
	elif isinstance(arg, MutableMapping):
		arg = frappe._dict(arg)

	return arg

def check_url(what, raise_err = False):
	if not urlparse(what).scheme:
		if raise_err:
			raise ValueError('{what} not a valid URL.')
		else:
			return False

	return True

def create_test_user(module):
	try:
		test_user = frappe.new_doc('User')
		test_user.first_name = '{module}'.format(module = module)
		test_user.email      = 'testuser.{module}@example.com'.format(module = module)
		test_user.save()
	except DuplicateEntryError:
		frappe.log('Test User Chat Profile exists.')

def get_emojis():
	redis  = frappe.cache()
	emojis = redis.hget('frappe_emojis', 'emojis')

	if not emojis:
		resp  = requests.get('http://git.io/frappe-emoji')
		if resp.ok:
			emojis = resp.json()
			redis.hset('frappe_emojis', 'emojis', emojis)

	return dictify(emojis)
<|endoftext|>


<commit_before>import ConfigParser

config = ConfigParser.ConfigParser()
config.read(['.coffeerc'])

app_config = {
    'DEBUG': config.getboolean('coffee_server', 'debug'),
    'REDIS_DB': config.get('coffee_server', 'redis_db'),
    'REDIS_HOST': config.get('coffee_server', 'redis_host'),
    'REDIS_PORT': int(config.get('coffee_server', 'redis_port')),
}
try:
    app_config['REDIS_PW'] = config.get('coffee_server', 'redis_pw')
except ConfigParser.NoOptionError:
    app_config['REDIS_PW'] = None
<commit_msg>parse redis_db setting as int<commit_after>import ConfigParser

config = ConfigParser.ConfigParser()
config.read(['.coffeerc'])

app_config = {
    'DEBUG': config.getboolean('coffee_server', 'debug'),
    'REDIS_DB': int(config.get('coffee_server', 'redis_db')),
    'REDIS_HOST': config.get('coffee_server', 'redis_host'),
    'REDIS_PORT': int(config.get('coffee_server', 'redis_port')),
}
try:
    app_config['REDIS_PW'] = config.get('coffee_server', 'redis_pw')
except ConfigParser.NoOptionError:
    app_config['REDIS_PW'] = None
<|endoftext|>


<commit_before>"""
This cog will create messages that will manage channel perms with reacts.
"""
import discord
import datetime
from asyncpg.exceptions import UniqueViolationError
from collections import defaultdict
from .utils import helpers, checks
from discord.ext import commands

class Channels():
    """
    """

    def __init__(self, bot):
        super().__init__()
        self.reaction_emojis = [ '\N{WHITE HEAVY CHECK MARK}',
            '0\u20e3', '1\u20e3', '2\u20e3', '3\u20e3', '4\u20e3',
            '5\u20e3', '6\u20e3', '7\u20e3', '8\u20e3', '9\u20e3',
            '\N{REGIONAL INDICATOR SYMBOL LETTER A}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER B}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER C}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER D}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER E}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER F}',
            ]
        self.bot = bot

    @commands.group()
    @commands.guild_only()
    @checks.has_permissions(manage_channels=True)
    async def channel_message(self, ctx):
         if ctx.invoked_subcommand is None:
            await ctx.send(':thinking:')

    @channel_message.command(aliases=['add'])
    async def create(self, ctx, target_channel: discord.TextChannel, *, description: str):
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        local_embed = discord.Embed(
            title=f'#{target_channel.name}',
            description=f'{description[:2046]}',
            type="rich"
        )
        message = await ctx.send(embed=local_embed)
        await message.add_reaction(self.reaction_emojis[0])
        try:
            await self.bot.postgres_controller.add_channel_message(
                message.id, target_channel.id, ctx.channel.id)
        except UniqueViolationError:
            await message.delete()
            await ctx.send(
                f"There already exists a link to {target_channel.name} here.")
        await ctx.message.delete()

    @channel_message.command(aliases=['rem'])
    async def remove(self, ctx, target_channel: discord.TextChannel):
        """
        uhhh it removes the thing
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except Exception as e:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        for reaction in og_message.reactions:
            async for user in reaction.users():
                if user.bot:
                    continue
                await og_message.remove_reaction(reaction.emoji, user)
                await self.remove_perms(user, target_channel)
        await og_message.delete()
        await self.bot.postgres_controller.rem_channel_message(target_channel.id, ctx.channel.id)
        await ctx.message.delete()

    @channel_message.command()
    async def edit(self, ctx, target_channel: discord.TextChannel, *, edit: str):
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        og_embed.description = edit[:2046]
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command(aliases=['fix'])
    async def update(self, ctx, target_channel: discord.TextChannel):
        """
        This will update the title of the embed to the currrent title of the channel
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        og_embed.title = f'#{target_channel.name}'
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command(aliases=['color'])
    async def set_color(self, ctx, target_channel: discord.TextChannel, color):
        """
        This will update the color of the embed to a given color code
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        og_embed.color = color
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command()
    async def set_image(self, ctx, target_channel: discord.TextChannel, image_url):
        """
        This will update the image in the embed to the given url
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        try:
            og_embed.set_image(url=image_url)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
            await ctx.send('something broke again', delete_after=3)
            return
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command(aliases=['set_thumb'])
    async def set_thumbnail(self, ctx, target_channel: discord.TextChannel, image_url):
        """
        This will update the image in the embed to the given url
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        try:
            og_embed.set_thumbnail(url=image_url)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
            await ctx.send('something broke again', delete_after=3)
            return
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command()
    async def set_footer(self, ctx, target_channel: discord.TextChannel, *, footer):
        """
        This will update the footer in the embed to the given message
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        try:
            og_embed.set_footer(text=footer)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
            await ctx.send('something broke again', delete_after=3)
            return
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    async def on_raw_reaction_add(self, emoji, message_id, channel_id, user_id):
        """
        Called when an emoji is added
        """
        target_channel = await self.bot.postgres_controller.get_target_channel(channel_id, message_id)
        if not target_channel:
                return
        user = self.bot.get_user(user_id)
        channel = self.bot.get_channel(target_channel)
        reacts = await self.bot.postgres_controller.add_user_reaction(user_id, message_id)
        if int(reacts) in [10,20,100]:
                time = self.bot.timestamp()
                mod_info = self.bot.get_channel(259728514914189312)
                await mod_info.send(
                    f'**{time} | REACTION SPAM:** {user} has reacted {reacts} '\
                    f'times today on the permission message for #{channel}'
                )
        await self.add_perms(user, channel)

    async def on_raw_reaction_remove(self, emoji, message_id, channel_id, user_id):
        """
        Called when an emoji is removed
        """
        target_channel = await self.bot.postgres_controller.get_target_channel(channel_id, message_id)
        if not target_channel:
            return
        channel = self.bot.get_channel(target_channel)
        user = self.bot.get_user(user_id)
        await self.remove_perms(user, channel)

    async def add_perms(self, user, channel):
        """
        Adds a user to channels perms
        """
        try:
            await channel.set_permissions(user, read_messages=True)
        except Exception as e:
            self.bot.logger.warning(f'{e}')

    async def remove_perms(self, user, channel):
        """
        removes a users perms on a channel
        """
        try:
            await channel.set_permissions(user, read_messages=False)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
<commit_msg>maybe now?<commit_after>"""
This cog will create messages that will manage channel perms with reacts.
"""
import discord
import datetime
from asyncpg.exceptions import UniqueViolationError
from collections import defaultdict
from .utils import helpers, checks
from discord.ext import commands

class Channels():
    """
    """

    def __init__(self, bot):
        super().__init__()
        self.reaction_emojis = [ '\N{WHITE HEAVY CHECK MARK}',
            '0\u20e3', '1\u20e3', '2\u20e3', '3\u20e3', '4\u20e3',
            '5\u20e3', '6\u20e3', '7\u20e3', '8\u20e3', '9\u20e3',
            '\N{REGIONAL INDICATOR SYMBOL LETTER A}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER B}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER C}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER D}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER E}',
            '\N{REGIONAL INDICATOR SYMBOL LETTER F}',
            ]
        self.bot = bot

    @commands.group()
    @commands.guild_only()
    @checks.has_permissions(manage_channels=True)
    async def channel_message(self, ctx):
         if ctx.invoked_subcommand is None:
            await ctx.send(':thinking:')

    @channel_message.command(aliases=['add'])
    async def create(self, ctx, target_channel: discord.TextChannel, *, description: str):
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        local_embed = discord.Embed(
            title=f'#{target_channel.name}',
            description=f'{description[:2046]}',
            type="rich"
        )
        message = await ctx.send(embed=local_embed)
        await message.add_reaction(self.reaction_emojis[0])
        try:
            await self.bot.postgres_controller.add_channel_message(
                message.id, target_channel.id, ctx.channel.id)
        except UniqueViolationError:
            await message.delete()
            await ctx.send(
                f"There already exists a link to {target_channel.name} here.")
        await ctx.message.delete()

    @channel_message.command(aliases=['rem'])
    async def remove(self, ctx, target_channel: discord.TextChannel):
        """
        uhhh it removes the thing
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except Exception as e:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        for reaction in og_message.reactions:
            async for user in reaction.users():
                if user.bot:
                    continue
                await og_message.remove_reaction(reaction.emoji, user)
                await self.remove_perms(user, target_channel)
        await og_message.delete()
        await self.bot.postgres_controller.rem_channel_message(target_channel.id, ctx.channel.id)
        await ctx.message.delete()

    @channel_message.command()
    async def edit(self, ctx, target_channel: discord.TextChannel, *, edit: str):
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        og_embed.description = edit[:2046]
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command(aliases=['fix'])
    async def update(self, ctx, target_channel: discord.TextChannel):
        """
        This will update the title of the embed to the currrent title of the channel
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        og_embed.title = f'#{target_channel.name}'
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command(aliases=['color'])
    async def set_color(self, ctx, target_channel: discord.TextChannel, red, green, blue):
        """
        This will update the color of the embed to a given color code
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        og_embed.color = discord.Color.from_rgb(red, green, blue)
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command()
    async def set_image(self, ctx, target_channel: discord.TextChannel, image_url):
        """
        This will update the image in the embed to the given url
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        try:
            og_embed.set_image(url=image_url)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
            await ctx.send('something broke again', delete_after=3)
            return
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command(aliases=['set_thumb'])
    async def set_thumbnail(self, ctx, target_channel: discord.TextChannel, image_url):
        """
        This will update the image in the embed to the given url
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        try:
            og_embed.set_thumbnail(url=image_url)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
            await ctx.send('something broke again', delete_after=3)
            return
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    @channel_message.command()
    async def set_footer(self, ctx, target_channel: discord.TextChannel, *, footer):
        """
        This will update the footer in the embed to the given message
        """
        if not isinstance(target_channel, discord.TextChannel):
            await ctx.send("that is not a valid channel fam", delete_after=4)
            return
        try:
            message_id = await self.bot.postgres_controller.get_message_info(
                ctx.channel.id, target_channel.id)
        except:
            await ctx.send("something broke", delete_after=3)
            return
        if not message_id:
            return
        og_message = await ctx.channel.get_message(message_id)
        og_embed = og_message.embeds[0]
        try:
            og_embed.set_footer(text=footer)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
            await ctx.send('something broke again', delete_after=3)
            return
        await og_message.edit(embed=og_embed)
        await ctx.send(":ok_hand:", delete_after=3)
        await ctx.message.delete()

    async def on_raw_reaction_add(self, emoji, message_id, channel_id, user_id):
        """
        Called when an emoji is added
        """
        target_channel = await self.bot.postgres_controller.get_target_channel(channel_id, message_id)
        if not target_channel:
                return
        user = self.bot.get_user(user_id)
        channel = self.bot.get_channel(target_channel)
        reacts = await self.bot.postgres_controller.add_user_reaction(user_id, message_id)
        if int(reacts) in [10,20,100]:
                time = self.bot.timestamp()
                mod_info = self.bot.get_channel(259728514914189312)
                await mod_info.send(
                    f'**{time} | REACTION SPAM:** {user} has reacted {reacts} '\
                    f'times today on the permission message for #{channel}'
                )
        await self.add_perms(user, channel)

    async def on_raw_reaction_remove(self, emoji, message_id, channel_id, user_id):
        """
        Called when an emoji is removed
        """
        target_channel = await self.bot.postgres_controller.get_target_channel(channel_id, message_id)
        if not target_channel:
            return
        channel = self.bot.get_channel(target_channel)
        user = self.bot.get_user(user_id)
        await self.remove_perms(user, channel)

    async def add_perms(self, user, channel):
        """
        Adds a user to channels perms
        """
        try:
            await channel.set_permissions(user, read_messages=True)
        except Exception as e:
            self.bot.logger.warning(f'{e}')

    async def remove_perms(self, user, channel):
        """
        removes a users perms on a channel
        """
        try:
            await channel.set_permissions(user, read_messages=False)
        except Exception as e:
            self.bot.logger.warning(f'{e}')
<|endoftext|>


<commit_before>import os

# Use the same directories as scrapers_ca_app.
CACHE_DIR = os.path.join(os.getcwd(), '..', '_cache')
SCRAPED_DATA_DIR = os.path.join(os.getcwd(), '..', '_data')
DATABASE_URL = os.environ.get('DATABASE_URL', 'postgis://localhost/pupa')
os.environ['OCD_DIVISION_CSV'] = os.environ.get('OCD_DIVISION_CSV', os.path.join(os.path.abspath(os.path.dirname(__file__)), 'country-{}.csv'))

# @see https://github.com/opencivicdata/pupa/blob/master/pupa/settings.py
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': "%(asctime)s %(levelname)s %(name)s: %(message)s",
            'datefmt': '%H:%M:%S'
        }
    },
    'handlers': {
        'default': {'level': 'DEBUG',
                    'class': 'pupa.ext.ansistrm.ColorizingStreamHandler',
                    'formatter': 'standard'},
    },
    'loggers': {
        '': {
            'handlers': ['default'], 'level': 'DEBUG', 'propagate': True  # DEBUG
        },
        'scrapelib': {
            'handlers': ['default'], 'level': 'INFO', 'propagate': False  # INFO
        },
        'requests': {
            'handlers': ['default'], 'level': 'WARN', 'propagate': False
        },
        'boto3': {
            'handlers': ['default'], 'level': 'WARN', 'propagate': False
        },
    },
}
<commit_msg>working scraper<commit_after>import os

# Use the same directories as scrapers_ca_app.
CACHE_DIR = os.path.join(os.getcwd(), '..', '_cache')
SCRAPED_DATA_DIR = os.path.join(os.getcwd(), '..', '_data')
DATABASE_URL = os.environ.get('DATABASE_URL', 'postgis://root:root@localhost/pupa')
os.environ['OCD_DIVISION_CSV'] = os.environ.get('OCD_DIVISION_CSV', os.path.join(os.path.abspath(os.path.dirname(__file__)), 'country-{}.csv'))

# @see https://github.com/opencivicdata/pupa/blob/master/pupa/settings.py
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': "%(asctime)s %(levelname)s %(name)s: %(message)s",
            'datefmt': '%H:%M:%S'
        }
    },
    'handlers': {
        'default': {'level': 'DEBUG',
                    'class': 'pupa.ext.ansistrm.ColorizingStreamHandler',
                    'formatter': 'standard'},
    },
    'loggers': {
        '': {
            'handlers': ['default'], 'level': 'DEBUG', 'propagate': True  # DEBUG
        },
        'scrapelib': {
            'handlers': ['default'], 'level': 'INFO', 'propagate': False  # INFO
        },
        'requests': {
            'handlers': ['default'], 'level': 'WARN', 'propagate': False
        },
        'boto3': {
            'handlers': ['default'], 'level': 'WARN', 'propagate': False
        },
    },
}
<|endoftext|>