<commit_before>from tyr.servers import Server
import logging

class MongoDataWarehousingNode(Server):

    log = logging.getLogger('Servers.MongoDataWarehousingNode')
    log.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
            '%(asctime)s [%(name)s] %(levelname)s: %(message)s',
            datefmt = '%H:%M:%S')
    ch.setFormatter(formatter)
    log.addHandler(ch)

    def __init__(self, dry = None, verbose = None, size = None, cluster = None,
                    environment = None, ami = None, region = None, role = None,
                    keypair = None, availability_zone = None, chef_path = None,
                    security_groups = None, block_devices = None,
                    replica_set = None, data_volume_size=None,
                    data_volume_iops=None, role_policies=None):

        super(MongoDataNode, self).__init__(dry, verbose, size, cluster,
                                            environment, ami, region, role,
                                            keypair, availability_zone,
                                            security_groups, block_devices,
                                            role_policies)

        self.replica_set = replica_set
        self.chef_path = chef_path
        self.data_volume_size = data_volume_size
        self.data_volume_iops = data_volume_iops

    def configure(self):

        if self.role_policies is None:

            self.role_policies = {
                'allow-volume-control': """{
    "Statement": [
        {
            "Sid": "Stmt1367531520227",
            "Action": [
                "ec2:AttachVolume",
                "ec2:CreateVolume",
                "ec2:DescribeVolumeAttribute",
                "ec2:DescribeVolumeStatus",
                "ec2:DescribeVolumes",
                "ec2:EnableVolumeIO",
                "ec2:DetachVolume"
             ],
             "Effect": "Allow",
             "Resource": [
                "*"
             ]
        }
     ]
}"""
            }

        super(MongoDataNode, self).configure()

        if self.replica_set is None:
            self.log.warn('No replica set provided')
            self.replica_set = 1

        self.log.info('Using replica set "{set}"'.format(set=self.replica_set))

        if self.data_volume_size is None:
            self.log.warn('No data volume size provided')
            self.data_volume_size = 400
        elif self.data_volume_size < 1:
            self.log.critical('The data volume size is less than 1')
            sys.exit(1)

        self.log.info('Using data volume size "{size}"'.format(
                                            size = self.data_volume_size))

        if self.data_volume_iops is None:
            self.log.warn('No data volume iops provided')
            if self.environment == 'prod':
                self.data_volume_iops = 3000
            else:
                self.data_volume_iops = 0

        self.log.info('Using data volume iops "{iops}"'.format(
                                            iops = self.data_volume_iops))

        iops_size_ratio = self.data_volume_iops/self.data_volume_size

        self.log.info('The IOPS to Size ratio is "{ratio}"'.format(
                                            ratio = iops_size_ratio))

        if iops_size_ratio > 30:
            self.log.critical('The IOPS to Size ratio is greater than 30')
            sys.exit(1)

<commit_msg>Defined the name property for the Mongo Data Warehousing Node<commit_after>from tyr.servers import Server
import logging

class MongoDataWarehousingNode(Server):

    log = logging.getLogger('Servers.MongoDataWarehousingNode')
    log.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
            '%(asctime)s [%(name)s] %(levelname)s: %(message)s',
            datefmt = '%H:%M:%S')
    ch.setFormatter(formatter)
    log.addHandler(ch)

    def __init__(self, dry = None, verbose = None, size = None, cluster = None,
                    environment = None, ami = None, region = None, role = None,
                    keypair = None, availability_zone = None, chef_path = None,
                    security_groups = None, block_devices = None,
                    replica_set = None, data_volume_size=None,
                    data_volume_iops=None, role_policies=None):

        super(MongoDataNode, self).__init__(dry, verbose, size, cluster,
                                            environment, ami, region, role,
                                            keypair, availability_zone,
                                            security_groups, block_devices,
                                            role_policies)

        self.replica_set = replica_set
        self.chef_path = chef_path
        self.data_volume_size = data_volume_size
        self.data_volume_iops = data_volume_iops

    def configure(self):

        if self.role_policies is None:

            self.role_policies = {
                'allow-volume-control': """{
    "Statement": [
        {
            "Sid": "Stmt1367531520227",
            "Action": [
                "ec2:AttachVolume",
                "ec2:CreateVolume",
                "ec2:DescribeVolumeAttribute",
                "ec2:DescribeVolumeStatus",
                "ec2:DescribeVolumes",
                "ec2:EnableVolumeIO",
                "ec2:DetachVolume"
             ],
             "Effect": "Allow",
             "Resource": [
                "*"
             ]
        }
     ]
}"""
            }

        super(MongoDataNode, self).configure()

        if self.replica_set is None:
            self.log.warn('No replica set provided')
            self.replica_set = 1

        self.log.info('Using replica set "{set}"'.format(set=self.replica_set))

        if self.data_volume_size is None:
            self.log.warn('No data volume size provided')
            self.data_volume_size = 400
        elif self.data_volume_size < 1:
            self.log.critical('The data volume size is less than 1')
            sys.exit(1)

        self.log.info('Using data volume size "{size}"'.format(
                                            size = self.data_volume_size))

        if self.data_volume_iops is None:
            self.log.warn('No data volume iops provided')
            if self.environment == 'prod':
                self.data_volume_iops = 3000
            else:
                self.data_volume_iops = 0

        self.log.info('Using data volume iops "{iops}"'.format(
                                            iops = self.data_volume_iops))

        iops_size_ratio = self.data_volume_iops/self.data_volume_size

        self.log.info('The IOPS to Size ratio is "{ratio}"'.format(
                                            ratio = iops_size_ratio))

        if iops_size_ratio > 30:
            self.log.critical('The IOPS to Size ratio is greater than 30')
            sys.exit(1)

    @property
    def name(self):

        try:
            return self.unique_name
        except Exception:
            pass

        name = self.build_name(
            template='{envcl}-rs{set}-{zone}-fulla',
            supplemental={
                'set': self.replica_set,
                'zone': self.availability_zone[-1:]
            },
            search_prefix='{envcl}-rs{set}-{zone}-')

        self.unique_name = name

        self.log.info('Using node name {name}'.format(name=name))

        return name

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys

os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'

if __name__ == "__main__":

    from django.core.management import execute_from_command_line

    execute_from_command_line(sys.argv)
<commit_msg>minor manage.py code update<commit_after>#!/usr/bin/env python
import os
import sys

if __name__ == "__main__":
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')

    from django.core.management import execute_from_command_line

    execute_from_command_line(sys.argv)
<|endoftext|>


<commit_before>from django.db import models as django_models
from django.test import TestCase, mock
from django.contrib.contenttypes.models import ContentType

from .. import models
from .. import settings

from .utils import get_image_in_memory_data

class BaseTestModel(django_models.Model):
    name = django_models.CharField(max_length=100)

    def __str__(self):
        return self.name

    class Meta:
        abstract = True

class TestModel(models.ContentGalleryMixin, BaseTestModel):
    pass

class AnotherTestModel(models.ContentGalleryMixin, BaseTestModel):
    gallery_visible = False

class WrongTestModel(BaseTestModel):
    pass

class ImageTestCase(TestCase):

    @classmethod
    def setUpClass(cls):
        clean_db()
        cls.object = TestModel.objects.create(name="TestObject")

    @mock.patch('gallery.models.slugify_unique', return_value='foo')
    def setUp(self, slugify_unique):
        self.image = models.Image.objects.create(
            image=get_image_in_memory_data(),
            position=0,
            content_type=ContentType.objects.get_for_model(TestModel),
            object_id=self.object.id
        )
        self.image = self.get_image()

    def tearDown(self):
        self.image.delete()

    @classmethod
    def tearDownClass(cls):
        cls.object.delete()

    @staticmethod
    def get_image():
        return models.Image.objects.all()[0]

    @staticmethod
    def get_name(name):
        return "/".join([settings.GALLERY_PATH, name])

class MultipleObjectsImageTestCase(ImageTestCase):

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.second_object = TestModel.objects.create(
            name="SecondTestObject"
        )
        cls.another_object = AnotherTestModel.objects.create(
            name="AnotherTestObject"
        )

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()
        cls.second_object.delete()
        cls.another_object.delete()

class ViewsTestCase(TestCase):

    @classmethod
    def setUpClass(cls):
        clean_db()
        cls.ctype = ContentType.objects.get_for_model(TestModel)
        cls.object = TestModel.objects.create(name="Test object")
        cls.image1 = models.Image.objects.create(
            image=get_image_in_memory_data(),
            position=0,
            content_type=cls.ctype,
            object_id=cls.object.id
        )
        cls.image2 = models.Image.objects.create(
            image=get_image_in_memory_data(),
            position=1,
            content_type=cls.ctype,
            object_id=cls.object.id
        )
        cls.alone_object = TestModel.objects.create(
            name="Alone test object"
        )

    @classmethod
    def tearDownClass(cls):
        cls.image1.delete()
        cls.image2.delete()
        cls.object.delete()
        cls.alone_object.delete()

class AjaxRequestMixin:

    def send_ajax_request(self, url):
        return self.client.get(
            url,
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )

def clean_db():
    TestModel.objects.all().delete()
    AnotherTestModel.objects.all().delete()
    WrongTestModel.objects.all().delete()
    models.Image.objects.all().delete()
<commit_msg>Removed hardcoding 'gallery'<commit_after>from django.db import models as django_models
from django.test import TestCase, mock
from django.contrib.contenttypes.models import ContentType

from .. import models
from .. import settings

from .utils import get_image_in_memory_data

class BaseTestModel(django_models.Model):
    name = django_models.CharField(max_length=100)

    def __str__(self):
        return self.name

    class Meta:
        abstract = True

class TestModel(models.ContentGalleryMixin, BaseTestModel):
    pass

class AnotherTestModel(models.ContentGalleryMixin, BaseTestModel):
    gallery_visible = False

class WrongTestModel(BaseTestModel):
    pass

class ImageTestCase(TestCase):

    @classmethod
    def setUpClass(cls):
        clean_db()
        cls.object = TestModel.objects.create(name="TestObject")

    def setUp(self):
        with mock.patch.object(
            models,
            'slugify_unique',
            return_value='foo'
        ):
            self.image = models.Image.objects.create(
                image=get_image_in_memory_data(),
                position=0,
                content_type=ContentType.objects.get_for_model(TestModel),
                object_id=self.object.id
            )
        self.image = self.get_image()

    def tearDown(self):
        self.image.delete()

    @classmethod
    def tearDownClass(cls):
        cls.object.delete()

    @staticmethod
    def get_image():
        return models.Image.objects.all()[0]

    @staticmethod
    def get_name(name):
        return "/".join([settings.GALLERY_PATH, name])

class MultipleObjectsImageTestCase(ImageTestCase):

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.second_object = TestModel.objects.create(
            name="SecondTestObject"
        )
        cls.another_object = AnotherTestModel.objects.create(
            name="AnotherTestObject"
        )

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()
        cls.second_object.delete()
        cls.another_object.delete()

class ViewsTestCase(TestCase):

    @classmethod
    def setUpClass(cls):
        clean_db()
        cls.ctype = ContentType.objects.get_for_model(TestModel)
        cls.object = TestModel.objects.create(name="Test object")
        cls.image1 = models.Image.objects.create(
            image=get_image_in_memory_data(),
            position=0,
            content_type=cls.ctype,
            object_id=cls.object.id
        )
        cls.image2 = models.Image.objects.create(
            image=get_image_in_memory_data(),
            position=1,
            content_type=cls.ctype,
            object_id=cls.object.id
        )
        cls.alone_object = TestModel.objects.create(
            name="Alone test object"
        )

    @classmethod
    def tearDownClass(cls):
        cls.image1.delete()
        cls.image2.delete()
        cls.object.delete()
        cls.alone_object.delete()

class AjaxRequestMixin:

    def send_ajax_request(self, url):
        return self.client.get(
            url,
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )

def clean_db():
    TestModel.objects.all().delete()
    AnotherTestModel.objects.all().delete()
    WrongTestModel.objects.all().delete()
    models.Image.objects.all().delete()
<|endoftext|>


<commit_before>__version__ = "0.13.1"

assert len(__version__) <= 10  # The log table limits version to the 10 characters
<commit_msg>Bump version.<commit_after>__version__ = "0.13.2"

assert len(__version__) <= 10  # The log table limits version to the 10 characters
<|endoftext|>


<commit_before>import os

from .base import *

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    ('Sym Roe', 'lmtools@talusdesign.co.uk'),
)

MANAGERS = ADMINS

import dj_database_url
DATABASES['default'] = dj_database_url.config()
DATABASES['default']['ENGINE'] = 'django_postgrespool'

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME']
AWS_S3_HOST = os.environ['AWS_S3_HOST']

REDIS_URL = os.environ['REDISCLOUD_URL']

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
        },
    },
    'root': {
        'handlers': ['console'],
        'level': 'INFO',
    },
    'loggers': {
        'django.request': {  # debug logging of things that break requests
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
        'report.tasks': {  # debug logging of report tasks
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
    },
}
<commit_msg>Increase log threshold for report tasks to INFO<commit_after>import os

from .base import *

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    ('Sym Roe', 'lmtools@talusdesign.co.uk'),
)

MANAGERS = ADMINS

import dj_database_url
DATABASES['default'] = dj_database_url.config()
DATABASES['default']['ENGINE'] = 'django_postgrespool'

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
AWS_STORAGE_BUCKET_NAME = os.environ['AWS_STORAGE_BUCKET_NAME']
AWS_S3_HOST = os.environ['AWS_S3_HOST']

REDIS_URL = os.environ['REDISCLOUD_URL']

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
        },
    },
    'root': {
        'handlers': ['console'],
        'level': 'INFO',
    },
    'loggers': {
        'django.request': {  # debug logging of things that break requests
            'handlers': ['console'],
            'level': 'DEBUG',
            'propagate': True,
        },
        'report.tasks': {  # debug logging of report tasks
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True,
        },
    },
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
import socket
from os import environ as env
from tornado.netutil import Resolver
from tornado import gen
from tornado.httpclient import AsyncHTTPClient

class UnixResolver(Resolver):

    def initialize(self, resolver):
        self.resolver = resolver

    def close(self):
        self.resolver.close()

    @gen.coroutine
    def resolve(self, host, port, *args, **kwargs):

        scheme, path = Storage.DOCKER.split('://')
        if host == 'docker':

            if scheme == 'unix':
                raise gen.Return([(socket.AF_UNIX, path)])

            elif scheme == 'tcp':
                t = path.split(":")
                if len(t) > 1:
                    host, port = t
                    port = int(port)
                else:
                    host, port = t[0], 80

        result = yield self.resolver.resolve(host, port, *args, **kwargs)
        raise gen.Return(result)

AsyncHTTPClient.configure(
    None,
    resolver=UnixResolver(resolver=Resolver()),
    max_clients=20000
)

class Storage(object):
    CONTAINERS = set([])
    DOCKER = env.get('DOCKER_HOST', 'unix:///var/run/docker.sock')
    ELASTICSEARCH = env.get('ELASTICSEARCH', 'http://127.0.0.1:9200')
    http = AsyncHTTPClient()
<commit_msg>[fix] fix resolver<commit_after>#!/usr/bin/env python
# encoding: utf-8
import socket
from os import environ as env
from tornado.netutil import Resolver
from tornado import gen
from tornado.httpclient import AsyncHTTPClient

class UnixResolver(Resolver):

    def initialize(self, resolver):
        self.resolver = resolver

    def close(self):
        self.resolver.close()

    @gen.coroutine
    def resolve(self, host, port, *args, **kwargs):

        scheme, path = Storage.DOCKER.split('://')
        if host == 'docker':

            if scheme == 'unix':
                raise gen.Return([(socket.AF_UNIX, path)])

            elif scheme == 'tcp' or scheme == 'http':
                t = path.split(":")
                if len(t) > 1:
                    host, port = t
                    port = int(port)
                else:
                    host, port = t[0], 80

        result = yield self.resolver.resolve(host, port, *args, **kwargs)
        raise gen.Return(result)

AsyncHTTPClient.configure(
    None,
    resolver=UnixResolver(resolver=Resolver()),
    max_clients=20000
)

class Storage(object):
    CONTAINERS = set([])
    DOCKER = env.get('DOCKER_HOST', 'unix:///var/run/docker.sock')
    ELASTICSEARCH = env.get('ELASTICSEARCH', 'http://127.0.0.1:9200')
    http = AsyncHTTPClient()
<|endoftext|>


<commit_before>import sys
import seaborn as sns

sys.path.append('..')
from charts import *
from bamboo import *

countries = pd.read_csv("datasets/countries.csv")[["country", "continent", "flag"]].split_columns('country', "|").split_rows('country').set_index('country')

df = pd.read_csv("datasets/nobels.csv")
df = df[df['category'] == "Literature"][["name", "countries"]].split_columns("countries", "|")
df = df.assign_rows(continents=lambda r: tuple(sorted(set(countries.continent[c] for c in r.countries))))

NORDIC = ["Iceland", "Finland", "Sweden", "Norway", "Denmark"]
nordic_counts = df.update_columns(countries=lambda cs: tuple(sorted(set(c for c in cs if c in NORDIC)))).filter_rows(lambda r: any(r.countries)).groupby("countries").count().name
continent_counts = df.groupby("continents").count().name
counts = pd.concat((nordic_counts, continent_counts))

# chart

categories = { "Old World": ["Asia", "Africa"], "New World": ["South America", "North America", "Oceania"], "Europe": ["Europe"], "Nordic": NORDIC }
catlabels = { "Europe": "Europe (740m)", "New World": "Americas & Oceania (1,040m)", "Old World": "Asia & Africa (5,650m)", "Nordic": "Nordic countries (27m)" }

table = pd.DataFrame([{subcat: counts[subcat] if any(c in subcat for c in categories[cat]) else 0 for subcat in counts.index } for cat in categories], index=categories)
table = table.assign_rows(sum=lambda r: r.sum()).sort_values("sum", ascending=False).drop("sum", axis=1)
table = table[[ # hack #1 to get nice ordering
("Sweden",), ("Denmark",), ("Norway",), ("Finland",), ("Iceland",),
("Europe",),
("North America",), ("Europe", "North America"),
("Asia",), ("Asia", "Europe"),
("South America",), ("Europe", "South America"),
("Europe", "Oceania"),
("Africa",), ("Africa", "Europe"), ("Africa", "Oceania")
]]

WIDTH = 80
BAR = 3

PALETTE = ImageColor.from_floats(sns.xkcd_palette(["windows blue", "faded green", "amber", "dusty purple", "red", "brown"]))
CONTINENTS = [ "Europe", "North America", "South America",  "Oceania", "Asia","Africa" ]
def continent_colour(c):
    return PALETTE[CONTINENTS.index(c)]

def stripe(c1, c2=None):
    if c2 is None: c2 = c1
    return Image.from_column([Image.new("RGBA", (100,BAR), c1), Image.new("RGBA", (100,BAR), c2)])

def stripe_pattern(height, c1, c2=None):
    return Image.from_pattern(stripe(c1, c2), (WIDTH,height))

def colorfn(c,r):
    cs = table.columns[c] if isinstance(c, int) else (c,)
    if len(cs) == 2 and cs[0] not in categories[table.index[r]]: cs = cs[1], cs[0]
    if any(c in CONTINENTS for c in cs):
        return lambda size: stripe_pattern(size[1], *[continent_colour(c) for c in reversed(cs)])
    flagcolors = sorted(Image.from_url_with_cache(countries.flag[cs[0]]).convert("RGBA").getcolors(), reverse=True)
    return lambda size: Image.from_row([Image.new("RGBA", (16, size[1]), flagcolors[0][1]),
                                       Image.new("RGBA", (8, size[1]), flagcolors[1][1]),
                                       Image.new("RGBA", (WIDTH-16-8, size[1]), flagcolors[0][1])])

def rlabelfn(r):
    return Image.from_text(catlabels[table.index[r]], arial(14, bold=False), "black", "white", align="center", padding=2, max_width=WIDTH)

ymax = 100
chart = bar_chart(table, WIDTH, BAR*2*ymax, type=BarChartType.STACKED, spacing=10, colors=colorfn, grid_interval=10, tick_interval=5, ylabels=arial(14), rlabels=rlabelfn, bg="white", fg="black", ylabel=Image.from_text("# Nobel Literature Prize laureates", arial(14), padding=(0,2,0,10), bg="white").transpose(Image.ROTATE_90), ymax=ymax)

def lbox(c):
    count = counts.select(lambda cs: c in cs).sum()
    box = colorfn(c, 0)((WIDTH,WIDTH)).resize_fixed_aspect(width=BOXSIZE)
    return box.place(Image.from_text(str(count), arial(14), "black" if c == "Finland" else "white"))

BOXSIZE = 30
cboxes = [[lbox(c), Image.from_text(c, arial(14), padding=(5,0), fg="black", bg="white")] for c in CONTINENTS]
clegs = Image.from_array(cboxes, bg="white", xalign=0)
nboxes = [[lbox(c), Image.from_text(c, arial(14), padding=(5,0), fg="black", bg="white")] for c in sorted(NORDIC)]
nlegs = Image.from_array(nboxes, bg="white", xalign=0)
legend = Image.from_column([
Image.from_text("Continents", arial(14, bold=True)),
clegs,
Image.from_text("(stripes indicate winners with dual nationalities)", arial(14), max_width=150, padding=(0,0,0,10)),
Image.from_text("Countries", arial(14, bold=True)),
nlegs
], bg="white", padding=(0,3), xalign=0).pad(5, "white").pad(1, "black")

chart = Image.from_row([chart, legend], bg="white", yalign=0, padding=5)
title = Image.from_text("Geographic biases of the Nobel Prize in Literature", arial(24, bold=True)).pad((0,4,0,8), "white")
img = Image.from_column([title, chart], bg="white")
img.place(Image.from_text("/u/Udzu", font("arial", 16), fg="black", bg="white", padding=5).pad((1,1,0,0), "black"), align=1, padding=10, copy=False)
img.save("output/litnobels.png")
<commit_msg>[dataviz] typos<commit_after>import sys
import seaborn as sns

sys.path.append('..')
from charts import *
from bamboo import *

countries = pd.read_csv("datasets/countries.csv")[["country", "continent", "flag"]].split_columns('country', "|").split_rows('country').set_index('country')

df = pd.read_csv("datasets/nobels.csv")
df = df[df['category'] == "Literature"][["name", "countries"]].split_columns("countries", "|")
df = df.assign_rows(continents=lambda r: tuple(sorted(set(countries.continent[c] for c in r.countries))))

NORDIC = ["Iceland", "Finland", "Sweden", "Norway", "Denmark"]
nordic_counts = df.update_columns(countries=lambda cs: tuple(sorted(set(c for c in cs if c in NORDIC)))).filter_rows(lambda r: any(r.countries)).groupby("countries").count().name
continent_counts = df.groupby("continents").count().name
counts = pd.concat((nordic_counts, continent_counts))

# chart

categories = { "Old World": ["Asia", "Africa"], "New World": ["South America", "North America", "Oceania"], "Europe": ["Europe"], "Nordic": NORDIC }
catlabels = { "Europe": "Europe (740m)", "New World": "Americas & Oceania (1,040m)", "Old World": "Asia & Africa (5,650m)", "Nordic": "Nordic countries (27m)" }

table = pd.DataFrame([{subcat: counts[subcat] if any(c in subcat for c in categories[cat]) else 0 for subcat in counts.index } for cat in categories], index=categories)
table = table.assign_rows(sum=lambda r: r.sum()).sort_values("sum", ascending=False).drop("sum", axis=1)
table = table[[ # hack #1 to get nice ordering
("Sweden",), ("Denmark",), ("Norway",), ("Finland",), ("Iceland",),
("Europe",),
("North America",), ("Europe", "North America"),
("Asia",), ("Asia", "Europe"),
("South America",), ("Europe", "South America"),
("Europe", "Oceania"),
("Africa",), ("Africa", "Europe"), ("Africa", "Oceania")
]]

WIDTH = 80
BAR = 3

PALETTE = ImageColor.from_floats(sns.xkcd_palette(["windows blue", "faded green", "amber", "dusty purple", "red", "brown"]))
CONTINENTS = [ "Europe", "North America", "South America",  "Oceania", "Asia","Africa" ]
def continent_colour(c):
    return PALETTE[CONTINENTS.index(c)]

def stripe(c1, c2=None):
    if c2 is None: c2 = c1
    return Image.from_column([Image.new("RGBA", (100,BAR), c1), Image.new("RGBA", (100,BAR), c2)])

def stripe_pattern(height, c1, c2=None):
    return Image.from_pattern(stripe(c1, c2), (WIDTH,height))

def colorfn(c,r):
    cs = table.columns[c] if isinstance(c, int) else (c,)
    if len(cs) == 2 and cs[0] not in categories[table.index[r]]: cs = cs[1], cs[0]
    if any(c in CONTINENTS for c in cs):
        return lambda size: stripe_pattern(size[1], *[continent_colour(c) for c in reversed(cs)])
    flagcolors = sorted(Image.from_url_with_cache(countries.flag[cs[0]]).convert("RGBA").getcolors(), reverse=True)
    return lambda size: Image.from_row([Image.new("RGBA", (16, size[1]), flagcolors[0][1]),
                                       Image.new("RGBA", (8, size[1]), flagcolors[1][1]),
                                       Image.new("RGBA", (WIDTH-16-8, size[1]), flagcolors[0][1])])

def rlabelfn(r):
    return Image.from_text(catlabels[table.index[r]], arial(14, bold=False), "black", "white", align="center", padding=2, max_width=WIDTH)

ymax = 100
chart = bar_chart(table, WIDTH, BAR*2*ymax, type=BarChartType.STACKED, spacing=10, colors=colorfn, grid_interval=10, tick_interval=5, ylabels=arial(14), rlabels=rlabelfn, bg="white", fg="black", ylabel=Image.from_text("# Nobel Literature laureates", arial(14), padding=(0,2,0,10), bg="white").transpose(Image.ROTATE_90), ymax=ymax)

def lbox(c):
    count = counts.select(lambda cs: c in cs).sum()
    box = colorfn(c, 0)((WIDTH,WIDTH)).resize_fixed_aspect(width=BOXSIZE)
    return box.place(Image.from_text(str(count), arial(14), "black" if c == "Finland" else "white"))

BOXSIZE = 30
cboxes = [[lbox(c), Image.from_text(c, arial(14), padding=(5,0), fg="black", bg="white")] for c in CONTINENTS]
clegs = Image.from_array(cboxes, bg="white", xalign=0)
nboxes = [[lbox(c), Image.from_text(c, arial(14), padding=(5,0), fg="black", bg="white")] for c in sorted(NORDIC)]
nlegs = Image.from_array(nboxes, bg="white", xalign=0)
legend = Image.from_column([
Image.from_text("Continents", arial(14, bold=True)),
clegs,
Image.from_text("(stripes indicate winners with dual nationalities)", arial(14), max_width=150, padding=(0,0,0,10)),
Image.from_text("Countries", arial(14, bold=True)),
nlegs
], bg="white", padding=(0,3), xalign=0).pad(5, "white").pad(1, "black")

chart = Image.from_row([chart, legend], bg="white", yalign=0, padding=5)
title = Image.from_text("Geographic distribution of Literature Nobel laureates", arial(24, bold=True)).pad((0,4,0,8), "white")
img = Image.from_column([title, chart], bg="white")
img.place(Image.from_text("/u/Udzu", font("arial", 16), fg="black", bg="white", padding=5).pad((1,1,0,0), "black"), align=1, padding=10, copy=False)
img.save("output/litnobels.png")
<|endoftext|>


<commit_before>from slm_lab.agent import net
from slm_lab.agent.algorithm import policy_util
from slm_lab.agent.algorithm.actor_critic import ActorCritic
from slm_lab.agent.net import net_util
from slm_lab.lib import logger, util
from slm_lab.lib.decorator import lab_api
import numpy as np
import torch
import torch.nn.functional as F

logger = logger.get_logger(__name__)

class SoftActorCritic(ActorCritic):
    '''
    Implementation of Soft Actor-Critic (SAC)
    Original paper: "Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor"
    https://arxiv.org/abs/1801.01290
    Improvement of SAC paper: "Soft Actor-Critic Algorithms and Applications"
    https://arxiv.org/abs/1812.05905

    e.g. algorithm_spec
    "algorithm": {
        "name": "SoftActorCritic",
        "action_pdtype": "default",
        "action_policy": "default",
        "gamma": 0.99,
        "training_frequency": 1,
    }
    '''
    @lab_api
    def init_algorithm_params(self):
        '''Initialize other algorithm parameters'''
        # set default
        util.set_attr(self, dict(
            action_pdtype='default',
            action_policy='default',
            training_iter=self.body.env.num_envs,
            training_start_step=self.body.memory.batch_size,
        ))
        util.set_attr(self, self.algorithm_spec, [
            'action_pdtype',
            'action_policy',
            'gamma',  # the discount factor
            'training_iter',
            'training_frequency',
            'training_start_step',
        ])
        if self.body.is_discrete:
            assert self.action_pdtype == 'GumbelSoftmax'
        self.to_train = 0
        self.action_policy = getattr(policy_util, self.action_policy)

    @lab_api
    def init_nets(self, global_nets=None):
        '''
        Networks: net(actor/policy), q1_net, target_q1_net, q2_net, target_q2_net
        All networks are separate, and have the same hidden layer architectures and optim specs, so tuning is minimal
        '''
        self.shared = False  # SAC does not share networks
        NetClass = getattr(net, self.net_spec['type'])
        # main actor network
        self.net = NetClass(self.net_spec, self.body.state_dim, net_util.get_out_dim(self.body))
        self.net_names = ['net']
        # two critic Q-networks to mitigate positive bias in q_loss and speed up training, uses q_net.py with prefix Q
        QNetClass = getattr(net, 'Q' + self.net_spec['type'])
        q_in_dim = [self.body.state_dim, self.body.action_dim]
        self.q1_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.target_q1_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.q2_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.target_q2_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.net_names += ['q1_net', 'target_q1_net', 'q2_net', 'target_q2_net']
        net_util.copy(self.q1_net, self.target_q1_net)
        net_util.copy(self.q2_net, self.target_q2_net)
        # temperature variable to be learned, and its target entropy
        self.log_alpha = torch.zeros(1, requires_grad=True, device=self.net.device)
        self.alpha = self.log_alpha.detach().exp()
        self.target_entropy = - np.product(self.body.action_space.shape)

        # init net optimizer and its lr scheduler
        self.optim = net_util.get_optim(self.net, self.net.optim_spec)
        self.lr_scheduler = net_util.get_lr_scheduler(self.optim, self.net.lr_scheduler_spec)
        self.q1_optim = net_util.get_optim(self.q1_net, self.q1_net.optim_spec)
        self.q1_lr_scheduler = net_util.get_lr_scheduler(self.q1_optim, self.q1_net.lr_scheduler_spec)
        self.q2_optim = net_util.get_optim(self.q2_net, self.q2_net.optim_spec)
        self.q2_lr_scheduler = net_util.get_lr_scheduler(self.q2_optim, self.q2_net.lr_scheduler_spec)
        self.alpha_optim = net_util.get_optim(self.log_alpha, self.net.optim_spec)
        self.alpha_lr_scheduler = net_util.get_lr_scheduler(self.alpha_optim, self.net.lr_scheduler_spec)
        net_util.set_global_nets(self, global_nets)
        self.post_init_nets()

    @lab_api
    def act(self, state):
        if self.body.env.clock.frame < self.training_start_step:
            return policy_util.random(state, self, self.body).cpu().squeeze().numpy()
        else:
            action = self.action_policy(state, self, self.body)
            if not self.body.is_discrete:
                action = self.scale_action(torch.tanh(action))  # continuous action bound
            return action.cpu().squeeze().numpy()

    def scale_action(self, action):
        '''Scale continuous actions from tanh range'''
        action_space = self.body.action_space
        low, high = torch.from_numpy(action_space.low), torch.from_numpy(action_space.high)
        return action * (high - low) / 2 + (low + high) / 2

    def guard_q_actions(self, actions):
        '''Guard to convert actions to one-hot for input to Q-network'''
        if self.body.is_discrete:
            # TODO support multi-discrete actions
            actions = F.one_hot(actions.long(), self.body.action_dim).float()
        return actions

    def calc_log_prob_action(self, action_pd, reparam=False):
        '''Calculate log_probs and actions with option to reparametrize from paper eq. 11'''
        samples = action_pd.rsample() if reparam else action_pd.sample()
        if self.body.is_discrete:  # this is straightforward using GumbelSoftmax
            actions = samples
            log_probs = action_pd.log_prob(actions)
        else:
            mus = samples
            actions = self.scale_action(torch.tanh(mus))
            if actions.dim() == 1:  # handle shape consistency for single actions
                actions = actions.unsqueeze(dim=0)
            # paper Appendix C. Enforcing Action Bounds for continuous actions
            log_probs = (action_pd.log_prob(mus) - torch.log(1 - actions.pow(2) + 1e-6).sum(1))
        return log_probs, actions

    def calc_q(self, state, action, net):
        '''Forward-pass to calculate the predicted state-action-value from q1_net.'''
        if not self.body.is_discrete and action.dim() == 1:  # handle shape consistency for single continuous action
            action = action.unsqueeze(dim=0)
        q_pred = net(state, action).view(-1)
        return q_pred

    def calc_q_targets(self, batch):
        '''Q_tar = r + gamma * (target_Q(s', a') - alpha * log pi(a'|s'))'''
        next_states = batch['next_states']
        with torch.no_grad():
            pdparams = self.calc_pdparam(next_states)
            action_pd = policy_util.init_action_pd(self.body.ActionPD, pdparams)
            next_log_probs, next_actions = self.calc_log_prob_action(action_pd)
            next_actions = self.guard_q_actions(next_actions)  # non-reparam discrete actions need to be converted into one-hot

            next_target_q1_preds = self.calc_q(next_states, next_actions, self.target_q1_net)
            next_target_q2_preds = self.calc_q(next_states, next_actions, self.target_q2_net)
            next_target_q_preds = torch.min(next_target_q1_preds, next_target_q2_preds)
            q_targets = batch['rewards'] + self.gamma * (1 - batch['dones']) * (next_target_q_preds - self.alpha * next_log_probs)
        return q_targets

    def calc_reg_loss(self, preds, targets):
        '''Calculate the regression loss for V and Q values, using the same loss function from net_spec'''
        assert preds.shape == targets.shape, f'{preds.shape} != {targets.shape}'
        reg_loss = self.net.loss_fn(preds, targets)
        return reg_loss

    def calc_policy_loss(self, batch, log_probs, reparam_actions):
        '''policy_loss = alpha * log pi(f(a)|s) - Q1(s, f(a)), where f(a) = reparametrized action'''
        states = batch['states']
        q1_preds = self.calc_q(states, reparam_actions, self.q1_net)
        q2_preds = self.calc_q(states, reparam_actions, self.q2_net)
        q_preds = torch.min(q1_preds, q2_preds)
        policy_loss = (self.alpha * log_probs - q_preds).mean()
        return policy_loss

    def calc_alpha_loss(self, log_probs):
        alpha_loss = - (self.log_alpha * (log_probs.detach() + self.target_entropy)).mean()
        return alpha_loss

    def try_update_per(self, q_preds, q_targets):
        if 'Prioritized' in util.get_class_name(self.body.memory):  # PER
            with torch.no_grad():
                errors = (q_preds - q_targets).abs().cpu().numpy()
            self.body.memory.update_priorities(errors)

    def train_alpha(self, alpha_loss):
        '''Custom method to train the alpha variable'''
        self.alpha_lr_scheduler.step(epoch=self.body.env.clock.frame)
        self.alpha_optim.zero_grad()
        alpha_loss.backward()
        self.alpha_optim.step()
        self.alpha = self.log_alpha.detach().exp()

    def train(self):
        '''Train actor critic by computing the loss in batch efficiently'''
        if util.in_eval_lab_modes():
            return np.nan
        clock = self.body.env.clock
        if self.to_train == 1:
            for _ in range(self.training_iter):
                batch = self.sample()
                clock.set_batch_size(len(batch))

                states = batch['states']
                actions = self.guard_q_actions(batch['actions'])
                q_targets = self.calc_q_targets(batch)
                # Q-value loss for both Q nets
                q1_preds = self.calc_q(states, actions, self.q1_net)
                q1_loss = self.calc_reg_loss(q1_preds, q_targets)
                self.q1_net.train_step(q1_loss, self.q1_optim, self.q1_lr_scheduler, clock=clock, global_net=self.global_q1_net)

                q2_preds = self.calc_q(states, actions, self.q2_net)
                q2_loss = self.calc_reg_loss(q2_preds, q_targets)
                self.q2_net.train_step(q2_loss, self.q2_optim, self.q2_lr_scheduler, clock=clock, global_net=self.global_q2_net)

                # policy loss
                action_pd = policy_util.init_action_pd(self.body.ActionPD, self.calc_pdparam(states))
                log_probs, reparam_actions = self.calc_log_prob_action(action_pd, reparam=True)
                policy_loss = self.calc_policy_loss(batch, log_probs, reparam_actions)
                self.net.train_step(policy_loss, self.optim, self.lr_scheduler, clock=clock, global_net=self.global_net)

                # alpha loss
                alpha_loss = self.calc_alpha_loss(log_probs)
                self.train_alpha(alpha_loss)

                loss = q1_loss + q2_loss + policy_loss + alpha_loss
                # update target networks
                self.update_nets()
                # update PER priorities if availalbe
                self.try_update_per(torch.min(q1_preds, q2_preds), q_targets)

            # reset
            self.to_train = 0
            logger.debug(f'Trained {self.name} at epi: {clock.epi}, frame: {clock.frame}, t: {clock.t}, total_reward so far: {self.body.env.total_reward}, loss: {loss:g}')
            return loss.item()
        else:
            return np.nan

    def update_nets(self):
        '''Update target networks'''
        if util.frame_mod(self.body.env.clock.frame, self.q1_net.update_frequency, self.body.env.num_envs):
            if self.q1_net.update_type == 'replace':
                net_util.copy(self.q1_net, self.target_q1_net)
                net_util.copy(self.q2_net, self.target_q2_net)
            elif self.q1_net.update_type == 'polyak':
                net_util.polyak_update(self.q1_net, self.target_q1_net, self.q1_net.polyak_coef)
                net_util.polyak_update(self.q2_net, self.target_q2_net, self.q2_net.polyak_coef)
            else:
                raise ValueError('Unknown q1_net.update_type. Should be "replace" or "polyak". Exiting.')

    @lab_api
    def update(self):
        '''Override parent method to do nothing'''
        return self.body.explore_var
<commit_msg>correct unsqueeze dim<commit_after>from slm_lab.agent import net
from slm_lab.agent.algorithm import policy_util
from slm_lab.agent.algorithm.actor_critic import ActorCritic
from slm_lab.agent.net import net_util
from slm_lab.lib import logger, util
from slm_lab.lib.decorator import lab_api
import numpy as np
import torch
import torch.nn.functional as F

logger = logger.get_logger(__name__)

class SoftActorCritic(ActorCritic):
    '''
    Implementation of Soft Actor-Critic (SAC)
    Original paper: "Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor"
    https://arxiv.org/abs/1801.01290
    Improvement of SAC paper: "Soft Actor-Critic Algorithms and Applications"
    https://arxiv.org/abs/1812.05905

    e.g. algorithm_spec
    "algorithm": {
        "name": "SoftActorCritic",
        "action_pdtype": "default",
        "action_policy": "default",
        "gamma": 0.99,
        "training_frequency": 1,
    }
    '''
    @lab_api
    def init_algorithm_params(self):
        '''Initialize other algorithm parameters'''
        # set default
        util.set_attr(self, dict(
            action_pdtype='default',
            action_policy='default',
            training_iter=self.body.env.num_envs,
            training_start_step=self.body.memory.batch_size,
        ))
        util.set_attr(self, self.algorithm_spec, [
            'action_pdtype',
            'action_policy',
            'gamma',  # the discount factor
            'training_iter',
            'training_frequency',
            'training_start_step',
        ])
        if self.body.is_discrete:
            assert self.action_pdtype == 'GumbelSoftmax'
        self.to_train = 0
        self.action_policy = getattr(policy_util, self.action_policy)

    @lab_api
    def init_nets(self, global_nets=None):
        '''
        Networks: net(actor/policy), q1_net, target_q1_net, q2_net, target_q2_net
        All networks are separate, and have the same hidden layer architectures and optim specs, so tuning is minimal
        '''
        self.shared = False  # SAC does not share networks
        NetClass = getattr(net, self.net_spec['type'])
        # main actor network
        self.net = NetClass(self.net_spec, self.body.state_dim, net_util.get_out_dim(self.body))
        self.net_names = ['net']
        # two critic Q-networks to mitigate positive bias in q_loss and speed up training, uses q_net.py with prefix Q
        QNetClass = getattr(net, 'Q' + self.net_spec['type'])
        q_in_dim = [self.body.state_dim, self.body.action_dim]
        self.q1_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.target_q1_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.q2_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.target_q2_net = QNetClass(self.net_spec, q_in_dim, 1)
        self.net_names += ['q1_net', 'target_q1_net', 'q2_net', 'target_q2_net']
        net_util.copy(self.q1_net, self.target_q1_net)
        net_util.copy(self.q2_net, self.target_q2_net)
        # temperature variable to be learned, and its target entropy
        self.log_alpha = torch.zeros(1, requires_grad=True, device=self.net.device)
        self.alpha = self.log_alpha.detach().exp()
        self.target_entropy = - np.product(self.body.action_space.shape)

        # init net optimizer and its lr scheduler
        self.optim = net_util.get_optim(self.net, self.net.optim_spec)
        self.lr_scheduler = net_util.get_lr_scheduler(self.optim, self.net.lr_scheduler_spec)
        self.q1_optim = net_util.get_optim(self.q1_net, self.q1_net.optim_spec)
        self.q1_lr_scheduler = net_util.get_lr_scheduler(self.q1_optim, self.q1_net.lr_scheduler_spec)
        self.q2_optim = net_util.get_optim(self.q2_net, self.q2_net.optim_spec)
        self.q2_lr_scheduler = net_util.get_lr_scheduler(self.q2_optim, self.q2_net.lr_scheduler_spec)
        self.alpha_optim = net_util.get_optim(self.log_alpha, self.net.optim_spec)
        self.alpha_lr_scheduler = net_util.get_lr_scheduler(self.alpha_optim, self.net.lr_scheduler_spec)
        net_util.set_global_nets(self, global_nets)
        self.post_init_nets()

    @lab_api
    def act(self, state):
        if self.body.env.clock.frame < self.training_start_step:
            return policy_util.random(state, self, self.body).cpu().squeeze().numpy()
        else:
            action = self.action_policy(state, self, self.body)
            if not self.body.is_discrete:
                action = self.scale_action(torch.tanh(action))  # continuous action bound
            return action.cpu().squeeze().numpy()

    def scale_action(self, action):
        '''Scale continuous actions from tanh range'''
        action_space = self.body.action_space
        low, high = torch.from_numpy(action_space.low), torch.from_numpy(action_space.high)
        return action * (high - low) / 2 + (low + high) / 2

    def guard_q_actions(self, actions):
        '''Guard to convert actions to one-hot for input to Q-network'''
        if self.body.is_discrete:
            # TODO support multi-discrete actions
            actions = F.one_hot(actions.long(), self.body.action_dim).float()
        return actions

    def calc_log_prob_action(self, action_pd, reparam=False):
        '''Calculate log_probs and actions with option to reparametrize from paper eq. 11'''
        samples = action_pd.rsample() if reparam else action_pd.sample()
        if self.body.is_discrete:  # this is straightforward using GumbelSoftmax
            actions = samples
            log_probs = action_pd.log_prob(actions)
        else:
            mus = samples
            actions = self.scale_action(torch.tanh(mus))
            if actions.dim() == 1:  # handle shape consistency for single actions
                actions = actions.unsqueeze(dim=-1)
            # paper Appendix C. Enforcing Action Bounds for continuous actions
            log_probs = (action_pd.log_prob(mus) - torch.log(1 - actions.pow(2) + 1e-6).sum(1))
        return log_probs, actions

    def calc_q(self, state, action, net):
        '''Forward-pass to calculate the predicted state-action-value from q1_net.'''
        if not self.body.is_discrete and action.dim() == 1:  # handle shape consistency for single continuous action
            action = action.unsqueeze(dim=-1)
        q_pred = net(state, action).view(-1)
        return q_pred

    def calc_q_targets(self, batch):
        '''Q_tar = r + gamma * (target_Q(s', a') - alpha * log pi(a'|s'))'''
        next_states = batch['next_states']
        with torch.no_grad():
            pdparams = self.calc_pdparam(next_states)
            action_pd = policy_util.init_action_pd(self.body.ActionPD, pdparams)
            next_log_probs, next_actions = self.calc_log_prob_action(action_pd)
            next_actions = self.guard_q_actions(next_actions)  # non-reparam discrete actions need to be converted into one-hot

            next_target_q1_preds = self.calc_q(next_states, next_actions, self.target_q1_net)
            next_target_q2_preds = self.calc_q(next_states, next_actions, self.target_q2_net)
            next_target_q_preds = torch.min(next_target_q1_preds, next_target_q2_preds)
            q_targets = batch['rewards'] + self.gamma * (1 - batch['dones']) * (next_target_q_preds - self.alpha * next_log_probs)
        return q_targets

    def calc_reg_loss(self, preds, targets):
        '''Calculate the regression loss for V and Q values, using the same loss function from net_spec'''
        assert preds.shape == targets.shape, f'{preds.shape} != {targets.shape}'
        reg_loss = self.net.loss_fn(preds, targets)
        return reg_loss

    def calc_policy_loss(self, batch, log_probs, reparam_actions):
        '''policy_loss = alpha * log pi(f(a)|s) - Q1(s, f(a)), where f(a) = reparametrized action'''
        states = batch['states']
        q1_preds = self.calc_q(states, reparam_actions, self.q1_net)
        q2_preds = self.calc_q(states, reparam_actions, self.q2_net)
        q_preds = torch.min(q1_preds, q2_preds)
        policy_loss = (self.alpha * log_probs - q_preds).mean()
        return policy_loss

    def calc_alpha_loss(self, log_probs):
        alpha_loss = - (self.log_alpha * (log_probs.detach() + self.target_entropy)).mean()
        return alpha_loss

    def try_update_per(self, q_preds, q_targets):
        if 'Prioritized' in util.get_class_name(self.body.memory):  # PER
            with torch.no_grad():
                errors = (q_preds - q_targets).abs().cpu().numpy()
            self.body.memory.update_priorities(errors)

    def train_alpha(self, alpha_loss):
        '''Custom method to train the alpha variable'''
        self.alpha_lr_scheduler.step(epoch=self.body.env.clock.frame)
        self.alpha_optim.zero_grad()
        alpha_loss.backward()
        self.alpha_optim.step()
        self.alpha = self.log_alpha.detach().exp()

    def train(self):
        '''Train actor critic by computing the loss in batch efficiently'''
        if util.in_eval_lab_modes():
            return np.nan
        clock = self.body.env.clock
        if self.to_train == 1:
            for _ in range(self.training_iter):
                batch = self.sample()
                clock.set_batch_size(len(batch))

                states = batch['states']
                actions = self.guard_q_actions(batch['actions'])
                q_targets = self.calc_q_targets(batch)
                # Q-value loss for both Q nets
                q1_preds = self.calc_q(states, actions, self.q1_net)
                q1_loss = self.calc_reg_loss(q1_preds, q_targets)
                self.q1_net.train_step(q1_loss, self.q1_optim, self.q1_lr_scheduler, clock=clock, global_net=self.global_q1_net)

                q2_preds = self.calc_q(states, actions, self.q2_net)
                q2_loss = self.calc_reg_loss(q2_preds, q_targets)
                self.q2_net.train_step(q2_loss, self.q2_optim, self.q2_lr_scheduler, clock=clock, global_net=self.global_q2_net)

                # policy loss
                action_pd = policy_util.init_action_pd(self.body.ActionPD, self.calc_pdparam(states))
                log_probs, reparam_actions = self.calc_log_prob_action(action_pd, reparam=True)
                policy_loss = self.calc_policy_loss(batch, log_probs, reparam_actions)
                self.net.train_step(policy_loss, self.optim, self.lr_scheduler, clock=clock, global_net=self.global_net)

                # alpha loss
                alpha_loss = self.calc_alpha_loss(log_probs)
                self.train_alpha(alpha_loss)

                loss = q1_loss + q2_loss + policy_loss + alpha_loss
                # update target networks
                self.update_nets()
                # update PER priorities if availalbe
                self.try_update_per(torch.min(q1_preds, q2_preds), q_targets)

            # reset
            self.to_train = 0
            logger.debug(f'Trained {self.name} at epi: {clock.epi}, frame: {clock.frame}, t: {clock.t}, total_reward so far: {self.body.env.total_reward}, loss: {loss:g}')
            return loss.item()
        else:
            return np.nan

    def update_nets(self):
        '''Update target networks'''
        if util.frame_mod(self.body.env.clock.frame, self.q1_net.update_frequency, self.body.env.num_envs):
            if self.q1_net.update_type == 'replace':
                net_util.copy(self.q1_net, self.target_q1_net)
                net_util.copy(self.q2_net, self.target_q2_net)
            elif self.q1_net.update_type == 'polyak':
                net_util.polyak_update(self.q1_net, self.target_q1_net, self.q1_net.polyak_coef)
                net_util.polyak_update(self.q2_net, self.target_q2_net, self.q2_net.polyak_coef)
            else:
                raise ValueError('Unknown q1_net.update_type. Should be "replace" or "polyak". Exiting.')

    @lab_api
    def update(self):
        '''Override parent method to do nothing'''
        return self.body.explore_var
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
   smartformat.ext.korean
   ~~~~~~~~~~~~~~~~~~~~~~

   A SmartFormat extension for Korean.

   :copyright: (c) 2016 by What! Studio
   :license: BSD, see LICENSE for more details.

"""
import re

from smartformat import extension

from .hangul import is_hangul
from .particles import CombinableParticle, Euro, Ida, Particle

__all__ = ['ko']

#: Allomorphic Korean particles with the general rule.
PARTICLES = [
    # Simple allomorphic rule.
    Particle(u'은', u'는'),
    Particle(u'이', u'가'),
    Particle(u'을', u'를'),
    # Vocative particles.
    Particle(u'아', u'야'),
    Particle(u'이여', u'여'),
    Particle(u'이시여', u'시여'),
    CombinableParticle(u'과', u'와'),
    Euro,
]

#: Matches to particles which have no allomorphic rules.
INVARIANT_PARTICLE_PATTERN = re.compile(
    u'^([의도]$|만|보다|부터|까지|마저|마냥|조차|처럼|에|께|하고)'
)

patterns = []
_particles = {}
for x, p in enumerate(PARTICLES):
    group = u'_%d' % x
    _particles[group] = p
    patterns.append(u'(?P<%s>%s)' % (group, p.regex_pattern()))
_particle_pattern = re.compile(u'|'.join(patterns))

@extension(['ko', ''])
def ko(formatter, value, name, option, format):
    """Chooses different allomorphic forms for Korean particles.

    Implicit Spec: `{:[-]particle}`
    Explicit Spec: `{:ko(particle):item}`

    Example::

       >>> smart.format(u'{pokemon:은} {skill:을} 시전했다!',
       ...              pokemon=u'피카츄', skill=u'전광석화')
       피카츄는 전광석화를 시전했다!
       >>> smart.format(u'{subj:는} {obj:다}.',
       ...              subj=u'대한민국', obj=u'민주공화국')
       대한민국은 민주공화국이다.

    """
    if not name:
        if format.startswith(u'-'):
            __, __, option = format.partition(u'-')
            format = u''
        else:
            option, format = format, u'{}'
        if not option or not all(is_hangul(l) or l in u'()' for l in option):
            # All option letters have to be Hangul
            # to use this extension implicitly.
            return
    if INVARIANT_PARTICLE_PATTERN.match(option):
        # Invariant particle given.
        suffix = option
    else:
        m = _particle_pattern.match(option)
        if m is None:
            particle = Ida
        else:
            particle = _particles[m.lastgroup]
        suffix = particle[value:option]
    return formatter.format(format, value) + suffix
<commit_msg>Add more invariant particles<commit_after># -*- coding: utf-8 -*-
"""
   smartformat.ext.korean
   ~~~~~~~~~~~~~~~~~~~~~~

   A SmartFormat extension for Korean.

   :copyright: (c) 2016 by What! Studio
   :license: BSD, see LICENSE for more details.

"""
import re

from smartformat import extension

from .hangul import is_hangul
from .particles import CombinableParticle, Euro, Ida, Particle

__all__ = ['ko']

#: Allomorphic Korean particles with the general rule.
PARTICLES = [
    # Simple allomorphic rule.
    Particle(u'은', u'는'),
    Particle(u'이', u'가'),
    Particle(u'을', u'를'),
    # Vocative particles.
    Particle(u'아', u'야'),
    Particle(u'이여', u'여'),
    Particle(u'이시여', u'시여'),
    CombinableParticle(u'과', u'와'),
    Euro,
]

#: Matches to particles which have no allomorphic rules.
INVARIANT_PARTICLE_PATTERN = re.compile(
    u'^([의도]$|만|보다|밖에|같이|부터|까지'
    u'|마저|마냥|조차|처럼|에|께|하고|뿐|대로)'
)

patterns = []
_particles = {}
for x, p in enumerate(PARTICLES):
    group = u'_%d' % x
    _particles[group] = p
    patterns.append(u'(?P<%s>%s)' % (group, p.regex_pattern()))
_particle_pattern = re.compile(u'|'.join(patterns))

@extension(['ko', ''])
def ko(formatter, value, name, option, format):
    """Chooses different allomorphic forms for Korean particles.

    Implicit Spec: `{:[-]particle}`
    Explicit Spec: `{:ko(particle):item}`

    Example::

       >>> smart.format(u'{pokemon:은} {skill:을} 시전했다!',
       ...              pokemon=u'피카츄', skill=u'전광석화')
       피카츄는 전광석화를 시전했다!
       >>> smart.format(u'{subj:는} {obj:다}.',
       ...              subj=u'대한민국', obj=u'민주공화국')
       대한민국은 민주공화국이다.

    """
    if not name:
        if format.startswith(u'-'):
            __, __, option = format.partition(u'-')
            format = u''
        else:
            option, format = format, u'{}'
        if not option or not all(is_hangul(l) or l in u'()' for l in option):
            # All option letters have to be Hangul
            # to use this extension implicitly.
            return
    if INVARIANT_PARTICLE_PATTERN.match(option):
        # Invariant particle given.
        suffix = option
    else:
        m = _particle_pattern.match(option)
        if m is None:
            particle = Ida
        else:
            particle = _particles[m.lastgroup]
        suffix = particle[value:option]
    return formatter.format(format, value) + suffix
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2014 Toyota Industrial Equipment SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    "name": "Followers",
    "version": "1.0",
    "author": "Smile",
    "website": 'http://www.smile.fr',
    "category": "Tools",
    "description": """
Manage models following
------------------------------------

TODO: add code comments
TODO: describe usage

Code sample

from openerp.addons.smile_followers.tools import AddFollowers, add_followers

@AddFollowers()
class StockMove(models.Model):
    _name = 'stock.move'
    _inherit = ['stock.move', 'mail.thread']

    @add_followers()
    @api.model
    def create(self, vals):
        return super(StockMove, self).create(vals)

    @add_followers()
    @api.multi
    def write(self, vals):
        return super(StockMove, self).write(vals)

Suggestions & Feedback to: isabelle.richard@smile.fr
""",
    "depends": [
        'base',
    ],
    "data": [
        'views/res_partner_view.xml',
    ],
    "demo": [],
    "installable": True,
    "active": False,
}
<commit_msg>[IMP] Update smile_followers description<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2014 Toyota Industrial Equipment SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    "name": "Followers",
    "version": "1.0",
    "author": "Smile",
    "website": 'http://www.smile.fr',
    "category": "Tools",
    "description": """
Manage models following
------------------------------------

* Disable auto-subscribe for users
** activation from user preferences

TODO: add code comments
TODO: describe usage

Code sample

from openerp.addons.smile_followers.tools import AddFollowers, add_followers

@AddFollowers()
class StockMove(models.Model):
    _name = 'stock.move'
    _inherit = ['stock.move', 'mail.thread']

    @add_followers()
    @api.model
    def create(self, vals):
        return super(StockMove, self).create(vals)

    @add_followers()
    @api.multi
    def write(self, vals):
        return super(StockMove, self).write(vals)

Suggestions & Feedback to: isabelle.richard@smile.fr
""",
    "depends": [
        'base',
        'mail',
    ],
    "data": [
        'views/res_partner_view.xml',
        'views/res_users_view.xml',
    ],
    "demo": [],
    "installable": True,
    "active": False,
}
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.http import HttpResponse
from django.views.generic import RedirectView, TemplateView, View

import forecastio
import requests
from twilio import twiml

from .icons import icons, phases
from .mixins import CsrfExemptMixin

class SmsView(CsrfExemptMixin, View):

    def get_moon(self, percentage):
        percentage *= 100
        if 0 <= percentage < 6.25 or 93.75 <= percentage <= 100:
            return phases['new-moon']
        elif 6.25 <= percentage < 18.75:
            return phases['waxing-crescent']
        elif 18.75 <= percentage < 31.25:
            return phases['first-quarter']
        elif 31.25 <= percentage < 43.75:
            return phases['waxing-gibbous']
        elif 43.75 <= percentage < 56.25:
            return phases['full-moon']
        elif 56.25 <= percentage < 68.75:
            return phases['waning-gibbous']
        elif 68.75 <= percentage < 81.25:
            return phases['last-quarter']
        elif 81.25 <= percentage < 93.75:
            return phases['waning-crescent']
        else:
            return phases['unknown']

    def post(self, request, *args, **kwargs):
        body = request.POST.get('Body', None)
        response = twiml.Response()
        try:

            # Get geocoded location
            r = requests.get('https://maps.googleapis.com/maps/api/geocode/json', params={'address': body})
            json = r.json()
            formatted_address = json['results'][0]['formatted_address']
            location = json['results'][0]['geometry']['location']
            latitude = location['lat']
            longitude = location['lng']

            # Get weather forecast
            forecast = forecastio.load_forecast(settings.DARKSKY_API_KEY, latitude, longitude)
            currently = forecast.currently()
            daily = forecast.daily()
            weather = {
                'icon': icons.get(currently.icon, ''),
                'summary': currently.summary,
                'temperature': str(int(currently.temperature)),
                'moon': self.get_moon(daily.data[0].moonPhase),
            }

            # Create Twilio response
            response.message('%s %s and %s°. %s %s. %s' % (
                weather.get('icon'),
                weather.get('summary'),
                weather.get('temperature'),
                weather.get('moon').get('icon'),
                weather.get('moon').get('name'),
                formatted_address
            ))

        except Exception as e:
            response.message('We\'re sorry, but an error occurred: %s' % e)
        return HttpResponse(response, content_type='text/xml')

class HomeView(TemplateView):
    template_name = 'home.html'

class FaviconView(RedirectView):
    url = staticfiles_storage.url('img/favicon.ico')
    permanent = True
<commit_msg>Added error handling<commit_after>from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.http import HttpResponse
from django.views.generic import RedirectView, TemplateView, View

import forecastio
import requests
from twilio import twiml

from .icons import icons, phases
from .mixins import CsrfExemptMixin

class SmsView(CsrfExemptMixin, View):

    errors = {
        'zero_results': "We're sorry, but we could not find that address.",
        'over_query_limit': "We're sorry, but the request quota has been reached.",
        'request_denied': "We're sorry, but your request was denied.",
        'invalid_request': "We're sorry, but we could not find that address.",
        'unknown_error': "We're sorry, but an error occurred.",
    }

    def get_moon(self, percentage):
        percentage *= 100
        if 0 <= percentage < 6.25 or 93.75 <= percentage <= 100:
            return phases['new-moon']
        elif 6.25 <= percentage < 18.75:
            return phases['waxing-crescent']
        elif 18.75 <= percentage < 31.25:
            return phases['first-quarter']
        elif 31.25 <= percentage < 43.75:
            return phases['waxing-gibbous']
        elif 43.75 <= percentage < 56.25:
            return phases['full-moon']
        elif 56.25 <= percentage < 68.75:
            return phases['waning-gibbous']
        elif 68.75 <= percentage < 81.25:
            return phases['last-quarter']
        elif 81.25 <= percentage < 93.75:
            return phases['waning-crescent']
        else:
            return phases['unknown']

    def post(self, request, *args, **kwargs):
        body = request.POST.get('Body', None)
        response = twiml.Response()
        try:

            # Get geocoded location
            r = requests.get('https://maps.googleapis.com/maps/api/geocode/json', params={'address': body})
            json = r.json()

            if json['status'] == 'OK':

                # Get coordinates
                formatted_address = json['results'][0]['formatted_address']
                location = json['results'][0]['geometry']['location']
                latitude = location['lat']
                longitude = location['lng']

                # Get weather forecast
                forecast = forecastio.load_forecast(settings.DARKSKY_API_KEY, latitude, longitude)
                currently = forecast.currently()
                daily = forecast.daily()
                weather = {
                    'icon': icons.get(currently.icon, ''),
                    'summary': currently.summary,
                    'temperature': str(int(currently.temperature)),
                    'moon': self.get_moon(daily.data[0].moonPhase),
                }

                # Create Twilio response
                response.message('%s %s and %s°. %s %s. %s' % (
                    weather.get('icon'),
                    weather.get('summary'),
                    weather.get('temperature'),
                    weather.get('moon').get('icon'),
                    weather.get('moon').get('name'),
                    formatted_address
                ))

            elif json['status'] == 'ZERO_RESULTS':
                response.message(self.errors['zero_results'])
            elif json['status'] == 'OVER_QUERY_LIMIT':
                response.message(self.errors['over_query_limit'])
            elif json['status'] == 'REQUEST_DENIED':
                response.message(self.errors['request_denied'])
            elif json['status'] == 'INVALID_REQUEST':
                response.message(self.errors['invalid_request'])
            elif json['status'] == 'UNKNOWN_ERROR':
                response.message(self.errors['unknown_error'])
            else:
                response.message(self.errors['unknown_error'])

        except Exception as e:
            response.message("%s %s" % (self.errors['unknown_error'], e))
        return HttpResponse(response, content_type='text/xml')

class HomeView(TemplateView):
    template_name = 'home.html'

class FaviconView(RedirectView):
    url = staticfiles_storage.url('img/favicon.ico')
    permanent = True
<|endoftext|>


<commit_before>from flask import Flask, render_template, flash
from flask_bootstrap import Bootstrap
from flask_wtf import Form
from wtforms import TextField, SubmitField, validators
from wtforms.validators import Required
from handlers import test_handler

class BasicForm(Form):
    softwarename = TextField('Software Name', description='Software Name.')
    githubproject = TextField('GitHub Project', description='Github Project.')
    submit_button = SubmitField('Submit Form')

def create_app(configfile=None):
    app = Flask(__name__)
    Bootstrap(app)

    app.config['SECRET_KEY'] = 'devkey'

    @app.route('/')
    def index():
        form = BasicForm()
        form.validate_on_submit()
        flash('critical message', 'critical')
        flash('error message', 'error')
        flash('warning message', 'warning')
        flash('info message', 'info')
        flash('debug message', 'debug')
        flash('different message', 'different')
        flash('uncategorized message')
        return render_template('index.html', form=form)

    @app.route('/about')
    def about():
       return render_template('about.html')

    @app.route('/index/test/')
    @app.route('/index/test/<software>')
    def test(software=None, score=-2):
        handler=test_handler.test_handler()
        score=handler.get_score(software)
        description=handler.get_description()
        return render_template('test.html', software=software, score=score, description=description)

    return app

if __name__ == '__main__':
    create_app().run(debug=True)
<commit_msg>Added SlideShare handler<commit_after>from flask import Flask, render_template, flash
from flask_bootstrap import Bootstrap
from flask_wtf import Form
from wtforms import TextField, SubmitField, validators
from wtforms.validators import Required
from handlers import test_handler, slideshare_handler

class BasicForm(Form):
    softwarename = TextField('Software Name', description='Software Name.')
    githubproject = TextField('GitHub Project', description='Github Project.')
    submit_button = SubmitField('Submit Form')

def create_app(configfile=None):
    app = Flask(__name__)
    Bootstrap(app)

    app.config['SECRET_KEY'] = 'devkey'

    @app.route('/')
    def index():
        form = BasicForm()
        form.validate_on_submit()
        flash('critical message', 'critical')
        flash('error message', 'error')
        flash('warning message', 'warning')
        flash('info message', 'info')
        flash('debug message', 'debug')
        flash('different message', 'different')
        flash('uncategorized message')
        return render_template('index.html', form=form)

    @app.route('/about')
    def about():
       return render_template('about.html')

    @app.route('/index/test/')
    @app.route('/index/test/<software>')
    def test(software=None, score=-2):
        handler=test_handler.test_handler()
        score=handler.get_score(software)
        description=handler.get_description()
        return render_template('test.html', software=software, score=score, description=description)

    @app.route('/index/slideshare/')
    @app.route('/index/slideshare/<software>')
    def slideshare(software=None, score=-2):
        handler=slideshare_handler.slideshare_handler()
        score=handler.get_score(software)
        description=handler.get_description()
        return render_template('test.html', software=software, score=score, description=description)

    return app

if __name__ == '__main__':
    create_app().run(debug=True)
<|endoftext|>


<commit_before>from .view import MatplotView
from matplotlib import pyplot as plt
from pylab import imshow, contour
from math import sqrt
import numpy as np
import scipy

class UMatrixView(MatplotView):

    def build_u_matrix(self, som, distance=1, row_normalized=False):
        UD2 = som.calculate_map_dist()
        Umatrix = np.zeros((som.codebook.nnodes, 1))
        codebook = som.codebook.matrix
        if row_normalized:
            vector = som._normalizer.normalize_by(codebook.T, codebook.T,
                                                  method='var').T
        else:
            vector = codebook

        for i in range(som.codebook.nnodes):
            codebook_i = vector[i][np.newaxis, :]
            neighborbor_ind = UD2[i][0:] <= distance
            neighborbor_codebooks = vector[neighborbor_ind]
            Umatrix[i] = scipy.spatial.distance_matrix(
                codebook_i, neighborbor_codebooks).mean()

        return Umatrix.reshape(som.codebook.mapsize)

    def show(self, som, distance2=1, row_normalized=False, show_data=True,
             contooor=True, blob=False, labels=False):
        umat = self.build_u_matrix(som, distance=distance2,
                                   row_normalized=row_normalized)
        msz = som.codebook.mapsize
        proj = som.project_data(som.data_raw)
        coord = som.bmu_ind_to_xy(proj)

        fig, ax = plt.subplots(1, 1)
        imshow(umat, cmap=plt.cm.get_cmap('RdYlBu_r'), alpha=1)

        if contooor:
            mn = np.min(umat.flatten())
            mx = np.max(umat.flatten())
            std = np.std(umat.flatten())
            md = np.median(umat.flatten())
            mx = md + 0*std
            contour(umat, np.linspace(mn, mx, 15), linewidths=0.7,
                    cmap=plt.cm.get_cmap('Blues'))

        if show_data:
            plt.scatter(coord[:, 1], coord[:, 0], s=2, alpha=1., c='Gray',
                        marker='o', cmap='jet', linewidths=3, edgecolor='Gray')
            plt.axis('off')

        if labels:
            if labels is True:
                labels = som.build_data_labels()
            for label, x, y in zip(labels, coord[:, 1], coord[:, 0]):
                plt.annotate(str(label), xy=(x, y),
                             horizontalalignment='center',
                             verticalalignment='center')

        ratio = float(msz[0])/(msz[0]+msz[1])
        fig.set_size_inches((1-ratio)*15, ratio*15)
        plt.tight_layout()
        plt.subplots_adjust(hspace=.00, wspace=.000)
        sel_points = list()

        if blob:
            from skimage.color import rgb2gray
            from skimage.feature import blob_log

            image = 1 / umat
            rgb2gray(image)

            # 'Laplacian of Gaussian'
            blobs = blob_log(image, max_sigma=5, num_sigma=4, threshold=.152)
            blobs[:, 2] = blobs[:, 2] * sqrt(2)
            imshow(umat, cmap=plt.cm.get_cmap('RdYlBu_r'), alpha=1)
            sel_points = list()

            for blob in blobs:
                row, col, r = blob
                c = plt.Circle((col, row), r, color='red', linewidth=2,
                               fill=False)
                ax.add_patch(c)
                dist = scipy.spatial.distance_matrix(
                    coord[:, :2], np.array([row, col])[np.newaxis, :])
                sel_point = dist <= r
                plt.plot(coord[:, 1][sel_point[:, 0]],
                         coord[:, 0][sel_point[:, 0]], '.r')
                sel_points.append(sel_point[:, 0])

        plt.show()
        return sel_points, umat
<commit_msg>UMatrixView save figure fix<commit_after>from .view import MatplotView
from matplotlib import pyplot as plt
from pylab import imshow, contour
from math import sqrt
import numpy as np
import scipy

class UMatrixView(MatplotView):

    def build_u_matrix(self, som, distance=1, row_normalized=False):
        UD2 = som.calculate_map_dist()
        Umatrix = np.zeros((som.codebook.nnodes, 1))
        codebook = som.codebook.matrix
        if row_normalized:
            vector = som._normalizer.normalize_by(codebook.T, codebook.T,
                                                  method='var').T
        else:
            vector = codebook

        for i in range(som.codebook.nnodes):
            codebook_i = vector[i][np.newaxis, :]
            neighborbor_ind = UD2[i][0:] <= distance
            neighborbor_codebooks = vector[neighborbor_ind]
            Umatrix[i] = scipy.spatial.distance_matrix(
                codebook_i, neighborbor_codebooks).mean()

        return Umatrix.reshape(som.codebook.mapsize)

    def show(self, som, distance2=1, row_normalized=False, show_data=True,
             contooor=True, blob=False, labels=False):
        umat = self.build_u_matrix(som, distance=distance2,
                                   row_normalized=row_normalized)
        msz = som.codebook.mapsize
        proj = som.project_data(som.data_raw)
        coord = som.bmu_ind_to_xy(proj)

        self._fig, ax = plt.subplots(1, 1)
        imshow(umat, cmap=plt.cm.get_cmap('RdYlBu_r'), alpha=1)

        if contooor:
            mn = np.min(umat.flatten())
            mx = np.max(umat.flatten())
            std = np.std(umat.flatten())
            md = np.median(umat.flatten())
            mx = md + 0*std
            contour(umat, np.linspace(mn, mx, 15), linewidths=0.7,
                    cmap=plt.cm.get_cmap('Blues'))

        if show_data:
            plt.scatter(coord[:, 1], coord[:, 0], s=2, alpha=1., c='Gray',
                        marker='o', cmap='jet', linewidths=3, edgecolor='Gray')
            plt.axis('off')

        if labels:
            if labels is True:
                labels = som.build_data_labels()
            for label, x, y in zip(labels, coord[:, 1], coord[:, 0]):
                plt.annotate(str(label), xy=(x, y),
                             horizontalalignment='center',
                             verticalalignment='center')

        ratio = float(msz[0])/(msz[0]+msz[1])
        self._fig.set_size_inches((1-ratio)*15, ratio*15)
        plt.tight_layout()
        plt.subplots_adjust(hspace=.00, wspace=.000)
        sel_points = list()

        if blob:
            from skimage.color import rgb2gray
            from skimage.feature import blob_log

            image = 1 / umat
            rgb2gray(image)

            # 'Laplacian of Gaussian'
            blobs = blob_log(image, max_sigma=5, num_sigma=4, threshold=.152)
            blobs[:, 2] = blobs[:, 2] * sqrt(2)
            imshow(umat, cmap=plt.cm.get_cmap('RdYlBu_r'), alpha=1)
            sel_points = list()

            for blob in blobs:
                row, col, r = blob
                c = plt.Circle((col, row), r, color='red', linewidth=2,
                               fill=False)
                ax.add_patch(c)
                dist = scipy.spatial.distance_matrix(
                    coord[:, :2], np.array([row, col])[np.newaxis, :])
                sel_point = dist <= r
                plt.plot(coord[:, 1][sel_point[:, 0]],
                         coord[:, 0][sel_point[:, 0]], '.r')
                sel_points.append(sel_point[:, 0])

        plt.show()
        return sel_points, umat
<|endoftext|>