<commit_before>import logging
import os
import sqlite3
import numpy as np
import time
from os.path import dirname, abspath
from abc import ABC, abstractmethod

from src import Utils
# Utils.set_keras_backend("tensorflow")

project_root_dir = dirname(dirname(dirname(abspath(__file__))))
log_dir = os.path.join(project_root_dir, 'logs')
db_path = os.path.join(project_root_dir, 'data', 'db.sqlite')
if not os.path.exists(db_path):
    print('no db found at: ' + db_path)
    exit(1)

class BaseLearn(ABC):

    def __init__(self):
        self.db = sqlite3.connect(db_path)
        cursor = self.db.cursor()
        self.logger = Utils.get_unique_file_logger(self, logging.INFO)
        self.numb_all_games = cursor.execute(
            'SELECT COUNT(*) FROM meta').fetchone()[0]
        self.games_table_length = cursor.execute(
            'SELECT COUNT(*) FROM games').fetchone()[0]
        self.invalid_game_ids = cursor.execute(
            'SELECT id FROM meta WHERE all_moves_imported=0').fetchall()
        self.log('''database contains {} games,
            {} are invalid and won\'t be used for training'''.format(
            self.numb_all_games, len(self.invalid_game_ids)))
        self.training_size = self.games_table_length  # override this in your Learn class as desired
        self.data_retrieval_command = '''SELECT games.*
                                          FROM games, meta
                                          WHERE games.id == meta.id
                                          AND meta.all_moves_imported!=0
                                          LIMIT ?'''

    def log(self, msg):
        self.logger.info(msg)
        print(msg)

    # can be used by Learn classes extending this class that require more game-info during training
    def get_sgf(self, game_id):
        return self.db.cursor().execute('SELECT sgf_content FROM meta WHERE id=?', (game_id,)).fetchone()[0]

    @abstractmethod
    def handle_data(self, training_data):
        pass

    @abstractmethod
    def setup_and_compile_model(self):
        pass

    @abstractmethod
    def train(self, model, X, Y):
        pass

    @abstractmethod
    def get_path_to_self(self):
        pass

    @staticmethod
    def get_symmetries(boards, moves, other_data=None):
        """Given array containing boards and moves recreate all symmetries

        Right now it assumes that moves are given as categorical data already.
        Output arrays will be 8 times as long as the input data, to help
        keeping this in line with the other data you want to use for your
        training you can pass an additional array to `other_data` which will
        be returned with the adjusted length. Lines will still match.
        """
        # print(boards.shape)
        boards = boards.reshape((boards.shape[0], 9, 9))

        passes, moves = moves[:, 81], moves[:, :81]
        moves = moves.reshape((moves.shape[0], 9, 9))

        boards_90 = np.rot90(boards, axes=(1, 2))
        moves_90 = np.rot90(moves, axes=(1, 2))
        boards_180 = np.rot90(boards, k=2, axes=(1, 2))
        moves_180 = np.rot90(moves, k=2, axes=(1, 2))
        boards_270 = np.rot90(boards, k=3, axes=(1, 2))
        moves_270 = np.rot90(moves, k=3, axes=(1, 2))
        boards_flipped = np.fliplr(boards)
        moves_flipped = np.fliplr(moves)
        boards_flipped_90 = np.rot90(np.fliplr(boards), axes=(1, 2))
        moves_flipped_90 = np.rot90(np.fliplr(moves), axes=(1, 2))
        boards_flipped_180 = np.rot90(np.fliplr(boards), k=2, axes=(1, 2))
        moves_flipped_180 = np.rot90(np.fliplr(moves), k=2, axes=(1, 2))
        boards_flipped_270 = np.rot90(np.fliplr(boards), k=3, axes=(1, 2))
        moves_flipped_270 = np.rot90(np.fliplr(moves), k=3, axes=(1, 2))

        boards = np.concatenate((
            boards,
            boards_90,
            boards_180,
            boards_270,
            boards_flipped,
            boards_flipped_90,
            boards_flipped_180,
            boards_flipped_270))
        boards = boards.reshape((boards.shape[0], 81))

        moves = np.concatenate((
            moves,
            moves_90,
            moves_180,
            moves_270,
            moves_flipped,
            moves_flipped_90,
            moves_flipped_180,
            moves_flipped_270))
        moves = moves.reshape((moves.shape[0], 81))
        passes = np.concatenate(
            (passes, passes, passes, passes, passes, passes, passes, passes))
        print(passes[:, None].shape)
        moves = np.concatenate((moves, passes[:, None]), axis=1)

        print('boards.shape:', boards.shape)
        print('moves.shape:', moves.shape)
        if other_data is not None:
            other_data = np.concatenate(
                (other_data, other_data, other_data, other_data,
                 other_data, other_data, other_data, other_data))
            return boards, moves, other_data
        else:
            return boards, moves

    def run(self):
        start_time = time.time()
        # self.log('starting the training with moves from '
        #          + ('all ' if self.numb_games_to_learn_from == self.numb_all_games else '')
        #          + str(self.numb_games_to_learn_from) + ' games as input ' + self.get_path_to_self())

        # Get data from Database
        cursor = self.db.cursor()
        cursor.execute(self.data_retrieval_command,
                       [self.training_size])
        training_data = np.array(cursor.fetchall())  # this is a gigantic array, has millions of rows

        self.log('working with {} rows'.format(len(training_data)))
        X, y = self.handle_data(training_data)

        # SET UP AND STORE NETWORK TOPOLOGY
        model = self.setup_and_compile_model()
        architecture_path = os.path.join(dirname(self.get_path_to_self()), 'model_architecture.json')
        json_file = open(architecture_path, 'w')
        json_file.write(model.to_json())
        json_file.close()

        # TRAIN AND STORE WEIGHTS
        self.train(model, X, y)
        weights_path = os.path.join(dirname(self.get_path_to_self()), 'model_weights.h5')
        model.save_weights(weights_path)

        # EVALUATE
        scores = model.evaluate(X, y)
        print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))

        # DONE
        elapsed_time = time.time() - start_time
        self.log('training ended after {:.0f}s'.format(elapsed_time))
        self.log('model trained on {} moves from {} games'.format(
                 len(X), self.numb_all_games))
        self.log('model architecture saved to: ' + architecture_path)
        self.log('model weights saved to: ' + weights_path)
<commit_msg>No need to evaluate the model if we don't split into training and test<commit_after>import logging
import os
import sqlite3
import numpy as np
import time
from os.path import dirname, abspath
from abc import ABC, abstractmethod

from src import Utils
Utils.set_keras_backend("tensorflow")

project_root_dir = dirname(dirname(dirname(abspath(__file__))))
log_dir = os.path.join(project_root_dir, 'logs')
db_path = os.path.join(project_root_dir, 'data', 'half_db.sqlite')
if not os.path.exists(db_path):
    print('no db found at: ' + db_path)
    exit(1)

class BaseLearn(ABC):

    def __init__(self):
        self.db = sqlite3.connect(db_path)
        cursor = self.db.cursor()
        self.logger = Utils.get_unique_file_logger(self, logging.INFO)
        self.numb_all_games = cursor.execute(
            'SELECT COUNT(*) FROM meta').fetchone()[0]
        self.games_table_length = cursor.execute(
            'SELECT COUNT(*) FROM games').fetchone()[0]
        self.invalid_game_ids = cursor.execute(
            'SELECT id FROM meta WHERE all_moves_imported=0').fetchall()
        self.log('''database contains {} games,
            {} are invalid and won\'t be used for training'''.format(
            self.numb_all_games, len(self.invalid_game_ids)))
        self.training_size = self.games_table_length  # override this in your Learn class as desired
        self.data_retrieval_command = '''SELECT games.*
                                          FROM games, meta
                                          WHERE games.id == meta.id
                                          AND meta.all_moves_imported!=0
                                          LIMIT ?'''

    def log(self, msg):
        self.logger.info(msg)
        print(msg)

    # can be used by Learn classes extending this class that require more game-info during training
    def get_sgf(self, game_id):
        return self.db.cursor().execute('SELECT sgf_content FROM meta WHERE id=?', (game_id,)).fetchone()[0]

    @abstractmethod
    def handle_data(self, training_data):
        pass

    @abstractmethod
    def setup_and_compile_model(self):
        pass

    @abstractmethod
    def train(self, model, X, Y):
        pass

    @abstractmethod
    def get_path_to_self(self):
        pass

    @staticmethod
    def get_symmetries(boards, moves, other_data=None):
        """Given array containing boards and moves recreate all symmetries

        Right now it assumes that moves are given as categorical data already.
        Output arrays will be 8 times as long as the input data, to help
        keeping this in line with the other data you want to use for your
        training you can pass an additional array to `other_data` which will
        be returned with the adjusted length. Lines will still match.
        """
        # print(boards.shape)
        boards = boards.reshape((boards.shape[0], 9, 9))

        passes, moves = moves[:, 81], moves[:, :81]
        moves = moves.reshape((moves.shape[0], 9, 9))

        boards_90 = np.rot90(boards, axes=(1, 2))
        moves_90 = np.rot90(moves, axes=(1, 2))
        boards_180 = np.rot90(boards, k=2, axes=(1, 2))
        moves_180 = np.rot90(moves, k=2, axes=(1, 2))
        boards_270 = np.rot90(boards, k=3, axes=(1, 2))
        moves_270 = np.rot90(moves, k=3, axes=(1, 2))
        boards_flipped = np.fliplr(boards)
        moves_flipped = np.fliplr(moves)
        boards_flipped_90 = np.rot90(np.fliplr(boards), axes=(1, 2))
        moves_flipped_90 = np.rot90(np.fliplr(moves), axes=(1, 2))
        boards_flipped_180 = np.rot90(np.fliplr(boards), k=2, axes=(1, 2))
        moves_flipped_180 = np.rot90(np.fliplr(moves), k=2, axes=(1, 2))
        boards_flipped_270 = np.rot90(np.fliplr(boards), k=3, axes=(1, 2))
        moves_flipped_270 = np.rot90(np.fliplr(moves), k=3, axes=(1, 2))

        boards = np.concatenate((
            boards,
            boards_90,
            boards_180,
            boards_270,
            boards_flipped,
            boards_flipped_90,
            boards_flipped_180,
            boards_flipped_270))
        boards = boards.reshape((boards.shape[0], 81))

        moves = np.concatenate((
            moves,
            moves_90,
            moves_180,
            moves_270,
            moves_flipped,
            moves_flipped_90,
            moves_flipped_180,
            moves_flipped_270))
        moves = moves.reshape((moves.shape[0], 81))
        passes = np.concatenate(
            (passes, passes, passes, passes, passes, passes, passes, passes))
        moves = np.concatenate((moves, passes[:, None]), axis=1)

        # print('boards.shape:', boards.shape)
        # print('moves.shape:', moves.shape)
        if other_data is not None:
            other_data = np.concatenate(
                (other_data, other_data, other_data, other_data,
                 other_data, other_data, other_data, other_data))
            return boards, moves, other_data
        else:
            return boards, moves

    def run(self):
        start_time = time.time()
        # self.log('starting the training with moves from '
        #          + ('all ' if self.numb_games_to_learn_from == self.numb_all_games else '')
        #          + str(self.numb_games_to_learn_from) + ' games as input ' + self.get_path_to_self())

        # Get data from Database
        cursor = self.db.cursor()
        cursor.execute(self.data_retrieval_command,
                       [self.training_size])
        training_data = np.array(cursor.fetchall())  # this is a gigantic array, has millions of rows

        self.log('working with {} rows'.format(len(training_data)))
        X, y = self.handle_data(training_data)

        # Save input and output dimensions for easier, more modular use
        # Implicit assumtion is that X, y are two-dimensional
        self.input_dim = X.shape[1]
        self.output_dim = y.shape[1]

        # SET UP AND STORE NETWORK TOPOLOGY
        model = self.setup_and_compile_model()
        architecture_path = os.path.join(dirname(self.get_path_to_self()), 'model_architecture.json')
        json_file = open(architecture_path, 'w')
        json_file.write(model.to_json())
        json_file.close()

        # TRAIN AND STORE WEIGHTS
        self.train(model, X, y)
        weights_path = os.path.join(dirname(self.get_path_to_self()), 'model_weights.h5')
        model.save_weights(weights_path)

        # EVALUATE
        # scores = model.evaluate(X, y)
        # print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))

        # DONE
        elapsed_time = time.time() - start_time
        self.log('training ended after {:.0f}s'.format(elapsed_time))
        self.log('model trained on {} moves from {} games'.format(
                 len(X), self.numb_all_games))
        self.log('model architecture saved to: ' + architecture_path)
        self.log('model weights saved to: ' + weights_path)
<|endoftext|>


<commit_before><commit_msg>Removing commented code and order by from query.<commit_after><|endoftext|>


<commit_before>from django.contrib import admin
from .models import Tag

class TagAdmin(admin.ModelAdmin):
	search_fields = ['tag']
    model = Tag

admin.site.register(Tag, TagAdmin)
<commit_msg>indentation<commit_after>from django.contrib import admin
from .models import Tag

class TagAdmin(admin.ModelAdmin):
	search_fields = ['tag']
	model = Tag

admin.site.register(Tag, TagAdmin)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import simple_audit
import logging
from sets import Set
from django.db import models
from django.contrib.auth.models import User, Group
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from django.db.models.signals import pre_save, post_save, pre_delete, m2m_changed
from django.dispatch import receiver
from django_extensions.db.fields.encrypted import EncryptedCharField

from util.models import BaseModel
from util.notifications import notify_new_user_creation
from .helper import find_ldap_groups_from_user

LOG = logging.getLogger(__name__)

class AccountUser(User):
    class Meta:
        proxy = True
        verbose_name_plural = _("users")
        verbose_name = _("user")

class Role(Group):
    class Meta:
        proxy = True

class TeamUsersManager(models.Manager):
    """manager for returning """
    def get_query_set(self):
        return User.objects.filter(id__in=[user.id for user in Team.users_without_team()])

class Team(BaseModel):

    name = models.CharField(_('name'), max_length=80, unique=True)
    role = models.ForeignKey(Role)
    users = models.ManyToManyField(User)
    objects = models.Manager()  # The default manager.
    user_objects = TeamUsersManager()  # The Dahl-specific manager.

    class Meta:
        # putting permissions for account user and role in team model, because it
        # clashes with the proxied classes permissions
        permissions = (
            ("change_accountuser", "Can change account user"),
            ("add_accountuser", "Can add account user"),
            ("delete_accountuser", "Can delete account user"),
            ("change_role", "Can change role"),
            ("add_role", "Can add role"),
            ("delete_role", "Can delete role"),
        )

    def __str__(self):
        return self.name

    def natural_key(self):
        return (self.name,)

    @classmethod
    def get_all_permissions_for(cls, user=None):
        """return all permissions for user"""
        permissions = []
        if not user.is_active:
            return set(permissions)
        else:
            teams = Team.objects.filter(users=user)
            if teams.count() > 1:
                LOG.warning("user %s is in more than one team! %s" % (user, teams))

            for team in teams:
                permissions = permissions + list(team.role.permissions.all())

            return set(["%s.%s" % (p.content_type.app_label, p.codename) for p in permissions])
            #return permissions

    @classmethod
    def users_without_team(cls):
        """get all users without team"""
        users = []
        all_users = Set(User.objects.all())
        teams = Team.objects.all()
        for team in teams:
            for user in team.users.all():
                if user not in users:
                    users.append(user)

        return list(all_users.difference(Set(users)))

def sync_ldap_groups_with_user(user=None):
    """
    Sync ldap groups (aka team) with the user
    """
    LOG.debug("User %s groups before: %s" % (user, user.groups.all()))
    ldap_groups = find_ldap_groups_from_user(username=user.username)
    groups = Group.objects.filter(name__in=ldap_groups).exclude(user__username=user.username).order_by("name")
    LOG.info("LDAP's team created in the system and not set to user %s: %s" % (user, groups))
    group = None
    if groups:
        group = groups[0]
        user.groups.add(group)
        LOG.info("group %s added to user %s" % (groups[0], user))

    LOG.debug("User %s groups: %s after" % (user, user.groups.all()))

    return group

simple_audit.register(Team, AccountUser, Role)

#####################################################################################################
# SIGNALS
#####################################################################################################
#all role name should start with role_
def user_post_save_wrapper(kwargs={}):
    user = kwargs.get('instance')
    created = kwargs.get('created')
    if created:
        LOG.debug("new user %s created" % user)
        user.is_active = True
        user.is_staff = True
        user.save()
        #notify new user create
        notify_new_user_creation(user)

@receiver(pre_save, sender=Role)
def role_pre_save(sender, **kwargs):
    role = kwargs.get('instance')
    if not role.name.startswith('role_'):
        role.name = "role_" + role.name

#all users should be is_staff True
@receiver(pre_save, sender=AccountUser)
def user_pre_save(sender, **kwargs):
    user = kwargs.get('instance')
    #LOG.debug("user %s pre save signal" % user)

@receiver(post_save, sender=AccountUser)
def account_user_post_save(sender, **kwargs):
    user_post_save_wrapper(kwargs)
    #sync_ldap_groups_with_user(user=user)

@receiver(post_save, sender=User)
def user_post_save(sender, **kwargs):
    user_post_save_wrapper(kwargs)

# def user_m2m_changed(sender, **kwargs):
#     """
#     Using m2m signal to sync user.groups relation in the db with the ones in the LDAP. The action post_clear
#     must be used in order to accomplish this. (https://docs.djangoproject.com/en/1.5/ref/signals/)
#     Remember, remember: only the groups that exists in ldap AND in the database will be synced.
#     You can however, add a group to user that is not created in ldap.
#     """
#     # LOG.debug("m2m_changed kwargs: %s" % kwargs)
#     user = kwargs.get('instance')
#     action = kwargs.get('action')
#     if action == 'post_clear':
#         LOG.info("user %s m2m_changed post_clear signal" % user)
#         #sync_ldap_groups_with_user(user=user)
#
#
# m2m_changed.connect(user_m2m_changed, sender=User.groups.through)
# m2m_changed.connect(user_m2m_changed, sender=AccountUser.groups.through)
<commit_msg>new field email<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import simple_audit
import logging
from sets import Set
from django.db import models
from django.contrib.auth.models import User, Group
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from django.db.models.signals import pre_save, post_save, pre_delete, m2m_changed
from django.dispatch import receiver
from django_extensions.db.fields.encrypted import EncryptedCharField

from util.models import BaseModel
from util.notifications import notify_new_user_creation
from .helper import find_ldap_groups_from_user

LOG = logging.getLogger(__name__)

class AccountUser(User):
    class Meta:
        proxy = True
        verbose_name_plural = _("users")
        verbose_name = _("user")

class Role(Group):
    class Meta:
        proxy = True

class TeamUsersManager(models.Manager):
    """manager for returning """
    def get_query_set(self):
        return User.objects.filter(id__in=[user.id for user in Team.users_without_team()])

class Team(BaseModel):

    name = models.CharField(_('name'), max_length=80, unique=True)
    role = models.ForeignKey(Role)
    users = models.ManyToManyField(User)
    objects = models.Manager()  # The default manager.
    user_objects = TeamUsersManager()  # The Dahl-specific manager.
    email = models.EmailField(null=True, blank=True)

    class Meta:
        # putting permissions for account user and role in team model, because it
        # clashes with the proxied classes permissions
        permissions = (
            ("change_accountuser", "Can change account user"),
            ("add_accountuser", "Can add account user"),
            ("delete_accountuser", "Can delete account user"),
            ("change_role", "Can change role"),
            ("add_role", "Can add role"),
            ("delete_role", "Can delete role"),
        )

    def __str__(self):
        return self.name

    def natural_key(self):
        return (self.name,)

    @classmethod
    def get_all_permissions_for(cls, user=None):
        """return all permissions for user"""
        permissions = []
        if not user.is_active:
            return set(permissions)
        else:
            teams = Team.objects.filter(users=user)
            if teams.count() > 1:
                LOG.warning("user %s is in more than one team! %s" % (user, teams))

            for team in teams:
                permissions = permissions + list(team.role.permissions.all())

            return set(["%s.%s" % (p.content_type.app_label, p.codename) for p in permissions])
            #return permissions

    @classmethod
    def users_without_team(cls):
        """get all users without team"""
        users = []
        all_users = Set(User.objects.all())
        teams = Team.objects.all()
        for team in teams:
            for user in team.users.all():
                if user not in users:
                    users.append(user)

        return list(all_users.difference(Set(users)))

def sync_ldap_groups_with_user(user=None):
    """
    Sync ldap groups (aka team) with the user
    """
    LOG.debug("User %s groups before: %s" % (user, user.groups.all()))
    ldap_groups = find_ldap_groups_from_user(username=user.username)
    groups = Group.objects.filter(name__in=ldap_groups).exclude(user__username=user.username).order_by("name")
    LOG.info("LDAP's team created in the system and not set to user %s: %s" % (user, groups))
    group = None
    if groups:
        group = groups[0]
        user.groups.add(group)
        LOG.info("group %s added to user %s" % (groups[0], user))

    LOG.debug("User %s groups: %s after" % (user, user.groups.all()))

    return group

simple_audit.register(Team, AccountUser, Role)

#####################################################################################################
# SIGNALS
#####################################################################################################
#all role name should start with role_
def user_post_save_wrapper(kwargs={}):
    user = kwargs.get('instance')
    created = kwargs.get('created')
    if created:
        LOG.debug("new user %s created" % user)
        user.is_active = True
        user.is_staff = True
        user.save()
        #notify new user create
        notify_new_user_creation(user)

@receiver(pre_save, sender=Role)
def role_pre_save(sender, **kwargs):
    role = kwargs.get('instance')
    if not role.name.startswith('role_'):
        role.name = "role_" + role.name

#all users should be is_staff True
@receiver(pre_save, sender=AccountUser)
def user_pre_save(sender, **kwargs):
    user = kwargs.get('instance')
    #LOG.debug("user %s pre save signal" % user)

@receiver(post_save, sender=AccountUser)
def account_user_post_save(sender, **kwargs):
    user_post_save_wrapper(kwargs)
    #sync_ldap_groups_with_user(user=user)

@receiver(post_save, sender=User)
def user_post_save(sender, **kwargs):
    user_post_save_wrapper(kwargs)

# def user_m2m_changed(sender, **kwargs):
#     """
#     Using m2m signal to sync user.groups relation in the db with the ones in the LDAP. The action post_clear
#     must be used in order to accomplish this. (https://docs.djangoproject.com/en/1.5/ref/signals/)
#     Remember, remember: only the groups that exists in ldap AND in the database will be synced.
#     You can however, add a group to user that is not created in ldap.
#     """
#     # LOG.debug("m2m_changed kwargs: %s" % kwargs)
#     user = kwargs.get('instance')
#     action = kwargs.get('action')
#     if action == 'post_clear':
#         LOG.info("user %s m2m_changed post_clear signal" % user)
#         #sync_ldap_groups_with_user(user=user)
#
#
# m2m_changed.connect(user_m2m_changed, sender=User.groups.through)
# m2m_changed.connect(user_m2m_changed, sender=AccountUser.groups.through)
<|endoftext|>


<commit_before>import os
import sys
import threading
import traceback
import zmq

from . import common, configs

def _get_appsock_from_handshake(handshake_endpoint):
    app_sock = zmq.Context().socket(zmq.REP)
    host = 'tcp://127.0.0.1'
    app_port = app_sock.bind_to_random_port(host)
    app_endpoint = '%s:%s' % (host, app_port)

    handshake_sock = zmq.Context().socket(zmq.REQ)
    handshake_sock.connect(handshake_endpoint)
    handshake_sock.send(app_endpoint)
    handshake_sock.recv()
    handshake_sock.close()

    return app_sock

def _get_appsock_from_config(config):
    sock = zmq.Context().socket(zmq.REP)
    sock.bind('tcp://%s:%s' % (config.host, config.port))
    return sock

def start_server():
    configname = os.getenv(common.ENV_CONFIGNAME)
    if not configname:
        sys.exit('%s must be set.' % common.ENV_CONFIGNAME)
    config = configs.config_by_name(configname)

    handshake_endpoint = os.getenv(common.ENV_HANDSHAKE)
    if handshake_endpoint:
        sock = _get_appsock_from_handshake(handshake_endpoint)
    else:
        sock = _get_appsock_from_config(configname)

    while True:
        recved = sock.recv()
        try:
            func, arg = config.loads(recved)
            code = common.SUCCESS
            response = None
            if func == 'exec':
                exec arg in globals(), globals()
            elif func == 'eval':
                response = eval(arg, globals(), globals())
            else:
                code = common.INVALID_METHOD
                response = func
            pickled = config.dumps({
                'code': code,
                'value': response
            })
        except Exception as ex:
            pickled = config.dumps({
                'code': common.UNHANDLED_ERROR,
                'errtype': ex.__class__.__name__,
                'traceback': ''.join(traceback.format_exc())
            })
        sock.send(pickled)

def start_server_thread():
    t = threading.Thread(target=start_server)
    t.start()
<commit_msg>Cleaned up duplicated code.<commit_after>import os
import sys
import threading
import traceback
import zmq

from . import common, configs, utils

def _get_appsock_from_handshake(handshake_endpoint):
    app_info = utils.create_rep_socket_bound_to_random()

    handshake_sock = zmq.Context().socket(zmq.REQ)
    handshake_sock.connect(handshake_endpoint)
    handshake_sock.send(app_info.endpoint)
    handshake_sock.recv()
    handshake_sock.close()

    return app_info.socket

def _get_appsock_from_config(config):
    sock = zmq.Context().socket(zmq.REP)
    sock.bind('tcp://%s:%s' % (config.host, config.port))
    return sock

def start_server():
    configname = os.getenv(common.ENV_CONFIGNAME)
    if not configname:
        sys.exit('%s must be set.' % common.ENV_CONFIGNAME)
    config = configs.config_by_name(configname)

    handshake_endpoint = os.getenv(common.ENV_HANDSHAKE)
    if handshake_endpoint:
        sock = _get_appsock_from_handshake(handshake_endpoint)
    else:
        sock = _get_appsock_from_config(configname)

    while True:
        recved = sock.recv()
        try:
            func, arg = config.loads(recved)
            code = common.SUCCESS
            response = None
            if func == 'exec':
                exec arg in globals(), globals()
            elif func == 'eval':
                response = eval(arg, globals(), globals())
            else:
                code = common.INVALID_METHOD
                response = func
            pickled = config.dumps({
                'code': code,
                'value': response
            })
        except Exception as ex:
            pickled = config.dumps({
                'code': common.UNHANDLED_ERROR,
                'errtype': ex.__class__.__name__,
                'traceback': ''.join(traceback.format_exc())
            })
        sock.send(pickled)

def start_server_thread():
    t = threading.Thread(target=start_server)
    t.start()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2013, Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.

import logging
import threading
import struct
import time
import unittest

from vtdb import keyrange_constants
from vtdb import keyspace

import environment
import utils
import tablet

from zk import zkocc

SHARDED_KEYSPACE = "TEST_KEYSPACE_SHARDED"
UNSHARDED_KEYSPACE = "TEST_KEYSPACE_UNSHARDED"

# shards for SHARDED_KEYSPACE
# range "" - 80
shard_0_master = tablet.Tablet()
shard_0_replica = tablet.Tablet()
# range 80 - ""
shard_1_master = tablet.Tablet()
shard_1_replica = tablet.Tablet()

# shard for UNSHARDED_KEYSPACE
unsharded_master = tablet.Tablet()
unsharded_replica = tablet.Tablet()

shard_names = ['-80', '80-']
shard_kid_map = {'-80': [527875958493693904, 626750931627689502,
                         345387386794260318, 332484755310826578,
                         1842642426274125671, 1326307661227634652,
                         1761124146422844620, 1661669973250483744,
                         3361397649937244239, 2444880764308344533],
                 '80-': [9767889778372766922, 9742070682920810358,
                         10296850775085416642, 9537430901666854108,
                         10440455099304929791, 11454183276974683945,
                         11185910247776122031, 10460396697869122981,
                         13379616110062597001, 12826553979133932576],
                 }

create_vt_insert_test = '''create table vt_insert_test (
id bigint auto_increment,
msg varchar(64),
keyspace_id bigint(20) unsigned NOT NULL,
primary key (id)
) Engine=InnoDB'''

def setUpModule():
  try:
    environment.topo_server().setup()

    setup_procs = [
        shard_0_master.init_mysql(),
        shard_0_replica.init_mysql(),
        shard_1_master.init_mysql(),
        shard_1_replica.init_mysql(),
        unsharded_master.init_mysql(),
        unsharded_replica.init_mysql(),
        ]
    utils.wait_procs(setup_procs)
    setup_tablets()
  except:
    tearDownModule()
    raise

def tearDownModule():
  if utils.options.skip_teardown:
    return

  tablet.kill_tablets([shard_0_master, shard_0_replica,
                      shard_1_master, shard_1_replica])
  teardown_procs = [
      shard_0_master.teardown_mysql(),
      shard_0_replica.teardown_mysql(),
      shard_1_master.teardown_mysql(),
      shard_1_replica.teardown_mysql(),
      unsharded_master.teardown_mysql(),
      unsharded_replica.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica.remove_tree()
  unsharded_master.remove_tree()
  unsharded_replica.remove_tree()

def setup_tablets():
  setup_sharded_keyspace()
  setup_unsharded_keyspace()
  utils.VtGate().start()

def setup_sharded_keyspace():
  utils.run_vtctl(['CreateKeyspace', SHARDED_KEYSPACE])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', SHARDED_KEYSPACE,
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace=SHARDED_KEYSPACE, shard='-80')
  shard_0_replica.init_tablet('replica', keyspace=SHARDED_KEYSPACE, shard='-80')
  shard_1_master.init_tablet('master', keyspace=SHARDED_KEYSPACE, shard='80-')
  shard_1_replica.init_tablet('replica', keyspace=SHARDED_KEYSPACE,  shard='80-')

  utils.run_vtctl(['RebuildKeyspaceGraph', SHARDED_KEYSPACE,], auto_log=True)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.create_db('vt_test_keyspace_sharded')
    t.mquery(shard_0_master.dbname, create_vt_insert_test)
    t.start_vttablet(wait_for_state=None)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['InitShardMaster', '%s/-80' % SHARDED_KEYSPACE,
                   shard_0_master.tablet_alias], auto_log=True)
  utils.run_vtctl(['InitShardMaster', '%s/80-' % SHARDED_KEYSPACE,
                   shard_1_master.tablet_alias], auto_log=True)

  utils.run_vtctl(['RebuildKeyspaceGraph', SHARDED_KEYSPACE],
                   auto_log=True)

  utils.check_srv_keyspace('test_nj', SHARDED_KEYSPACE,
                           'Partitions(master): -80 80-\n' +
                           'Partitions(rdonly): -80 80-\n' +
                           'Partitions(replica): -80 80-\n')

def setup_unsharded_keyspace():
  utils.run_vtctl(['CreateKeyspace', UNSHARDED_KEYSPACE])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', UNSHARDED_KEYSPACE,
                   'keyspace_id', 'uint64'])
  unsharded_master.init_tablet('master', keyspace=UNSHARDED_KEYSPACE, shard='0')
  unsharded_replica.init_tablet('replica', keyspace=UNSHARDED_KEYSPACE, shard='0')

  utils.run_vtctl(['RebuildKeyspaceGraph', UNSHARDED_KEYSPACE,], auto_log=True)

  for t in [unsharded_master, unsharded_replica]:
    t.create_db('vt_test_keyspace_unsharded')
    t.mquery(unsharded_master.dbname, create_vt_insert_test)
    t.start_vttablet(wait_for_state=None)

  for t in [unsharded_master, unsharded_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['InitShardMaster', '%s/0' % UNSHARDED_KEYSPACE,
                   unsharded_master.tablet_alias], auto_log=True)

  utils.run_vtctl(['RebuildKeyspaceGraph', UNSHARDED_KEYSPACE],
                   auto_log=True)

  utils.check_srv_keyspace('test_nj', UNSHARDED_KEYSPACE,
                           'Partitions(master): -\n' +
                           'Partitions(rdonly): -\n' +
                           'Partitions(replica): -\n')

ALL_DB_TYPES = ['master', 'rdonly', 'replica']

class TestKeyspace(unittest.TestCase):
  def _read_keyspace(self, keyspace_name):
    vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(),
                                          "test_nj", 30.0)
    return keyspace.read_keyspace(vtgate_client, keyspace_name)

  def test_get_keyspace(self):
    ki = utils.run_vtctl_json(['GetKeyspace', UNSHARDED_KEYSPACE])
    self.assertEqual('keyspace_id', ki['ShardingColumnName'])
    self.assertEqual('uint64', ki['ShardingColumnType'])

  def test_shard_count(self):
    sharded_ks = self._read_keyspace(SHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(sharded_ks.get_shard_count(db_type), 2)
    unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(unsharded_ks.get_shard_count(db_type), 1)

  def test_shard_names(self):
    sharded_ks = self._read_keyspace(SHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(sharded_ks.get_shard_names(db_type), ['-80', '80-'])
    unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(unsharded_ks.get_shard_names(db_type), ['0'])

  def test_keyspace_id_to_shard_name(self):
    sharded_ks = self._read_keyspace(SHARDED_KEYSPACE)
    for _, sn in enumerate(shard_names):
      for keyspace_id in shard_kid_map[sn]:
        self.assertEqual(sharded_ks.keyspace_id_to_shard_name_for_db_type(keyspace_id, 'master'), sn)
    unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE)
    for keyspace_id in shard_kid_map[sn]:
      self.assertEqual(unsharded_ks.keyspace_id_to_shard_name_for_db_type(keyspace_id, 'master'), '0')

  def test_get_srv_keyspace_names(self):
    stdout, stderr = utils.run_vtctl(['GetSrvKeyspaceNames', 'test_nj'],
                                     trap_output=True)
    self.assertFalse(stderr)
    self.assertEqual(set(stdout.splitlines()), {SHARDED_KEYSPACE, UNSHARDED_KEYSPACE})

if __name__ == '__main__':
  utils.main()
<commit_msg>Deflake keyspace_test.<commit_after>#!/usr/bin/env python
#
# Copyright 2013, Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.

import logging
import threading
import struct
import time
import unittest

from vtdb import keyrange_constants
from vtdb import keyspace

import environment
import utils
import tablet

from zk import zkocc

SHARDED_KEYSPACE = "TEST_KEYSPACE_SHARDED"
UNSHARDED_KEYSPACE = "TEST_KEYSPACE_UNSHARDED"

# shards for SHARDED_KEYSPACE
# range "" - 80
shard_0_master = tablet.Tablet()
shard_0_replica = tablet.Tablet()
# range 80 - ""
shard_1_master = tablet.Tablet()
shard_1_replica = tablet.Tablet()

# shard for UNSHARDED_KEYSPACE
unsharded_master = tablet.Tablet()
unsharded_replica = tablet.Tablet()

shard_names = ['-80', '80-']
shard_kid_map = {'-80': [527875958493693904, 626750931627689502,
                         345387386794260318, 332484755310826578,
                         1842642426274125671, 1326307661227634652,
                         1761124146422844620, 1661669973250483744,
                         3361397649937244239, 2444880764308344533],
                 '80-': [9767889778372766922, 9742070682920810358,
                         10296850775085416642, 9537430901666854108,
                         10440455099304929791, 11454183276974683945,
                         11185910247776122031, 10460396697869122981,
                         13379616110062597001, 12826553979133932576],
                 }

create_vt_insert_test = '''create table vt_insert_test (
id bigint auto_increment,
msg varchar(64),
keyspace_id bigint(20) unsigned NOT NULL,
primary key (id)
) Engine=InnoDB'''

def setUpModule():
  try:
    environment.topo_server().setup()

    setup_procs = [
        shard_0_master.init_mysql(),
        shard_0_replica.init_mysql(),
        shard_1_master.init_mysql(),
        shard_1_replica.init_mysql(),
        unsharded_master.init_mysql(),
        unsharded_replica.init_mysql(),
        ]
    utils.wait_procs(setup_procs)
    setup_tablets()
  except:
    tearDownModule()
    raise

def tearDownModule():
  if utils.options.skip_teardown:
    return

  tablet.kill_tablets([shard_0_master, shard_0_replica,
                      shard_1_master, shard_1_replica])
  teardown_procs = [
      shard_0_master.teardown_mysql(),
      shard_0_replica.teardown_mysql(),
      shard_1_master.teardown_mysql(),
      shard_1_replica.teardown_mysql(),
      unsharded_master.teardown_mysql(),
      unsharded_replica.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica.remove_tree()
  unsharded_master.remove_tree()
  unsharded_replica.remove_tree()

def setup_tablets():
  setup_sharded_keyspace()
  setup_unsharded_keyspace()
  utils.VtGate().start()

def setup_sharded_keyspace():
  utils.run_vtctl(['CreateKeyspace', SHARDED_KEYSPACE])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', SHARDED_KEYSPACE,
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace=SHARDED_KEYSPACE, shard='-80')
  shard_0_replica.init_tablet('replica', keyspace=SHARDED_KEYSPACE, shard='-80')
  shard_1_master.init_tablet('master', keyspace=SHARDED_KEYSPACE, shard='80-')
  shard_1_replica.init_tablet('replica', keyspace=SHARDED_KEYSPACE,  shard='80-')

  utils.run_vtctl(['RebuildKeyspaceGraph', SHARDED_KEYSPACE,], auto_log=True)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.create_db('vt_test_keyspace_sharded')
    t.mquery(shard_0_master.dbname, create_vt_insert_test)
    t.start_vttablet(wait_for_state=None)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['InitShardMaster', '%s/-80' % SHARDED_KEYSPACE,
                   shard_0_master.tablet_alias], auto_log=True)
  utils.run_vtctl(['InitShardMaster', '%s/80-' % SHARDED_KEYSPACE,
                   shard_1_master.tablet_alias], auto_log=True)

  utils.run_vtctl(['RebuildKeyspaceGraph', SHARDED_KEYSPACE],
                   auto_log=True)

  utils.check_srv_keyspace('test_nj', SHARDED_KEYSPACE,
                           'Partitions(master): -80 80-\n' +
                           'Partitions(rdonly): -80 80-\n' +
                           'Partitions(replica): -80 80-\n')

def setup_unsharded_keyspace():
  utils.run_vtctl(['CreateKeyspace', UNSHARDED_KEYSPACE])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', UNSHARDED_KEYSPACE,
                   'keyspace_id', 'uint64'])
  unsharded_master.init_tablet('master', keyspace=UNSHARDED_KEYSPACE, shard='0')
  unsharded_replica.init_tablet('replica', keyspace=UNSHARDED_KEYSPACE, shard='0')

  utils.run_vtctl(['RebuildKeyspaceGraph', UNSHARDED_KEYSPACE,], auto_log=True)

  for t in [unsharded_master, unsharded_replica]:
    t.create_db('vt_test_keyspace_unsharded')
    t.mquery(unsharded_master.dbname, create_vt_insert_test)
    t.start_vttablet(wait_for_state=None)

  for t in [unsharded_master, unsharded_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['InitShardMaster', '%s/0' % UNSHARDED_KEYSPACE,
                   unsharded_master.tablet_alias], auto_log=True)

  utils.run_vtctl(['RebuildKeyspaceGraph', UNSHARDED_KEYSPACE],
                   auto_log=True)

  utils.check_srv_keyspace('test_nj', UNSHARDED_KEYSPACE,
                           'Partitions(master): -\n' +
                           'Partitions(rdonly): -\n' +
                           'Partitions(replica): -\n')

ALL_DB_TYPES = ['master', 'rdonly', 'replica']

class TestKeyspace(unittest.TestCase):
  def _read_keyspace(self, keyspace_name):
    vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(),
                                          "test_nj", 30.0)
    return keyspace.read_keyspace(vtgate_client, keyspace_name)

  def test_get_keyspace(self):
    ki = utils.run_vtctl_json(['GetKeyspace', UNSHARDED_KEYSPACE])
    self.assertEqual('keyspace_id', ki['ShardingColumnName'])
    self.assertEqual('uint64', ki['ShardingColumnType'])

  def test_shard_count(self):
    sharded_ks = self._read_keyspace(SHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(sharded_ks.get_shard_count(db_type), 2)
    unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(unsharded_ks.get_shard_count(db_type), 1)

  def test_shard_names(self):
    sharded_ks = self._read_keyspace(SHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(sharded_ks.get_shard_names(db_type), ['-80', '80-'])
    unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE)
    for db_type in ALL_DB_TYPES:
      self.assertEqual(unsharded_ks.get_shard_names(db_type), ['0'])

  def test_keyspace_id_to_shard_name(self):
    sharded_ks = self._read_keyspace(SHARDED_KEYSPACE)
    for _, sn in enumerate(shard_names):
      for keyspace_id in shard_kid_map[sn]:
        self.assertEqual(sharded_ks.keyspace_id_to_shard_name_for_db_type(keyspace_id, 'master'), sn)
    unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE)
    for keyspace_id in shard_kid_map[sn]:
      self.assertEqual(unsharded_ks.keyspace_id_to_shard_name_for_db_type(keyspace_id, 'master'), '0')

  def test_get_srv_keyspace_names(self):
    stdout, _ = utils.run_vtctl(['GetSrvKeyspaceNames', 'test_nj'],
                                trap_output=True)
    self.assertEqual(set(stdout.splitlines()), {SHARDED_KEYSPACE, UNSHARDED_KEYSPACE})

if __name__ == '__main__':
  utils.main()
<|endoftext|>


<commit_before><commit_msg>init file was blank and so not uploading<commit_after>__author__ = 'dgaston'
<|endoftext|>


<commit_before>import theano
from theano import tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
import numpy as np
from load import seq_load
from theano.tensor.nnet.conv import conv2d
from theano.tensor.signal.downsample import max_pool_2d
from theano.tensor.signal.downsample import DownsampleFactorMax

theano.config.floatX='float32'

srng = RandomStreams()

def floatX(X):
    return np.asarray(X, dtype=theano.config.floatX)

def init_weights(shape):
    return theano.shared(floatX(np.random.randn(*shape) * 0.01))

def rectify(X):
    return T.maximum(X, 0.)

def softmax(X):
    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))
    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')

def dropout(X, p=0.):
    if p > 0:
        retain_prob = 1 - p
        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)
        X /= retain_prob
    return X

def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):
    grads = T.grad(cost=cost, wrt=params)
    updates = []
    for p, g in zip(params, grads):
        acc = theano.shared(p.get_value() * 0.)
        acc_new = rho * acc + (1 - rho) * g ** 2
        gradient_scaling = T.sqrt(acc_new + epsilon)
        g = g / gradient_scaling
        updates.append((acc, acc_new))
        updates.append((p, p - lr * g))
    return updates

# convolution: filters are moved by one position at a time, see parameter subsample=(1, 1)
#
# max pooling:
#   scaling the input before applying the maxpool filter and
#   displacement (stride) when sliding the max pool filters
stride1=2
downscale1=3

stride2=2
downscale2=2

stride3=2
downscale3=1

def model(X, w, w2, w3, w4, p_drop_conv, p_drop_hidden):
    # TODO spremeni max_pool argumente
    l1a = rectify(conv2d(X, w, border_mode='valid', subsample=(1, 1))) # stride along one (horizontal) dimension only
    l1 = max_pool_2d(l1a, (1, downscale1), st=(1, stride1)) # (1,1)=(vertical, horizontal) downscale, st=(1, step): move to every stride1 column and perform max_pooling there
    l1 = dropout(l1, p_drop_conv)

    l2a = rectify(conv2d(l1, w2, subsample=(1, 1))) # stride along horizontal
    # l2 = max_pool_2d(l2a, (2, 2))
    l2 = max_pool_2d(l2a, (1, downscale2), st=(1, stride2))
    l2 = dropout(l2, p_drop_conv)

    l3a = rectify(conv2d(l2, w3, subsample=(1, 1))) # stride along horizontal
    # l3b = max_pool_2d(l3a, (2, 2))
    l3b = max_pool_2d(l3a, (1, downscale3), st=(1, stride3))
    l3 = T.flatten(l3b, outdim=2)
    l3 = dropout(l3, p_drop_conv)

    l4 = rectify(T.dot(l3, w4))
    l4 = dropout(l4, p_drop_hidden)

    pyx = softmax(T.dot(l4, w_o))
    return l1, l2, l3, l4, pyx

num_of_classes = 104
trX, teX, trY, teY = seq_load(number_of_classes=num_of_classes, onehot=True)

print(trX.shape)
trX = trX.reshape(-1, 1, 1, 100)  # TODO spremeni argumente tukaj
teX = teX.reshape(-1, 1, 1, 100)

X = T.ftensor4()
Y = T.fmatrix()

# size of convolution windows, for each layer different values can be used
cwin1=6
cwin2=5
cwin3=3
num_filters_1=32 # how many different filters to lear at each layer
num_filters_2=48
num_filters_3=64
w = init_weights((num_filters_1, 1, 1, cwin1)) # first convolution, 32 filters, stack size 1, 1 rows, cwin1 columns
w2 = init_weights((num_filters_2, num_filters_1, 1, cwin2)) # second convolution, 64 filters, stack size 32 (one stack for each filter from previous layer), 1 row, cwin2 columns
w3 = init_weights((num_filters_3, num_filters_2, 1, cwin3)) # third convolution, 128 filters, stack size 64 (one stack for each filter from previous layes), 1 row, cwin3 columns

# expected
es = 100
# l1 conv:
es = (es - cwin1 + 1)
# l1 max_pool:
es = DownsampleFactorMax.out_shape((1, es), (1, downscale1), st=(1, stride1))[1]

# l2 conv:
es = (es - cwin2 + 1)
# l2 max_pool:
es = DownsampleFactorMax.out_shape((1, es), (1, downscale2), st=(1, stride2))[1]

# l3 conv:
es = (es - cwin3 + 1)
# l3 max_pool:
es = DownsampleFactorMax.out_shape((1, es), (1, downscale3), st=(1, stride3))[1]
print es

w4 = init_weights((num_filters_3 * es, 500))  # fully conected last layer, connects the outputs of 128 filters to 500 (arbitrary) hidden nodes, which are then connected to the output nodes
w_o = init_weights((500, num_of_classes))  # namesto 10 damo stevilo koncnih razredov

noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, 0.2, 0.5)
l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, 0., 0.)
y_x = T.argmax(py_x, axis=1)

cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))
params = [w, w2, w3, w4, w_o]
updates = RMSprop(cost, params, lr=0.001)

train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)
predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)

for i in range(100):
    for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):
        cost = train(trX[start:end], trY[start:end])
    print np.mean(np.argmax(teY, axis=1) == predict(teX))

<commit_msg>popravljen stride na prvem layerju (4 bit)<commit_after>import theano
from theano import tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
import numpy as np
from load import seq_load
from theano.tensor.nnet.conv import conv2d
from theano.tensor.signal.downsample import max_pool_2d
from theano.tensor.signal.downsample import DownsampleFactorMax

theano.config.floatX='float32'

srng = RandomStreams()

def floatX(X):
    return np.asarray(X, dtype=theano.config.floatX)

def init_weights(shape):
    return theano.shared(floatX(np.random.randn(*shape) * 0.01))

def rectify(X):
    return T.maximum(X, 0.)

def softmax(X):
    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))
    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')

def dropout(X, p=0.):
    if p > 0:
        retain_prob = 1 - p
        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)
        X /= retain_prob
    return X

def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):
    grads = T.grad(cost=cost, wrt=params)
    updates = []
    for p, g in zip(params, grads):
        acc = theano.shared(p.get_value() * 0.)
        acc_new = rho * acc + (1 - rho) * g ** 2
        gradient_scaling = T.sqrt(acc_new + epsilon)
        g = g / gradient_scaling
        updates.append((acc, acc_new))
        updates.append((p, p - lr * g))
    return updates

# convolution: filters are moved by one position at a time, see parameter subsample=(1, 1)
#
# max pooling:
#   scaling the input before applying the maxpool filter and
#   displacement (stride) when sliding the max pool filters
stride1=2
downscale1=3

stride2=2
downscale2=2

stride3=2
downscale3=1

def model(X, w, w2, w3, w4, p_drop_conv, p_drop_hidden):
    # TODO spremeni max_pool argumente
    l1a = rectify(conv2d(X, w, border_mode='valid', subsample=(1, 4))) # stride along one (horizontal) dimension only
    l1 = max_pool_2d(l1a, (1, downscale1), st=(1, stride1)) # (1,1)=(vertical, horizontal) downscale, st=(1, step): move to every stride1 column and perform max_pooling there
    l1 = dropout(l1, p_drop_conv)

    l2a = rectify(conv2d(l1, w2, subsample=(1, 1))) # stride along horizontal
    # l2 = max_pool_2d(l2a, (2, 2))
    l2 = max_pool_2d(l2a, (1, downscale2), st=(1, stride2))
    l2 = dropout(l2, p_drop_conv)

    l3a = rectify(conv2d(l2, w3, subsample=(1, 1))) # stride along horizontal
    # l3b = max_pool_2d(l3a, (2, 2))
    l3b = max_pool_2d(l3a, (1, downscale3), st=(1, stride3))
    l3 = T.flatten(l3b, outdim=2)
    l3 = dropout(l3, p_drop_conv)

    l4 = rectify(T.dot(l3, w4))
    l4 = dropout(l4, p_drop_hidden)

    pyx = softmax(T.dot(l4, w_o))
    return l1, l2, l3, l4, pyx

trX, teX, trY, teY = seq_load(onehot=True)

print(trX.shape)
trX = trX.reshape(-1, 1, 1, 100)  # TODO spremeni argumente tukaj
teX = teX.reshape(-1, 1, 1, 100)

X = T.ftensor4()
Y = T.fmatrix()

# size of convolution windows, for each layer different values can be used
cwin1=6
cwin2=5
cwin3=3
num_filters_1=32 # how many different filters to lear at each layer
num_filters_2=48
num_filters_3=64
w = init_weights((num_filters_1, 1, 1, cwin1)) # first convolution, 32 filters, stack size 1, 1 rows, cwin1 columns
w2 = init_weights((num_filters_2, num_filters_1, 1, cwin2)) # second convolution, 64 filters, stack size 32 (one stack for each filter from previous layer), 1 row, cwin2 columns
w3 = init_weights((num_filters_3, num_filters_2, 1, cwin3)) # third convolution, 128 filters, stack size 64 (one stack for each filter from previous layes), 1 row, cwin3 columns

# expected
es = 100
# l1 conv:
es = (es - cwin1 + 1)
# l1 max_pool:
es = DownsampleFactorMax.out_shape((1, es), (1, downscale1), st=(1, stride1))[1]

# l2 conv:
es = (es - cwin2 + 1)
# l2 max_pool:
es = DownsampleFactorMax.out_shape((1, es), (1, downscale2), st=(1, stride2))[1]

# l3 conv:
es = (es - cwin3 + 1)
# l3 max_pool:
es = DownsampleFactorMax.out_shape((1, es), (1, downscale3), st=(1, stride3))[1]
print es

w4 = init_weights((num_filters_3 * es, 500))  # fully conected last layer, connects the outputs of 128 filters to 500 (arbitrary) hidden nodes, which are then connected to the output nodes
w_o = init_weights((500, num_of_classes))  # namesto 10 damo stevilo koncnih razredov

noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, 0.2, 0.5)
l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, 0., 0.)
y_x = T.argmax(py_x, axis=1)

cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))
params = [w, w2, w3, w4, w_o]
updates = RMSprop(cost, params, lr=0.001)

train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)
predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)

for i in range(100):
    for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):
        cost = train(trX[start:end], trY[start:end])
    print np.mean(np.argmax(teY, axis=1) == predict(teX))<|endoftext|>


<commit_before>__version__ = '0.3.0'

class SchemaError(Exception):

    """Error during Schema validation."""

    def __init__(self, autos, errors):
        self.autos = autos if type(autos) is list else [autos]
        self.errors = errors if type(errors) is list else [errors]
        Exception.__init__(self, self.code)

    @property
    def code(self):
        def uniq(seq):
            seen = set()
            seen_add = seen.add
            return [x for x in seq if x not in seen and not seen_add(x)]
        a = uniq(i for i in self.autos if i is not None)
        e = uniq(i for i in self.errors if i is not None)
        if e:
            return '\n'.join(e)
        return '\n'.join(a)

class And(object):

    def __init__(self, *args, **kw):
        self._args = args
        assert list(kw) in (['error'], [])
        self._error = kw.get('error')

    def __repr__(self):
        # Switched to use a map operation instead of list comprehension.
        return '%s(%s)' % (self.__class__.__name__,
                           ', '.join(map(repr, self._args)))

    def validate(self, data):
        for s in [Schema(s, error=self._error) for s in self._args]:
            data = s.validate(data)
        return data

class Or(And):

    def validate(self, data):
        x = SchemaError([], [])
        for s in [Schema(s, error=self._error) for s in self._args]:
            try:
                return s.validate(data)
            except SchemaError as _x:
                x = _x
        raise SchemaError(['%r did not validate %r' % (self, data)] + x.autos,
                          [self._error] + x.errors)

class Use(object):

    def __init__(self, callable_, error=None):
        assert callable(callable_)
        self._callable = callable_
        self._error = error

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self._callable)

    def validate(self, data):
        try:
            return self._callable(data)
        except SchemaError as x:
            raise SchemaError([None] + x.autos, [self._error] + x.errors)
        except BaseException as x:
            f = self._callable.__name__
            raise SchemaError('%s(%r) raised %r' % (f, data, x), self._error)

def priority(s):
    """Return priority for a give object."""
    # Previously this value was calculated in place many times which is
    # expensive.  Do it once early.
    type_of_s = type(s)
    if type_of_s in (list, tuple, set, frozenset):
        return [6]
    if type_of_s is dict:
        return [5]
    if hasattr(s, 'validate'):
        p = [4]
        if hasattr(s, "_schema"):
            p.extend(priority(s._schema))
        return p
    if type_of_s is type:
        return [3]
    if callable(s):
        return [2]
    else:
        return [1]

class Schema(object):

    def __init__(self, schema, error=None):
        self._schema = schema
        self._error = error

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self._schema)

    def validate(self, data):
        s = self._schema
        # Previously this value was calculated in place many times which is
        # expensive.  Do it once early.
        type_of_s = type(s)
        e = self._error
        if type_of_s in (list, tuple, set, frozenset):
            data = Schema(type_of_s, error=e).validate(data)
            return type_of_s(Or(*s, error=e).validate(d) for d in data)
        if type_of_s is dict:
            # Here we are validating that the data is an instance of the same
            # type as the schema (a dict).  However creating a whole new
            # instance of ourselves is wasteful.  Instead we inline the check
            # that would have be undertaken.  The previous approach remains
            # commented below.
#            data = Schema(dict, error=e).validate(data)
            if not isinstance(data, dict):
                raise SchemaError('%r should be instance of %r' % (data, s), e)
            new = type(data)()  # new - is a dict of the validated values
            x = None
            coverage = set()  # non-optional schema keys that were matched
            # For each key and value find a schema entry matching them, if any.
            # As there is not a one-to-one mapping between keys in the
            # dictionary being validated and the schema this section would
            # attempt to find the correct section of the schema to use by
            # calling itself.  For example, to validate the following:
            #
            # schema = Schema({
            #     Optional('foo'):int
            #     Optional(basestring):int
            # })
            #
            # data = {
            #     'foo':1,
            #     'bar':1,
            # }
            #
            # a prioritised list of keys from the schema are validated against
            # the current key being validated from the data. If that validation
            # passes then the value from the schema is used to validate the
            # value of the current key. This is very inefficient as every key
            # in data is (potentially) being compared against every key in
            # schema. This is expensive. Now, we use the same approach as
            # rez.util._LazyAttributeValidator and try and build a mapping
            # between schema keys and data keys, resorting to the original
            # approach only in the (very) few cases where this map is
            # insufficient.
            sorted_skeys = None
            schema_key_map = {}
            for key in s:
                key_name = key
                while isinstance(key_name, Schema):
                    key_name = key_name._schema
                if isinstance(key_name, basestring):
                    schema_key_map[key_name] = key
            for key, value in data.items():
                nkey = None
                if key in schema_key_map:
                    nkey = key
                    svalue = s[schema_key_map[key]]
                    skey = schema_key_map[key]
                else:
                    if not sorted_skeys:
                        sorted_skeys = list(sorted(s, key=priority))
                    for skey in sorted_skeys:
                        svalue = s[skey]
                        try:
                            nkey = Schema(skey, error=e).validate(key)
                        except SchemaError:
                            pass
                if not nkey:
                    continue
                try:
                    nvalue = Schema(svalue, error=e).validate(value)
                except SchemaError as _x:
                    x = _x
                    raise
                else:
                    # Only add to the set if the type of skey is not
                    # Optional.  Previously this was done as a secondary
                    # step (which remains commented out below).
                    if type(skey) is not Optional:
                        coverage.add(skey)
                    valid = True
                    #break
                if valid:
                    new[nkey] = nvalue
                elif skey is not None:
                    if x is not None:
                        raise SchemaError(['invalid value for key %r' % key] +
                                          x.autos, [e] + x.errors)
#            coverage = set(k for k in coverage if type(k) is not Optional)
            required = set(k for k in s if type(k) is not Optional)
            if coverage != required:
                raise SchemaError('missed keys %r' % (required - coverage), e)
            if len(new) != len(data):
                wrong_keys = set(data.keys()) - set(new.keys())
                s_wrong_keys = ', '.join('%r' % k for k in sorted(wrong_keys))
                raise SchemaError('wrong keys %s in %r' % (s_wrong_keys, data),
                                  e)
            return new
        if hasattr(s, 'validate'):
            try:
                return s.validate(data)
            except SchemaError as x:
                raise SchemaError([None] + x.autos, [e] + x.errors)
            except BaseException as x:
                raise SchemaError('%r.validate(%r) raised %r' % (s, data, x),
                                  self._error)
        if type_of_s is type:
            if isinstance(data, s):
                return data
            else:
                raise SchemaError('%r should be instance of %r' % (data, s), e)
        if callable(s):
            f = s.__name__
            try:
                if s(data):
                    return data
            except SchemaError as x:
                raise SchemaError([None] + x.autos, [e] + x.errors)
            except BaseException as x:
                raise SchemaError('%s(%r) raised %r' % (f, data, x),
                                  self._error)
            raise SchemaError('%s(%r) should evaluate to True' % (f, data), e)
        if s == data:
            return data
        else:
            raise SchemaError('%r does not match %r' % (s, data), e)

class Optional(Schema):

    """Marker for an optional part of Schema."""
<commit_msg>schema: add a standardized REZ: comment to all changes made by rez<commit_after>__version__ = '0.3.0'

class SchemaError(Exception):

    """Error during Schema validation."""

    def __init__(self, autos, errors):
        self.autos = autos if type(autos) is list else [autos]
        self.errors = errors if type(errors) is list else [errors]
        Exception.__init__(self, self.code)

    @property
    def code(self):
        def uniq(seq):
            seen = set()
            seen_add = seen.add
            return [x for x in seq if x not in seen and not seen_add(x)]
        a = uniq(i for i in self.autos if i is not None)
        e = uniq(i for i in self.errors if i is not None)
        if e:
            return '\n'.join(e)
        return '\n'.join(a)

class And(object):

    def __init__(self, *args, **kw):
        self._args = args
        assert list(kw) in (['error'], [])
        self._error = kw.get('error')

    def __repr__(self):
        # REZ: Switched to use a map operation instead of list comprehension.
        return '%s(%s)' % (self.__class__.__name__,
                           ', '.join(map(repr, self._args)))

    def validate(self, data):
        for s in [Schema(s, error=self._error) for s in self._args]:
            data = s.validate(data)
        return data

class Or(And):

    def validate(self, data):
        x = SchemaError([], [])
        for s in [Schema(s, error=self._error) for s in self._args]:
            try:
                return s.validate(data)
            except SchemaError as _x:
                x = _x
        raise SchemaError(['%r did not validate %r' % (self, data)] + x.autos,
                          [self._error] + x.errors)

class Use(object):

    def __init__(self, callable_, error=None):
        assert callable(callable_)
        self._callable = callable_
        self._error = error

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self._callable)

    def validate(self, data):
        try:
            return self._callable(data)
        except SchemaError as x:
            raise SchemaError([None] + x.autos, [self._error] + x.errors)
        except BaseException as x:
            f = self._callable.__name__
            raise SchemaError('%s(%r) raised %r' % (f, data, x), self._error)

def priority(s):
    """Return priority for a give object."""
    # REZ: Previously this value was calculated in place many times which is
    #      expensive.  Do it once early.
    # REZ: Changed this to output a list, so that can nicely sort "validate"
    #      items by the sub-priority of their schema
    type_of_s = type(s)
    if type_of_s in (list, tuple, set, frozenset):
        return [6]
    if type_of_s is dict:
        return [5]
    if hasattr(s, 'validate'):
        p = [4]
        if hasattr(s, "_schema"):
            p.extend(priority(s._schema))
        return p
    if type_of_s is type:
        return [3]
    if callable(s):
        return [2]
    else:
        return [1]

class Schema(object):

    def __init__(self, schema, error=None):
        self._schema = schema
        self._error = error

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self._schema)

    def validate(self, data):
        s = self._schema
        # REZ: Previously this value was calculated in place many times which is
        #      expensive.  Do it once early.
        type_of_s = type(s)
        e = self._error
        if type_of_s in (list, tuple, set, frozenset):
            data = Schema(type_of_s, error=e).validate(data)
            return type_of_s(Or(*s, error=e).validate(d) for d in data)
        if type_of_s is dict:
            # REZ: Here we are validating that the data is an instance of the
            #      same type as the schema (a dict).  However creating a whole
            #      new instance of ourselves is wasteful.  Instead we inline
            #      the check that would have be undertaken.  The previous
            #      approach remains commented below.
#            data = Schema(dict, error=e).validate(data)
            if not isinstance(data, dict):
                raise SchemaError('%r should be instance of %r' % (data, s), e)
            new = type(data)()  # new - is a dict of the validated values
            x = None
            coverage = set()  # non-optional schema keys that were matched
            # REZ: For each key and value find a schema entry matching them,
            #      if any. As there is not a one-to-one mapping between keys
            #      in the dictionary being validated and the schema this
            #      section would attempt to find the correct section of the
            #      schema to use by calling itself. For example, to validate
            #      the following:
            #
            #      schema = Schema({
            #          Optional('foo'):int
            #          Optional(basestring):int
            #      })
            #
            #      data = {
            #          'foo':1,
            #          'bar':1,
            #      }
            #
            #      a prioritised list of keys from the schema are validated
            #      against the current key being validated from the data. If
            #      that validation passes then the value from the schema is
            #      used to validate the value of the current key. This is
            #      very inefficient as every key in data is (potentially)
            #      being compared against every key in schema. This is
            #      expensive. Now, we use the same approach as
            #      rez.util._LazyAttributeValidator and try and build a
            #      mapping between schema keys and data keys, resorting to
            #      the original approach only in the (very) few cases where
            #      this map is insufficient.
            sorted_skeys = None
            schema_key_map = {}
            for key in s:
                key_name = key
                while isinstance(key_name, Schema):
                    key_name = key_name._schema
                if isinstance(key_name, basestring):
                    schema_key_map[key_name] = key
            for key, value in data.items():
                nkey = None
                if key in schema_key_map:
                    nkey = key
                    svalue = s[schema_key_map[key]]
                    skey = schema_key_map[key]
                else:
                    if not sorted_skeys:
                        sorted_skeys = list(sorted(s, key=priority))
                    for skey in sorted_skeys:
                        svalue = s[skey]
                        try:
                            nkey = Schema(skey, error=e).validate(key)
                        except SchemaError:
                            pass
                if not nkey:
                    continue
                try:
                    nvalue = Schema(svalue, error=e).validate(value)
                except SchemaError as _x:
                    x = _x
                    raise
                else:
                    # Only add to the set if the type of skey is not
                    # Optional.  Previously this was done as a secondary
                    # step (which remains commented out below).
                    if type(skey) is not Optional:
                        coverage.add(skey)
                    valid = True
                    #break
                if valid:
                    new[nkey] = nvalue
                elif skey is not None:
                    if x is not None:
                        raise SchemaError(['invalid value for key %r' % key] +
                                          x.autos, [e] + x.errors)
            # REZ: we now check for Optional as we add to coverage
#            coverage = set(k for k in coverage if type(k) is not Optional)
            required = set(k for k in s if type(k) is not Optional)
            if coverage != required:
                raise SchemaError('missed keys %r' % (required - coverage), e)
            if len(new) != len(data):
                wrong_keys = set(data.keys()) - set(new.keys())
                s_wrong_keys = ', '.join('%r' % k for k in sorted(wrong_keys))
                raise SchemaError('wrong keys %s in %r' % (s_wrong_keys, data),
                                  e)
            return new
        if hasattr(s, 'validate'):
            try:
                return s.validate(data)
            except SchemaError as x:
                raise SchemaError([None] + x.autos, [e] + x.errors)
            except BaseException as x:
                raise SchemaError('%r.validate(%r) raised %r' % (s, data, x),
                                  self._error)
        if type_of_s is type:
            if isinstance(data, s):
                return data
            else:
                raise SchemaError('%r should be instance of %r' % (data, s), e)
        if callable(s):
            f = s.__name__
            try:
                if s(data):
                    return data
            except SchemaError as x:
                raise SchemaError([None] + x.autos, [e] + x.errors)
            except BaseException as x:
                raise SchemaError('%s(%r) raised %r' % (f, data, x),
                                  self._error)
            raise SchemaError('%s(%r) should evaluate to True' % (f, data), e)
        if s == data:
            return data
        else:
            raise SchemaError('%r does not match %r' % (s, data), e)

class Optional(Schema):

    """Marker for an optional part of Schema."""
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2010 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import logging
import os

GENERATOR_URL_PREFIX = '/web-page-replay-generate-'
POST_IMAGE_URL_PREFIX = '/web-page-replay-post-image-'

class CustomHandlers(object):

  def __init__(self, save_images_dir=None):
    if save_images_dir and not os.path.exists(save_images_dir):
      try:
        os.makedirs(save_images_dir)
      except:
        logging.error('%s does not exist and could not be created.',
                      save_images_dir)
        save_images_dir = None
    self.save_images_dir = save_images_dir

  def handle(self, request):
    """Handles special URLs needed for the benchmark.

    Args:
      request: an http request
    Returns:
      If request is for a special URL, a 3-digit integer like 404.
      Otherwise, None.
    """
    response_code = self.get_generator_url_response_code(request.path)
    if response_code:
      return response_code

    response_code = self.handle_possible_post_image(request)
    if response_code:
      return response_code

    return None

  def get_generator_url_response_code(self, request_path):
    """Parse special generator URLs for the embedded response code.

    Clients like perftracker can use URLs of this form to request
    a response with a particular response code.

    Args:
      request_path: a string like "/foo", or "/web-page-replay-generator-404"
    Returns:
      On a match, a 3-digit integer like 404.
      Otherwise, None.
    """
    prefix, response_code = request_path[:-3], request_path[-3:]
    if prefix == GENERATOR_URL_PREFIX and response_code.isdigit():
      return int(response_code)
    return None

  def handle_possible_post_image(self, request):
    """

    Clients like perftracker can use URLs of this form to request
    a response with a particular response code.

    Args:
      request_path: a string like "/foo", or "/web-page-replay-set-phase-cold"

    Returns:
      True if request was recognized as a set phase request.
      False otherwise.
    """
    if not self.save_images_dir:
      return None

    prefix = request.path[:len(POST_IMAGE_URL_PREFIX)]
    basename = request.path[len(POST_IMAGE_URL_PREFIX):]
    if prefix != POST_IMAGE_URL_PREFIX or not basename.isalpha():
      return None

    data = request.request_body
    PREFIX = 'data:image/png;base64,'
    if not data.startswith(PREFIX):
      logging.error('Unexpected image format for: %s', basename)
      return 400

    data = data[len(PREFIX):]
    png = base64.b64decode(data)
    filename = '%s/%s-%s.png' % (self.save_images_dir, request.host, basename)
    f = file(filename, 'w')
    f.write(png)
    f.close()
    return 200
<commit_msg>Fix comments.<commit_after>#!/usr/bin/env python
# Copyright 2010 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import logging
import os

GENERATOR_URL_PREFIX = '/web-page-replay-generate-'
POST_IMAGE_URL_PREFIX = '/web-page-replay-post-image-'

class CustomHandlers(object):

  def __init__(self, save_images_dir=None):
    if save_images_dir and not os.path.exists(save_images_dir):
      try:
        os.makedirs(save_images_dir)
      except:
        logging.error('%s does not exist and could not be created.',
                      save_images_dir)
        save_images_dir = None
    self.save_images_dir = save_images_dir

  def handle(self, request):
    """Handles special URLs needed for the benchmark.

    Args:
      request: an http request
    Returns:
      If request is for a special URL, a 3-digit integer like 404.
      Otherwise, None.
    """
    response_code = self.get_generator_url_response_code(request.path)
    if response_code:
      return response_code

    response_code = self.handle_possible_post_image(request)
    if response_code:
      return response_code

    return None

  def get_generator_url_response_code(self, request_path):
    """Parse special generator URLs for the embedded response code.

    Clients like perftracker can use URLs of this form to request
    a response with a particular response code.

    Args:
      request_path: a string like "/foo", or "/web-page-replay-generator-404"
    Returns:
      On a match, a 3-digit integer like 404.
      Otherwise, None.
    """
    prefix, response_code = request_path[:-3], request_path[-3:]
    if prefix == GENERATOR_URL_PREFIX and response_code.isdigit():
      return int(response_code)
    return None

  def handle_possible_post_image(self, request):
    """If sent, saves embedded image to local directory.

    Expects a special url containing the filename. If sent, saves the base64
    encoded request body as a PNG image locally. This feature is enabled by
    passing in save_images_dir to the initializer for this class.

    Args:
      request: an http request

    Returns:
      On a match, a 3-digit integer response code.
      False otherwise.
    """
    if not self.save_images_dir:
      return None

    prefix = request.path[:len(POST_IMAGE_URL_PREFIX)]
    basename = request.path[len(POST_IMAGE_URL_PREFIX):]
    if prefix != POST_IMAGE_URL_PREFIX or not basename.isalpha():
      return None

    data = request.request_body
    PREFIX = 'data:image/png;base64,'
    if not data.startswith(PREFIX):
      logging.error('Unexpected image format for: %s', basename)
      return 400

    data = data[len(PREFIX):]
    png = base64.b64decode(data)
    filename = '%s/%s-%s.png' % (self.save_images_dir, request.host, basename)
    f = file(filename, 'w')
    f.write(png)
    f.close()
    return 200
<|endoftext|>


<commit_before>from .widgets import *
<commit_msg>Register a widget comm target in the widgets, not ipython_kernel.<commit_after>from .widgets import *

# Register comm target for Javascript initialized widgets..
ip = get_ipython()
ip.kernel.comm_manager.register_target('ipython.widget', Widget.handle_comm_opened)
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *

urlpatterns = patterns('',

        url(r'^$',
            'django.views.generic.simple.direct_to_template',
            {'template': 'index.html',
            },
            name='cwod_docs'),

        (r'^api/', include('cwod_api.urls')),
        (r'^/', include('cwod.urls')),
)
<commit_msg>Update urls to reflect front-end structure<commit_after>from django.conf.urls.defaults import *

urlpatterns = patterns('',

        (r'^api/', include('cwod_api.urls')),

        (r'^', include('cwod.urls')),
)
<|endoftext|>


<commit_before>from PIL import Image
import numpy as np
import tensorflow as tf

# colour map
# label_colours = [(0,0,0)
#                 # 0=background
#                 ,(128,0,0),(0,128,0),(128,128,0),(0,0,128),(128,0,128)
#                 # 1=aeroplane, 2=bicycle, 3=bird, 4=boat, 5=bottle
#                 ,(0,128,128),(128,128,128),(64,0,0),(192,0,0),(64,128,0)
#                 # 6=bus, 7=car, 8=cat, 9=chair, 10=cow
#                 ,(192,128,0),(64,0,128),(192,0,128),(64,128,128),(192,128,128)
#                 # 11=diningtable, 12=dog, 13=horse, 14=motorbike, 15=person
#                 ,(0,64,0),(128,64,0),(0,192,0),(128,192,0),(0,64,128)]
#                 # 16=potted plant, 17=sheep, 18=sofa, 19=train, 20=tv/monitor

#for LUNA 16
# label_colours = [(0,0,0)
#                 # 0=background
#                 ,(128,0,0),(0,128,0),(128,128,0),(0,0,128)]
#                 # 1=left lung, 2=right lung, 3=pipe, 4=nodule,

#for ILD
label_colours = [(0,0,0)
                # 0=background
                ,(128,128,0),(0,0,128)]
                # 1= lungs, 2=nodules,

def decode_labels(mask, num_images=1, num_classes=21):
    """Decode batch of segmentation masks.

    Args:
      mask: result of inference after taking argmax.
      num_images: number of images to decode from the batch.
      num_classes: number of classes to predict (including background).

    Returns:
      A batch with num_images RGB images of the same size as the input.
    """
    n, h, w, c = mask.shape
    assert(n >= num_images), 'Batch size %d should be greater or equal than number of images to save %d.' % (n, num_images)
    outputs = np.zeros((num_images, h, w, 3), dtype=np.uint8)
    for i in range(num_images):
      img = Image.new('RGB', (len(mask[i, 0]), len(mask[i])))
      pixels = img.load()
      for j_, j in enumerate(mask[i, :, :, 0]):
          for k_, k in enumerate(j):
              if k < num_classes:
                  pixels[k_,j_] = label_colours[k]
      outputs[i] = np.array(img)
    return outputs

def prepare_label(input_batch, new_size, num_classes, one_hot=True):
    """Resize masks and perform one-hot encoding.

    Args:
      input_batch: input tensor of shape [batch_size H W 1].
      new_size: a tensor with new height and width.
      num_classes: number of classes to predict (including background).
      one_hot: whether perform one-hot encoding.

    Returns:
      Outputs a tensor of shape [batch_size h w 21]
      with last dimension comprised of 0's and 1's only.
    """
    with tf.name_scope('label_encode'):
        input_batch = tf.image.resize_nearest_neighbor(input_batch, new_size) # as labels are integer numbers, need to use NN interp.
        input_batch = tf.squeeze(input_batch, squeeze_dims=[3]) # reducing the channel dimension.
        if one_hot:
            input_batch = tf.one_hot(input_batch, depth=num_classes)
    return input_batch

def inv_preprocess(imgs, num_images, img_mean):
    """Inverse preprocessing of the batch of images.
       Add the mean vector and convert from BGR to RGB.

    Args:
      imgs: batch of input images.
      num_images: number of images to apply the inverse transformations on.
      img_mean: vector of mean colour values.

    Returns:
      The batch of the size num_images with the same spatial dimensions as the input.
    """
    n, h, w, c = imgs.shape
    assert(n >= num_images), 'Batch size %d should be greater or equal than number of images to save %d.' % (n, num_images)
    outputs = np.zeros((num_images, h, w, c), dtype=np.uint8)
    for i in range(num_images):
        outputs[i] = (imgs[i] + img_mean)[:, :, ::-1].astype(np.uint8)
    return outputs
<commit_msg>add color map for ImageNet 2016 scene-parsing<commit_after>from PIL import Image
import numpy as np
import tensorflow as tf

#colour map
label_colours = [(0,0,0)
                # 0=background
                ,(128,0,0),(0,128,0),(128,128,0),(0,0,128),(128,0,128)
                # 1=aeroplane, 2=bicycle, 3=bird, 4=boat, 5=bottle
                ,(0,128,128),(128,128,128),(64,0,0),(192,0,0),(64,128,0)
                # 6=bus, 7=car, 8=cat, 9=chair, 10=cow
                ,(192,128,0),(64,0,128),(192,0,128),(64,128,128),(192,128,128)
                # 11=diningtable, 12=dog, 13=horse, 14=motorbike, 15=person
                ,(0,64,0),(128,64,0),(0,192,0),(128,192,0),(0,64,128)]
                # 16=potted plant, 17=sheep, 18=sofa, 19=train, 20=tv/monitor

#for LUNA 16
# label_colours = [(0,0,0)
#                 # 0=background
#                 ,(128,0,0),(0,128,0),(128,128,0),(0,0,128)]
#                 # 1=left lung, 2=right lung, 3=pipe, 4=nodule,

# #for ILD
# label_colours = [(0,0,0)
#                 # 0=background
#                 ,(128,128,0),(0,0,128)]
#                 # 1= lungs, 2=nodules,
#for ImageNet2016 Scene-parsing
label_colours = [(0,0,0), (251,209,244), (44,230,121), (156,40,149), (166,219,98), (35,229,138), (143,56,194), (144,223,70), (200,162,57), (120,225,199), (87,203,13), (185,1,136), (16,167,16), (29,249,241), (17,192,40), (199,44,241), (193,196,159), (241,172,78), (56,94,128), (231,166,116), (50,209,252), (217,56,227), (168,198,178), (77,179,188), (236,191,103), (248,138,151), (214,251,89), (208,204,187), (115,104,49), (29,202,113), (159,160,95), (78,188,13), (83,203,82), (8,234,116), (80,159,200), (124,194,2), (192,146,237), (64,3,73), (17,213,58), (106,54,105), (125,72,155), (202,36,231), (79,144,4), (118,185,128), (138,61,178), (23,182,182), (154,114,4), (201,0,83), (21,134,53), (194,77,237), (198,81,106), (37,222,181), (203,185,14), (134,140,113), (220,196,79), (64,26,68), (128,89,2), (199,228,65), (62,215,111), (124,148,166), (221,119,245), (68,57,158), (80,47,26), (143,59,56), (14,80,215), (212,132,31), (2,234,129), (134,179,44), (53,21,129), (80,176,236), (154,39,168), (221,44,139), (103,56,185), (224,138,83), (243,93,235), (80,158,63), (81,229,38), (116,215,38), (103,69,182), (66,81,5), (96,157,229), (164,49,170), (14,42,146), (164,67,44), (108,116,151), (144,8,144), (85,68,228), (16,236,72), (108,7,86), (172,27,94), (119,247,193), (155,240,152), (49,158,204), (23,193,204), (228,66,107), (69,36,163), (238,158,228), (202,226,35), (194,243,151), (192,56,76), (16,115,240), (61,190,185), (7,134,32), (192,87,171), (45,11,254), (179,183,31), (181,175,146), (13,187,133), (12,1,2), (63,199,190), (221,248,32), (183,221,51), (90,111,162), (82,0,6), (40,0,239), (252,81,54), (110,245,152), (0,187,93), (163,154,153), (134,66,99), (123,150,242), (38,144,137), (59,180,230), (144,212,16), (132,125,200), (26,3,35), (199,56,92), (83,223,224), (203,47,137), (74,74,251), (246,81,197), (168,130,178), (136,85,200), (186,147,103), (170,21,85), (104,52,182), (166,147,202), (103,119,71), (74,161,165), (14,9,83), (129,194,43), (7,100,55), (13,12,170), (30,21,22), (224,189,139), (40,77,25), (194,14,94), (178,8,231), (234,166,8), (248,25,7), (139,181,248)]
def decode_labels(mask, num_images=1, num_classes=21):
    """Decode batch of segmentation masks.

    Args:
      mask: result of inference after taking argmax.
      num_images: number of images to decode from the batch.
      num_classes: number of classes to predict (including background).

    Returns:
      A batch with num_images RGB images of the same size as the input.
    """
    n, h, w, c = mask.shape
    assert(n >= num_images), 'Batch size %d should be greater or equal than number of images to save %d.' % (n, num_images)
    outputs = np.zeros((num_images, h, w, 3), dtype=np.uint8)
    for i in range(num_images):
      img = Image.new('RGB', (len(mask[i, 0]), len(mask[i])))
      pixels = img.load()
      for j_, j in enumerate(mask[i, :, :, 0]):
          for k_, k in enumerate(j):
              if k < num_classes:
                  pixels[k_,j_] = label_colours[k]
      outputs[i] = np.array(img)
    return outputs

def prepare_label(input_batch, new_size, num_classes, one_hot=True):
    """Resize masks and perform one-hot encoding.

    Args:
      input_batch: input tensor of shape [batch_size H W 1].
      new_size: a tensor with new height and width.
      num_classes: number of classes to predict (including background).
      one_hot: whether perform one-hot encoding.

    Returns:
      Outputs a tensor of shape [batch_size h w 21]
      with last dimension comprised of 0's and 1's only.
    """
    with tf.name_scope('label_encode'):
        input_batch = tf.image.resize_nearest_neighbor(input_batch, new_size) # as labels are integer numbers, need to use NN interp.
        input_batch = tf.squeeze(input_batch, squeeze_dims=[3]) # reducing the channel dimension.
        if one_hot:
            input_batch = tf.one_hot(input_batch, depth=num_classes)
    return input_batch

def inv_preprocess(imgs, num_images, img_mean):
    """Inverse preprocessing of the batch of images.
       Add the mean vector and convert from BGR to RGB.

    Args:
      imgs: batch of input images.
      num_images: number of images to apply the inverse transformations on.
      img_mean: vector of mean colour values.

    Returns:
      The batch of the size num_images with the same spatial dimensions as the input.
    """
    n, h, w, c = imgs.shape
    assert(n >= num_images), 'Batch size %d should be greater or equal than number of images to save %d.' % (n, num_images)
    outputs = np.zeros((num_images, h, w, c), dtype=np.uint8)
    for i in range(num_images):
        outputs[i] = (imgs[i] + img_mean)[:, :, ::-1].astype(np.uint8)
    return outputs
<|endoftext|>