<commit_before>from .base import BaseCommandDBConnector

class PgDumpConnector(BaseCommandDBConnector):
    """
    PostgreSQL connector, it uses pg_dump`` to create an SQL text file
    and ``psql`` for restore it.
    """
    extension = 'psql'
    dump_cmd = 'pg_dump'
    restore_cmd = 'psql'
    single_transaction = True
    drop = True

    def run_command(self, *args, **kwargs):
        if self.settings.get('PASSWORD'):
            env = kwargs.get('env', {})
            env['PGPASSWORD'] = self.settings['PASSWORD']
            kwargs['env'] = env
        return super(PgDumpConnector, self).run_command(*args, **kwargs)

    def _create_dump(self):
        cmd = '{} {}'.format(self.dump_cmd, self.settings['NAME'])
        if self.settings.get('HOST'):
            cmd += ' --host={}'.format(self.settings['HOST'])
        if self.settings.get('PORT'):
            cmd += ' --port={}'.format(self.settings['PORT'])
        if self.settings.get('USER'):
            cmd += ' --user={}'.format(self.settings['USER'])
        cmd += ' --no-password'
        for table in self.exclude:
            cmd += ' --exclude-table={}'.format(table)
        if self.drop:
            cmd += ' --clean'
        cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix)
        stdout, stderr = self.run_command(cmd, env=self.dump_env)
        return stdout

    def _restore_dump(self, dump):
        cmd = '{} {}'.format(self.restore_cmd, self.settings['NAME'])
        if self.settings.get('HOST'):
            cmd += ' --host={}'.format(self.settings['HOST'])
        if self.settings.get('PORT'):
            cmd += ' --port={}'.format(self.settings['PORT'])
        if self.settings.get('USER'):
            cmd += ' --user={}'.format(self.settings['USER'])
        cmd += ' --no-password'
        # without this, psql terminates with an exit value of 0 regardless of errors
        cmd += ' --set ON_ERROR_STOP=on'
        if self.single_transaction:
            cmd += ' --single-transaction'
        cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix)
        stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
        return stdout, stderr

class PgDumpGisConnector(PgDumpConnector):
    """
    PostgreGIS connector, same than :class:`PgDumpGisConnector` but enable
    postgis if not made.
    """
    psql_cmd = 'psql'

    def _enable_postgis(self):
        cmd = '{} -c "CREATE EXTENSION IF NOT EXISTS postgis;"'.format(
            self.psql_cmd)
        cmd += ' --user={}'.format(self.settings['ADMIN_USER'])
        cmd += ' --no-password'
        if self.settings.get('HOST'):
            cmd += ' --host={}'.format(self.settings['HOST'])
        if self.settings.get('PORT'):
            cmd += ' --port={}'.format(self.settings['PORT'])
        return self.run_command(cmd)

    def _restore_dump(self, dump):
        if self.settings.get('ADMIN_USER'):
            self._enable_postgis()
        return super(PgDumpGisConnector, self)._restore_dump(dump)
<commit_msg>Fix username argument for pg_dump<commit_after>from .base import BaseCommandDBConnector

class PgDumpConnector(BaseCommandDBConnector):
    """
    PostgreSQL connector, it uses pg_dump`` to create an SQL text file
    and ``psql`` for restore it.
    """
    extension = 'psql'
    dump_cmd = 'pg_dump'
    restore_cmd = 'psql'
    single_transaction = True
    drop = True

    def run_command(self, *args, **kwargs):
        if self.settings.get('PASSWORD'):
            env = kwargs.get('env', {})
            env['PGPASSWORD'] = self.settings['PASSWORD']
            kwargs['env'] = env
        return super(PgDumpConnector, self).run_command(*args, **kwargs)

    def _create_dump(self):
        cmd = '{} {}'.format(self.dump_cmd, self.settings['NAME'])
        if self.settings.get('HOST'):
            cmd += ' --host={}'.format(self.settings['HOST'])
        if self.settings.get('PORT'):
            cmd += ' --port={}'.format(self.settings['PORT'])
        if self.settings.get('USER'):
            cmd += ' --username={}'.format(self.settings['USER'])
        cmd += ' --no-password'
        for table in self.exclude:
            cmd += ' --exclude-table={}'.format(table)
        if self.drop:
            cmd += ' --clean'
        cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix)
        stdout, stderr = self.run_command(cmd, env=self.dump_env)
        return stdout

    def _restore_dump(self, dump):
        cmd = '{} {}'.format(self.restore_cmd, self.settings['NAME'])
        if self.settings.get('HOST'):
            cmd += ' --host={}'.format(self.settings['HOST'])
        if self.settings.get('PORT'):
            cmd += ' --port={}'.format(self.settings['PORT'])
        if self.settings.get('USER'):
            cmd += ' --user={}'.format(self.settings['USER'])
        cmd += ' --no-password'
        # without this, psql terminates with an exit value of 0 regardless of errors
        cmd += ' --set ON_ERROR_STOP=on'
        if self.single_transaction:
            cmd += ' --single-transaction'
        cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix)
        stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
        return stdout, stderr

class PgDumpGisConnector(PgDumpConnector):
    """
    PostgreGIS connector, same than :class:`PgDumpGisConnector` but enable
    postgis if not made.
    """
    psql_cmd = 'psql'

    def _enable_postgis(self):
        cmd = '{} -c "CREATE EXTENSION IF NOT EXISTS postgis;"'.format(
            self.psql_cmd)
        cmd += ' --user={}'.format(self.settings['ADMIN_USER'])
        cmd += ' --no-password'
        if self.settings.get('HOST'):
            cmd += ' --host={}'.format(self.settings['HOST'])
        if self.settings.get('PORT'):
            cmd += ' --port={}'.format(self.settings['PORT'])
        return self.run_command(cmd)

    def _restore_dump(self, dump):
        if self.settings.get('ADMIN_USER'):
            self._enable_postgis()
        return super(PgDumpGisConnector, self)._restore_dump(dump)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#   __init__.py — Model initialisation code
#
#   This file is part of debexpo - http://debexpo.workaround.org
#
#   Copyright © 2008 Jonny Lamb <jonny@debian.org>
#
#   Permission is hereby granted, free of charge, to any person
#   obtaining a copy of this software and associated documentation
#   files (the "Software"), to deal in the Software without
#   restriction, including without limitation the rights to use,
#   copy, modify, merge, publish, distribute, sublicense, and/or sell
#   copies of the Software, and to permit persons to whom the
#   Software is furnished to do so, subject to the following
#   conditions:
#
#   The above copyright notice and this permission notice shall be
#   included in all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#   OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#   OTHER DEALINGS IN THE SOFTWARE.

"""
Model initialization functions.
"""

__author__ = 'Jonny Lamb'
__copyright__ = 'Copyright © 2008 Jonny Lamb'
__license__ = 'MIT'

import sqlalchemy as sa
from sqlalchemy import orm

from debexpo.model import meta

def init_model(engine):
    """
    Initializes the model.
    This should be called before using any of the tables or classes in the model.

    ``engine``
        SQLAlchemy engine to bind to.
    """

    sm = orm.sessionmaker(autoflush=True, autocommit=False, bind=engine)

    meta.engine = engine
    meta.session = orm.scoped_session(sm)

def import_all_models():
    """
    Import all models from debexpo.models. This is useful when creating tables.
    """

    from debexpo.model import binary_packages, package_files, packages, source_packages, \
        user_metrics, package_comments, package_info, package_versions, user_countries, \
        users, package_subscriptions

class OrmObject(object):
    """
    A base class for ORM mapped objects.

    This class was found and then altered for debexpo from
    http://www.sqlalchemy.org/trac/wiki/UsageRecipes/GenericOrmBaseClass

    Contributed by ltbarcly (Justin Van Winkle).
    """
    def __init__(self, **kw):
        if not hasattr(self, 'foreign'):
            self.foreign = []

        self.__items__ = []
        for item in dir(self):
            if not item.startswith('_') and item is not "foreign":
                self.__items__.append(item)

        for key in kw:
            if key in self.__items__ or key in self.foreign:
                setattr(self, key, kw[key])
            else:
                raise AttributeError('Cannot set attribute which is ' +
                                     'not column in mapped table: %s' % (key,))

    def __repr__(self):
        atts = []
        for key in self.__items__:
            atts.append((key, getattr(self, key)))

        return self.__class__.__name__ + '(' + ', '.join(x[0] + '=' + repr(x[1]) for x in atts) + ')'

<commit_msg>Further changes to OrmObject to make it work, and not die on __repr__.<commit_after># -*- coding: utf-8 -*-
#
#   __init__.py — Model initialisation code
#
#   This file is part of debexpo - http://debexpo.workaround.org
#
#   Copyright © 2008 Jonny Lamb <jonny@debian.org>
#
#   Permission is hereby granted, free of charge, to any person
#   obtaining a copy of this software and associated documentation
#   files (the "Software"), to deal in the Software without
#   restriction, including without limitation the rights to use,
#   copy, modify, merge, publish, distribute, sublicense, and/or sell
#   copies of the Software, and to permit persons to whom the
#   Software is furnished to do so, subject to the following
#   conditions:
#
#   The above copyright notice and this permission notice shall be
#   included in all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#   OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#   OTHER DEALINGS IN THE SOFTWARE.

"""
Model initialization functions.
"""

__author__ = 'Jonny Lamb'
__copyright__ = 'Copyright © 2008 Jonny Lamb'
__license__ = 'MIT'

import sqlalchemy as sa
from sqlalchemy import orm

from debexpo.model import meta

def init_model(engine):
    """
    Initializes the model.
    This should be called before using any of the tables or classes in the model.

    ``engine``
        SQLAlchemy engine to bind to.
    """

    sm = orm.sessionmaker(autoflush=True, autocommit=False, bind=engine)

    meta.engine = engine
    meta.session = orm.scoped_session(sm)

def import_all_models():
    """
    Import all models from debexpo.models. This is useful when creating tables.
    """

    from debexpo.model import binary_packages, package_files, packages, source_packages, \
        user_metrics, package_comments, package_info, package_versions, user_countries, \
        users, package_subscriptions

class OrmObject(object):
    """
    A base class for ORM mapped objects.

    This class was found and then altered for debexpo from
    http://www.sqlalchemy.org/trac/wiki/UsageRecipes/GenericOrmBaseClass

    Contributed by ltbarcly (Justin Van Winkle).
    """
    def __init__(self, **kw):
        if not hasattr(self, 'foreign'):
            self.foreign = []

        items = dir(self)
        for key in kw:
            if key in items or key in self.foreign:
                setattr(self, key, kw[key])
            else:
                raise AttributeError('Cannot set attribute which is ' +
                                     'not column in mapped table: %s' % (key,))

    def __repr__(self):
        atts = []
        for key in dir(self):
            if not key.startswith('_') and key is not "foreign" and key not in self.foreign:
                atts.append((key, getattr(self, key)))

        return self.__class__.__name__ + '(' + ', '.join(x[0] + '=' + repr(x[1]) for x in atts) + ')'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, with_statement, absolute_import
import os
import tempfile
from contextlib import contextmanager
import subprocess

def safe_subprocess(command_array):
    """Executes shell command. Do not raise

        Args:
            command_array (list): ideally an array of string elements, but
                may be a string. Will be converted to an array of strings

        Returns:
            bool, str: True/False (command succeeded), command output
    """

    try:
        # Ensure all args are strings
        if isinstance(command_array, list):
            command_array_str = [str(x) for x in command_array]
        else:
            command_array_str = [str(command_array)]
        return True, subprocess.check_output(command_array_str,
                                             stderr=subprocess.STDOUT)
    except OSError as ex:
        return False, ex.__str__()
    except subprocess.CalledProcessError as ex:
        return False, ex.output

@contextmanager
def atomic_write(filepath):
    """
        Writeable file object that atomically updates a file (using a temporary file).

        Args:
            filepath (str): the file path to be opened
    """
    # Put tmp file to same directory as target file, to allow atomic move
    realpath = os.path.realpath(filepath)
    tmppath = os.path.dirname(realpath)
    with tempfile.NamedTemporaryFile(dir=tmppath, delete=False) as tempf:
        with open(tempf.name, mode='w+') as tmp:
            yield tmp
            tmp.flush()
            os.fsync(tmp.fileno())
        os.rename(tempf.name, realpath)
<commit_msg>chmod destination file to 0644 (closes #14)<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, with_statement, absolute_import
import os
import stat
import tempfile
from contextlib import contextmanager
import subprocess

def safe_subprocess(command_array):
    """Executes shell command. Do not raise

        Args:
            command_array (list): ideally an array of string elements, but
                may be a string. Will be converted to an array of strings

        Returns:
            bool, str: True/False (command succeeded), command output
    """

    try:
        # Ensure all args are strings
        if isinstance(command_array, list):
            command_array_str = [str(x) for x in command_array]
        else:
            command_array_str = [str(command_array)]
        return True, subprocess.check_output(command_array_str,
                                             stderr=subprocess.STDOUT)
    except OSError as ex:
        return False, ex.__str__()
    except subprocess.CalledProcessError as ex:
        return False, ex.output

@contextmanager
def atomic_write(filepath):
    """
        Writeable file object that atomically updates a file (using a temporary file).

        Args:
            filepath (str): the file path to be opened
    """
    # Put tmp file to same directory as target file, to allow atomic move
    realpath = os.path.realpath(filepath)
    tmppath = os.path.dirname(realpath)
    with tempfile.NamedTemporaryFile(dir=tmppath, delete=False) as tempf:
        with open(tempf.name, mode='w+') as tmp:
            yield tmp
            tmp.flush()
            os.fsync(tmp.fileno())
        os.rename(tempf.name, realpath)
        os.chmod(realpath, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)

<|endoftext|>


<commit_before># -*- coding: utf8 -*-

from time import time
import heapq

__author__ = 'sergey'

class StorageTTLseconds(object):

    # Maximum seconds after that cache is expired for writed blocks
    _max_write_ttl = 10
    # Maximum cache size in bytes for block that writed recently
    _max_write_cache_size = 256*1024*1024
    # Maximum seconds after that cache is expired for readed blocks
    _max_read_ttl = 20
    # Maximum cache size in bytes for block that accessed recently
    _max_read_cache_size = 256*1024*1024
    # Expired maximum cache size in %
    _max_size_trsh = 10

    _inodes = None
    _block_size = 128*1024

    def __init__(self):
        self._inodes = {}
        pass

    def __len__(self):
        s = 0
        for inode in self._inodes:
            s += len(self._inodes[inode])
        return s

    def setBlockSize(self, in_bytes):
        self._block_size = in_bytes
        return self

    def setMaxWriteCacheSize(self, in_bytes):
        self._max_write_cache_size = in_bytes
        return self

    def setMaxReadCacheSize(self, in_bytes):
        self._max_read_cache_size = in_bytes
        return self

    def setMaxWriteTtl(self, seconds):
        self._max_write_ttl = seconds
        return self

    def setMaxReadTtl(self, seconds):
        self._max_read_ttl = seconds
        return self

    def set(self, inode, block_number, block, writed=False):
        """
        @type   inode: int
        @type   block_number: int
        @type   block: BytesIO
        @type   writed: bool
        """

        inode = str(inode)
        block_number = str(block_number)

        new = False
        if inode not in self._inodes:
            self._inodes[ inode ] = {}
            new = True

        inode_data = self._inodes[inode]

        if block_number not in inode_data:
            inode_data[ block_number ] = {}
            new = True

        block_data = inode_data[block_number]

        block_data["time"] = time()
        block_data["block"] = block

        if writed:
            block_data["w"] = True
        else:
            block_data["w"] = block_data.get("w", False)

        return self

    def get(self, inode, block_number, default=None):

        inode = str(inode)
        block_number = str(block_number)

        inode_data = self._inodes.get(inode, {})

        block_data = inode_data.get(block_number, {})

        if not block_data:
            return default

        now = time()
        t = block_data["time"]
        if now - t > self._max_write_ttl:
            return default

        val = block_data.get("block", default)

        # update last request time
        block_data["time"] = time()

        return val

    def getCachedSize(self, writed=False):
        size = 0
        for inode in self._inodes.keys():
            for block_data in self._inodes[inode].values():
                if block_data["w"] != writed:
                    continue

                block_data["block"].seek(0, 2)
                size += block_data["block"].tell()
                block_data["block"].seek(0, 0)
        return size

    def isWritedCacheFull(self):
        filled = 100.0 * self.getCachedSize(True) / self._max_write_cache_size
        max_fill = 100 + self._max_size_trsh
        return filled > max_fill

    def isReadCacheFull(self):
        filled = 100.0 * self.getCachedSize(False) / self._max_read_cache_size
        max_fill = 100 + self._max_size_trsh
        return filled > max_fill

    def expired(self, writed=False):
        now = time()

        if writed:
            old_inodes = {}
        else:
            old_inodes = 0

        for inode in tuple(self._inodes.keys()):

            inode_data = self._inodes[inode]

            for bn in tuple(inode_data.keys()):
                block_data = inode_data[bn]

                t = block_data["time"]
                if block_data["w"] != writed:
                    continue

                if now - t > self._max_write_ttl:
                    if writed:
                        old_inode_data = old_inodes.get(inode, {})
                        old_inode_data[bn] = block_data.copy()
                        old_inodes[inode] = old_inode_data
                    else:
                        old_inodes += 1
                    del inode_data[bn]

            if not inode_data and inode in self._inodes:
                del self._inodes[inode]

        return old_inodes

    def expireByCount(self, writed=False):
        now = time()

        # 1. Fill heap

        heap = []

        inodesKeys = tuple(self._inodes.keys())

        for inode in inodesKeys:

            inode_data = self._inodes[inode]

            for bn in inode_data.keys():
                block_data = inode_data[bn]

                t = block_data["time"]
                if block_data["w"] != writed:
                    continue

                heapq.heappush(
                    heap, (
                        int((now - t)*10**6),
                        inode,
                        bn,
                        len(block_data["block"].getvalue())
                    )
                )

        # 2. Sort heap

        currentSize = self.getCachedSize(writed)

        if writed:
            maxSize = self._max_write_cache_size
        else:
            maxSize = self._max_read_cache_size

        needMaxSize = maxSize * (100 - self._max_size_trsh) / 100

        nget = int((currentSize - needMaxSize) / self._block_size)

        mostrecent = []
        while True:
            needSize = currentSize
            mostrecent = heapq.nsmallest(nget, heap)
            for dt, inode, bn, bsize in mostrecent:
                needSize -= bsize
            if needSize <= needMaxSize:
                break
            nget += 1

        # 3. Convert data

        heap_inodes = {}
        for dt, inode, bn, bsize in mostrecent:
            if inode not in heap_inodes:
                heap_inodes[ inode ] = ()
            heap_inodes[ inode ] += (bn,)

        del heap
        del mostrecent

        # 4. Expire cache, filter by new data

        if writed:
            old_inodes = {}
        else:
            old_inodes = 0

        for inode in inodesKeys:

            inode_data = self._inodes[inode]

            for bn in tuple(inode_data.keys()):

                bn = str(bn)

                block_data = inode_data[bn]

                if block_data["w"] != writed:
                    continue

                # Expire inodes that in heap or by block numbers
                if inode not in heap_inodes or\
                    bn not in heap_inodes[ inode ]:

                    if writed:
                        old_inode_data = old_inodes.get(inode, {})
                        old_inode_data[bn] = block_data.copy()
                        old_inodes[inode] = old_inode_data
                    else:
                        old_inodes += 1
                    del inode_data[bn]

            if not inode_data and inode in self._inodes:
                del self._inodes[inode]

        return old_inodes

    def clear(self):
        old_inodes = self._inodes.copy()
        self._inodes = {}
        return old_inodes
<commit_msg>Faster operations<commit_after># -*- coding: utf8 -*-

from time import time
import heapq

__author__ = 'sergey'

class StorageTTLseconds(object):

    # Maximum seconds after that cache is expired for writed blocks
    _max_write_ttl = 10

    # Maximum cache size in bytes for block that writed recently
    _max_write_cache_size = 256*1024*1024
    _cur_write_cache_size = 0

    # Maximum seconds after that cache is expired for readed blocks
    _max_read_ttl = 20

    # Maximum cache size in bytes for block that accessed recently
    _max_read_cache_size = 256*1024*1024
    _cur_read_cache_size = 0

    # Expired maximum cache size in %
    _max_size_trsh = 10

    _inodes = None
    _block_size = 128*1024

    def __init__(self):
        self._inodes = {}
        pass

    def __len__(self):
        s = 0
        for inode in self._inodes:
            s += len(self._inodes[inode])
        return s

    def setBlockSize(self, in_bytes):
        self._block_size = in_bytes
        return self

    def setMaxWriteCacheSize(self, in_bytes):
        self._max_write_cache_size = in_bytes
        return self

    def setMaxReadCacheSize(self, in_bytes):
        self._max_read_cache_size = in_bytes
        return self

    def setMaxWriteTtl(self, seconds):
        self._max_write_ttl = seconds
        return self

    def setMaxReadTtl(self, seconds):
        self._max_read_ttl = seconds
        return self

    def set(self, inode, block_number, block, writed=False):
        """
        @type   inode: int
        @type   block_number: int
        @type   block: BytesIO
        @type   writed: bool
        """

        inode = str(inode)
        block_number = str(block_number)

        new = False
        if inode not in self._inodes:
            self._inodes[ inode ] = {}
            new = True

        inode_data = self._inodes[inode]

        if block_number not in inode_data:
            inode_data[ block_number ] = {
                "size" : 0
            }
            new = True

        block_data = inode_data[block_number]

        blockSize = len(block.getvalue())

        if not new:
            # Not new block
            oldBlockSize = block_data["size"]
            if writed:
                # If it now writed
                if not block_data["w"]:
                    # But not was
                    self._cur_read_cache_size -= oldBlockSize
                else:
                    self._cur_write_cache_size -= oldBlockSize
            else:
                self._cur_read_cache_size -= oldBlockSize

        if writed:
            self._cur_write_cache_size += blockSize
        else:
            self._cur_read_cache_size += blockSize

        block_data["time"] = time()
        block_data["block"] = block
        block_data["size"] = blockSize

        if writed:
            block_data["w"] = True
        else:
            block_data["w"] = block_data.get("w", False)

        return self

    def get(self, inode, block_number, default=None):

        inode = str(inode)
        block_number = str(block_number)

        inode_data = self._inodes.get(inode, {})

        block_data = inode_data.get(block_number, {})

        if not block_data:
            return default

        now = time()
        t = block_data["time"]
        if now - t > self._max_write_ttl:
            return default

        val = block_data.get("block", default)

        # update last request time
        block_data["time"] = time()

        return val

    def getCachedSize(self, writed=False):
        size = 0
        for inode in self._inodes.keys():
            for block_data in self._inodes[inode].values():
                if block_data["w"] != writed:
                    continue

                block_data["block"].seek(0, 2)
                size += block_data["block"].tell()
                block_data["block"].seek(0, 0)
        return size

    def isWritedCacheFull(self):
        filled = 100.0 * self._cur_write_cache_size / self._max_write_cache_size
        max_fill = 100 + self._max_size_trsh
        return filled > max_fill

    def isReadCacheFull(self):
        filled = 100.0 * self._cur_read_cache_size / self._max_read_cache_size
        max_fill = 100 + self._max_size_trsh
        return filled > max_fill

    def expired(self, writed=False):
        now = time()

        if writed:
            old_inodes = {}
        else:
            old_inodes = 0

        for inode in tuple(self._inodes.keys()):

            inode_data = self._inodes[inode]

            for bn in tuple(inode_data.keys()):
                block_data = inode_data[bn]

                t = block_data["time"]
                if block_data["w"] != writed:
                    continue

                if now - t > self._max_write_ttl:
                    if writed:
                        old_inode_data = old_inodes.get(inode, {})
                        old_inode_data[bn] = block_data.copy()
                        old_inodes[inode] = old_inode_data

                        self._cur_write_cache_size -= block_data["size"]
                    else:
                        old_inodes += 1

                        self._cur_read_cache_size -= block_data["size"]

                    del inode_data[bn]

            if not inode_data and inode in self._inodes:
                del self._inodes[inode]

        return old_inodes

    def expireByCount(self, writed=False):
        now = time()

        # 1. Fill heap

        heap = []

        inodesKeys = tuple(self._inodes.keys())

        for inode in inodesKeys:

            inode_data = self._inodes[inode]

            for bn in inode_data.keys():
                block_data = inode_data[bn]

                t = block_data["time"]
                if block_data["w"] != writed:
                    continue

                heapq.heappush(
                    heap, (
                        int((now - t)*10**6),
                        inode,
                        bn,
                        len(block_data["block"].getvalue())
                    )
                )

        # 2. Sort heap

        currentSize = self.getCachedSize(writed)

        if writed:
            maxSize = self._max_write_cache_size
        else:
            maxSize = self._max_read_cache_size

        needMaxSize = maxSize * (100 - self._max_size_trsh) / 100

        nget = int((currentSize - needMaxSize) / self._block_size)

        mostrecent = []
        while True:
            needSize = currentSize
            mostrecent = heapq.nsmallest(nget, heap)
            for dt, inode, bn, bsize in mostrecent:
                needSize -= bsize
            if needSize <= needMaxSize:
                break
            nget += 1

        # 3. Convert data

        heap_inodes = {}
        for dt, inode, bn, bsize in mostrecent:
            if inode not in heap_inodes:
                heap_inodes[ inode ] = ()
            heap_inodes[ inode ] += (bn,)

        del heap
        del mostrecent

        # 4. Expire cache, filter by new data

        if writed:
            old_inodes = {}
        else:
            old_inodes = 0

        for inode in inodesKeys:

            inode_data = self._inodes[inode]

            for bn in tuple(inode_data.keys()):

                bn = str(bn)

                block_data = inode_data[bn]

                if block_data["w"] != writed:
                    continue

                # Expire inodes that in heap or by block numbers
                if inode not in heap_inodes or\
                    bn not in heap_inodes[ inode ]:

                    if writed:
                        old_inode_data = old_inodes.get(inode, {})
                        old_inode_data[bn] = block_data.copy()
                        old_inodes[inode] = old_inode_data

                        self._cur_write_cache_size -= block_data["size"]
                    else:
                        old_inodes += 1

                        self._cur_read_cache_size -= block_data["size"]

                    del inode_data[bn]

            if not inode_data and inode in self._inodes:
                del self._inodes[inode]

        return old_inodes

    def clear(self):
        old_inodes = self._inodes.copy()
        self._inodes = {}
        return old_inodes
<|endoftext|>


<commit_before>#######################################################################
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
# Permission given to modify the code as long as you keep this        #
# declaration at the top                                              #
#######################################################################
from .atari_wrapper import *
import multiprocessing as mp
import sys
from .bench import Monitor
from ..utils import *
import uuid

class BaseTask:
    def set_monitor(self, env, log_dir):
        if log_dir is None:
            return env
        mkdir(log_dir)
        return Monitor(env, '%s/%s' % (log_dir, uuid.uuid4()))

    def reset(self):
        return self.env.reset()

    def step(self, action):
        next_state, reward, done, info = self.env.step(action)
        if done:
            next_state = self.env.reset()
        return next_state, reward, done, info

    def seed(self, random_seed):
        return self.env.seed(random_seed)

class ClassicalControl(BaseTask):
    def __init__(self, name='CartPole-v0', max_steps=200, log_dir=None):
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(self.name)
        self.env._max_episode_steps = max_steps
        self.action_dim = self.env.action_space.n
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

class PixelAtari(BaseTask):
    def __init__(self, name, seed=0, log_dir=None,
                 frame_skip=4, history_length=4, dataset=False):
        BaseTask.__init__(self)
        env = make_atari(name, frame_skip)
        env.seed(seed)
        if dataset:
            env = DatasetEnv(env)
            self.dataset_env = env
        env = self.set_monitor(env, log_dir)
        env = wrap_deepmind(env, history_length=history_length)
        self.env = env
        self.action_dim = self.env.action_space.n
        self.state_dim = self.env.observation_space.shape
        self.name = name

class RamAtari(BaseTask):
    def __init__(self, name, no_op, frame_skip, log_dir=None):
        BaseTask.__init__(self)
        self.name = name
        env = gym.make(name)
        assert 'NoFrameskip' in env.spec.id
        env = self.set_monitor(env, log_dir)
        env = EpisodicLifeEnv(env)
        env = NoopResetEnv(env, noop_max=no_op)
        env = SkipEnv(env, skip=frame_skip)
        if 'FIRE' in env.unwrapped.get_action_meanings():
            env = FireResetEnv(env)
        self.env = env
        self.action_dim = self.env.action_space.n
        self.state_dim = 128

class Pendulum(BaseTask):
    def __init__(self, log_dir=None):
        BaseTask.__init__(self)
        self.name = 'Pendulum-v0'
        self.env = gym.make(self.name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(2 * action, -2, 2))

class Box2DContinuous(BaseTask):
    def __init__(self, name, log_dir=None):
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(self.name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(action, -1, 1))

class Roboschool(BaseTask):
    def __init__(self, name, log_dir=None):
        import roboschool
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(self.name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(action, -1, 1))

class Bullet(BaseTask):
    def __init__(self, name, log_dir=None):
        import pybullet_envs
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(action, -1, 1))

class PixelBullet(BaseTask):
    def __init__(self, name, seed=0, log_dir=None, frame_skip=4, history_length=4):
        self.name = name
        env = gym.make(name)
        env.seed(seed)
        env = RenderEnv(env)
        env = self.set_monitor(env, log_dir)
        env = SkipEnv(env, skip=frame_skip)
        env = WarpFrame(env)
        env = WrapPyTorch(env)
        if history_length:
            env = StackFrame(env, history_length)
        self.action_dim = env.action_space.shape[0]
        self.state_dim = env.observation_space.shape
        self.env = env

class ProcessTask:
    def __init__(self, task_fn, log_dir=None):
        self.pipe, worker_pipe = mp.Pipe()
        self.worker = ProcessWrapper(worker_pipe, task_fn, log_dir)
        self.worker.start()
        self.pipe.send([ProcessWrapper.SPECS, None])
        self.state_dim, self.action_dim, self.name = self.pipe.recv()

    def step(self, action):
        self.pipe.send([ProcessWrapper.STEP, action])
        return self.pipe.recv()

    def reset(self):
        self.pipe.send([ProcessWrapper.RESET, None])
        return self.pipe.recv()

    def close(self):
        self.pipe.send([ProcessWrapper.EXIT, None])

class ProcessWrapper(mp.Process):
    STEP = 0
    RESET = 1
    EXIT = 2
    SPECS = 3
    def __init__(self, pipe, task_fn, log_dir):
        mp.Process.__init__(self)
        self.pipe = pipe
        self.task_fn = task_fn
        self.log_dir = log_dir

    def run(self):
        np.random.seed()
        seed = np.random.randint(0, sys.maxsize)
        task = self.task_fn(log_dir=self.log_dir)
        task.seed(seed)
        while True:
            op, data = self.pipe.recv()
            if op == self.STEP:
                self.pipe.send(task.step(data))
            elif op == self.RESET:
                self.pipe.send(task.reset())
            elif op == self.EXIT:
                self.pipe.close()
                return
            elif op == self.SPECS:
                self.pipe.send([task.state_dim, task.action_dim, task.name])
            else:
                raise Exception('Unknown command')

class ParallelizedTask:
    def __init__(self, task_fn, num_workers, log_dir=None, single_process=False):
        if single_process:
            self.tasks = [task_fn(log_dir=log_dir) for _ in range(num_workers)]
        else:
            self.tasks = [ProcessTask(task_fn, log_dir) for _ in range(num_workers)]
        self.state_dim = self.tasks[0].state_dim
        self.action_dim = self.tasks[0].action_dim
        self.name = self.tasks[0].name

    def step(self, actions):
        results = [task.step(action) for task, action in zip(self.tasks, actions)]
        results = map(lambda x: np.stack(x), zip(*results))
        return results

    def reset(self):
        results = [task.reset() for task in self.tasks]
        return np.stack(results)

    def close(self):
        for task in self.tasks: task.close()<commit_msg>Import pybullet_envs<commit_after>#######################################################################
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
# Permission given to modify the code as long as you keep this        #
# declaration at the top                                              #
#######################################################################
from .atari_wrapper import *
import multiprocessing as mp
import sys
from .bench import Monitor
from ..utils import *
import uuid

class BaseTask:
    def set_monitor(self, env, log_dir):
        if log_dir is None:
            return env
        mkdir(log_dir)
        return Monitor(env, '%s/%s' % (log_dir, uuid.uuid4()))

    def reset(self):
        return self.env.reset()

    def step(self, action):
        next_state, reward, done, info = self.env.step(action)
        if done:
            next_state = self.env.reset()
        return next_state, reward, done, info

    def seed(self, random_seed):
        return self.env.seed(random_seed)

class ClassicalControl(BaseTask):
    def __init__(self, name='CartPole-v0', max_steps=200, log_dir=None):
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(self.name)
        self.env._max_episode_steps = max_steps
        self.action_dim = self.env.action_space.n
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

class PixelAtari(BaseTask):
    def __init__(self, name, seed=0, log_dir=None,
                 frame_skip=4, history_length=4, dataset=False):
        BaseTask.__init__(self)
        env = make_atari(name, frame_skip)
        env.seed(seed)
        if dataset:
            env = DatasetEnv(env)
            self.dataset_env = env
        env = self.set_monitor(env, log_dir)
        env = wrap_deepmind(env, history_length=history_length)
        self.env = env
        self.action_dim = self.env.action_space.n
        self.state_dim = self.env.observation_space.shape
        self.name = name

class RamAtari(BaseTask):
    def __init__(self, name, no_op, frame_skip, log_dir=None):
        BaseTask.__init__(self)
        self.name = name
        env = gym.make(name)
        assert 'NoFrameskip' in env.spec.id
        env = self.set_monitor(env, log_dir)
        env = EpisodicLifeEnv(env)
        env = NoopResetEnv(env, noop_max=no_op)
        env = SkipEnv(env, skip=frame_skip)
        if 'FIRE' in env.unwrapped.get_action_meanings():
            env = FireResetEnv(env)
        self.env = env
        self.action_dim = self.env.action_space.n
        self.state_dim = 128

class Pendulum(BaseTask):
    def __init__(self, log_dir=None):
        BaseTask.__init__(self)
        self.name = 'Pendulum-v0'
        self.env = gym.make(self.name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(2 * action, -2, 2))

class Box2DContinuous(BaseTask):
    def __init__(self, name, log_dir=None):
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(self.name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(action, -1, 1))

class Roboschool(BaseTask):
    def __init__(self, name, log_dir=None):
        import roboschool
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(self.name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(action, -1, 1))

class Bullet(BaseTask):
    def __init__(self, name, log_dir=None):
        import pybullet_envs
        BaseTask.__init__(self)
        self.name = name
        self.env = gym.make(name)
        self.action_dim = self.env.action_space.shape[0]
        self.state_dim = self.env.observation_space.shape[0]
        self.env = self.set_monitor(self.env, log_dir)

    def step(self, action):
        return BaseTask.step(self, np.clip(action, -1, 1))

class PixelBullet(BaseTask):
    def __init__(self, name, seed=0, log_dir=None, frame_skip=4, history_length=4):
        import pybullet_envs
        self.name = name
        env = gym.make(name)
        env.seed(seed)
        env = RenderEnv(env)
        env = self.set_monitor(env, log_dir)
        env = SkipEnv(env, skip=frame_skip)
        env = WarpFrame(env)
        env = WrapPyTorch(env)
        if history_length:
            env = StackFrame(env, history_length)
        self.action_dim = env.action_space.shape[0]
        self.state_dim = env.observation_space.shape
        self.env = env

class ProcessTask:
    def __init__(self, task_fn, log_dir=None):
        self.pipe, worker_pipe = mp.Pipe()
        self.worker = ProcessWrapper(worker_pipe, task_fn, log_dir)
        self.worker.start()
        self.pipe.send([ProcessWrapper.SPECS, None])
        self.state_dim, self.action_dim, self.name = self.pipe.recv()

    def step(self, action):
        self.pipe.send([ProcessWrapper.STEP, action])
        return self.pipe.recv()

    def reset(self):
        self.pipe.send([ProcessWrapper.RESET, None])
        return self.pipe.recv()

    def close(self):
        self.pipe.send([ProcessWrapper.EXIT, None])

class ProcessWrapper(mp.Process):
    STEP = 0
    RESET = 1
    EXIT = 2
    SPECS = 3
    def __init__(self, pipe, task_fn, log_dir):
        mp.Process.__init__(self)
        self.pipe = pipe
        self.task_fn = task_fn
        self.log_dir = log_dir

    def run(self):
        np.random.seed()
        seed = np.random.randint(0, sys.maxsize)
        task = self.task_fn(log_dir=self.log_dir)
        task.seed(seed)
        while True:
            op, data = self.pipe.recv()
            if op == self.STEP:
                self.pipe.send(task.step(data))
            elif op == self.RESET:
                self.pipe.send(task.reset())
            elif op == self.EXIT:
                self.pipe.close()
                return
            elif op == self.SPECS:
                self.pipe.send([task.state_dim, task.action_dim, task.name])
            else:
                raise Exception('Unknown command')

class ParallelizedTask:
    def __init__(self, task_fn, num_workers, log_dir=None, single_process=False):
        if single_process:
            self.tasks = [task_fn(log_dir=log_dir) for _ in range(num_workers)]
        else:
            self.tasks = [ProcessTask(task_fn, log_dir) for _ in range(num_workers)]
        self.state_dim = self.tasks[0].state_dim
        self.action_dim = self.tasks[0].action_dim
        self.name = self.tasks[0].name

    def step(self, actions):
        results = [task.step(action) for task, action in zip(self.tasks, actions)]
        results = map(lambda x: np.stack(x), zip(*results))
        return results

    def reset(self):
        results = [task.reset() for task in self.tasks]
        return np.stack(results)

    def close(self):
        for task in self.tasks: task.close()<|endoftext|>


<commit_before>import tensorflow as tf

class Loss:
  """A loss function for use in training models."""

  def __call__(self, output, labels):
    """Compute the loss function.

    The inputs are tensors containing the model's outputs and the labels for a
    batch.  The return value should be a tensor of shape (batch_size) or
    (batch_size, tasks) containing the value of the loss function on each
    sample or sample/task.

    Parameters
    ----------
    output: tensor
      the output of the model
    labels: tensor
      the expected output

    Returns
    -------
    The value of the loss function on each sample or sample/task pair
    """
    raise NotImplementedError("Subclasses must implement this")

class L1Loss(Loss):
  """The absolute difference between the true and predicted values."""

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.abs(output - labels)

class L2Loss(Loss):
  """The squared difference between the true and predicted values."""

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.square(output - labels)

class HingeLoss(Loss):
  """The hinge loss function.

  The 'output' argument should contain logits, and all elements of 'labels'
  should equal 0 or 1.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.keras.losses.hinge(labels, output)

class BinaryCrossEntropy(Loss):
  """The cross entropy between pairs of probabilities.

  The arguments should each have shape (batch_size) or (batch_size, tasks) and
  contain probabilities.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.keras.losses.binary_crossentropy(labels, output)

class CategoricalCrossEntropy(Loss):
  """The cross entropy between two probability distributions.

  The arguments should each have shape (batch_size, classes) or
  (batch_size, tasks, classes), and represent a probability distribution over
  classes.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.keras.losses.categorical_crossentropy(labels, output)

class SigmoidCrossEntropy(Loss):
  """The cross entropy between pairs of probabilities.

  The arguments should each have shape (batch_size) or (batch_size, tasks).  The
  labels should be probabilities, while the outputs should be logits that are
  converted to probabilities using a sigmoid function.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.nn.sigmoid_cross_entropy_with_logits(labels, output)

class SoftmaxCrossEntropy(Loss):
  """The cross entropy between two probability distributions.

  The arguments should each have shape (batch_size, classes) or
  (batch_size, tasks, classes).  The labels should be probabilities, while the
  outputs should be logits that are converted to probabilities using a softmax
  function.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.nn.softmax_cross_entropy_with_logits(labels, output)

class SparseSoftmaxCrossEntropy(Loss):
  """The cross entropy between two probability distributions.

  The labels should have shape (batch_size) or (batch_size, tasks), and be
  integer class labels.  The outputs have shape (batch_size, classes) or
  (batch_size, tasks, classes) and be logits that are converted to probabilities
  using a softmax function.
  """

  def __call__(self, output, labels):
    labels = tf.cast(labels, tf.int32)
    return tf.nn.sparse_softmax_cross_entropy_with_logits(labels, output)

def _make_shapes_consistent(output, labels):
  """Try to make inputs have the same shape by adding dimensions of size 1."""
  shape1 = output.shape
  shape2 = labels.shape
  len1 = len(shape1)
  len2 = len(shape2)
  if len1 == len2:
    return (output, labels)
  if isinstance(shape1, tf.TensorShape):
    shape1 = tuple(shape1.as_list())
  if isinstance(shape2, tf.TensorShape):
    shape2 = tuple(shape2.as_list())
  if len1 > len2 and all(i == 1 for i in shape1[len2:]):
    for i in range(len1 - len2):
      labels = tf.expand_dims(labels, -1)
    return (output, labels)
  if len2 > len1 and all(i == 1 for i in shape2[len1:]):
    for i in range(len2 - len1):
      output = tf.expand_dims(output, -1)
    return (output, labels)
  raise ValueError("Incompatible shapes for outputs and labels: %s versus %s" %
                   (str(shape1), str(shape2)))
<commit_msg>Be more tolerant of dtypes when computing loss<commit_after>import tensorflow as tf

class Loss:
  """A loss function for use in training models."""

  def __call__(self, output, labels):
    """Compute the loss function.

    The inputs are tensors containing the model's outputs and the labels for a
    batch.  The return value should be a tensor of shape (batch_size) or
    (batch_size, tasks) containing the value of the loss function on each
    sample or sample/task.

    Parameters
    ----------
    output: tensor
      the output of the model
    labels: tensor
      the expected output

    Returns
    -------
    The value of the loss function on each sample or sample/task pair
    """
    raise NotImplementedError("Subclasses must implement this")

class L1Loss(Loss):
  """The absolute difference between the true and predicted values."""

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.abs(output - labels)

class L2Loss(Loss):
  """The squared difference between the true and predicted values."""

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.square(output - labels)

class HingeLoss(Loss):
  """The hinge loss function.

  The 'output' argument should contain logits, and all elements of 'labels'
  should equal 0 or 1.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    return tf.keras.losses.hinge(labels, output)

class BinaryCrossEntropy(Loss):
  """The cross entropy between pairs of probabilities.

  The arguments should each have shape (batch_size) or (batch_size, tasks) and
  contain probabilities.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    output, labels = _ensure_float(output, labels)
    return tf.keras.losses.binary_crossentropy(labels, output)

class CategoricalCrossEntropy(Loss):
  """The cross entropy between two probability distributions.

  The arguments should each have shape (batch_size, classes) or
  (batch_size, tasks, classes), and represent a probability distribution over
  classes.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    output, labels = _ensure_float(output, labels)
    return tf.keras.losses.categorical_crossentropy(labels, output)

class SigmoidCrossEntropy(Loss):
  """The cross entropy between pairs of probabilities.

  The arguments should each have shape (batch_size) or (batch_size, tasks).  The
  labels should be probabilities, while the outputs should be logits that are
  converted to probabilities using a sigmoid function.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    output, labels = _ensure_float(output, labels)
    return tf.nn.sigmoid_cross_entropy_with_logits(labels, output)

class SoftmaxCrossEntropy(Loss):
  """The cross entropy between two probability distributions.

  The arguments should each have shape (batch_size, classes) or
  (batch_size, tasks, classes).  The labels should be probabilities, while the
  outputs should be logits that are converted to probabilities using a softmax
  function.
  """

  def __call__(self, output, labels):
    output, labels = _make_shapes_consistent(output, labels)
    output, labels = _ensure_float(output, labels)
    return tf.nn.softmax_cross_entropy_with_logits(labels, output)

class SparseSoftmaxCrossEntropy(Loss):
  """The cross entropy between two probability distributions.

  The labels should have shape (batch_size) or (batch_size, tasks), and be
  integer class labels.  The outputs have shape (batch_size, classes) or
  (batch_size, tasks, classes) and be logits that are converted to probabilities
  using a softmax function.
  """

  def __call__(self, output, labels):
    labels = tf.cast(labels, tf.int32)
    return tf.nn.sparse_softmax_cross_entropy_with_logits(labels, output)

def _make_shapes_consistent(output, labels):
  """Try to make inputs have the same shape by adding dimensions of size 1."""
  shape1 = output.shape
  shape2 = labels.shape
  len1 = len(shape1)
  len2 = len(shape2)
  if len1 == len2:
    return (output, labels)
  if isinstance(shape1, tf.TensorShape):
    shape1 = tuple(shape1.as_list())
  if isinstance(shape2, tf.TensorShape):
    shape2 = tuple(shape2.as_list())
  if len1 > len2 and all(i == 1 for i in shape1[len2:]):
    for i in range(len1 - len2):
      labels = tf.expand_dims(labels, -1)
    return (output, labels)
  if len2 > len1 and all(i == 1 for i in shape2[len1:]):
    for i in range(len2 - len1):
      output = tf.expand_dims(output, -1)
    return (output, labels)
  raise ValueError("Incompatible shapes for outputs and labels: %s versus %s" %
                   (str(shape1), str(shape2)))

def _ensure_float(output, labels):
  """Make sure the outputs and labels are both floating point types."""
  if output.dtype not in (tf.float32, tf.float64):
    output = tf.cast(output, tf.float32)
  if labels.dtype not in (tf.float32, tf.float64):
    labels = tf.cast(labels, tf.float32)
  return (output, labels)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generator for DNS based network definitions."""

__author__ = 'msu@google.com (Martin Suess)'

import logging
import socket

from third_party import ipaddr
import generator

class DnsGeneratorError(Exception):
  """Exception to use when DnsGenerator fails."""

class DnsGenerator(generator.Generator):
  """Generator implementation for network definitions based on DNS."""

  SUPPORTED_TYPES = ['A', 'AAAA']

  def GenerateDefinition(self, config, unused_global_config):
    """Generates a list of all nodes in a network definition.

    This method basically processes all the configuration which is
    hierarchically below "networks" in the "definitions" section in the
    configuration file to generate a list of all nodes in that definition.

    Args:
      config: YAML configuration structure (dictionaries, lists and strings)
        representing the "networks" section in "definitions" of the
        configuration file.
      unused_global_config: YAML configuration structure (dictionaries, lists
        and strings) representing the "global" section of the configuration
        file.

    Returns:
      Tuples of IPNetwork objects and string comments representing all the nodes
      in one definition.

    Raises:
      DefinateConfigError: The configuration is not well formed.
      DnsGeneratorError: There is a problem generating the output.
    """
    nodes = []
    yaml_structure = {
        'names': ['str'],
        'types': ['str'],
        }
    for network in config:
      self._yaml_validator.CheckConfiguration(network, yaml_structure)
      for typ in network['types']:
        if typ not in self.SUPPORTED_TYPES:
          raise DnsGeneratorError('Unsupported DNS type found: %s' % typ)
      for name in network['names']:
        addr_list = socket.getaddrinfo(name, None)
        for family, _, _, _, sockaddr in addr_list:
          ip_addr = None
          if family == socket.AF_INET and 'A' in network['types']:
            # sockaddr = (address, port)
            ip_addr = ipaddr.IPv4Network(sockaddr[0])
          elif family == socket.AF_INET6 and 'AAAA' in network['types']:
            # sockaddr = (address, port, flow info, scope id)
            ip_addr = ipaddr.IPv6Network(sockaddr[0])
          else:
            logging.debug('Skipping unknown AF \'%d\' for: %s', family, name)
          if ip_addr:
            nodes.append((ip_addr, name))
    return nodes
<commit_msg>Make definate dns generator error message more informative (return hostname that fails to resolve)<commit_after>#!/usr/bin/python
#
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generator for DNS based network definitions."""

__author__ = 'msu@google.com (Martin Suess)'

import logging
import socket

from third_party import ipaddr
import generator

class DnsGeneratorError(Exception):
  """Exception to use when DnsGenerator fails."""

class DnsGenerator(generator.Generator):
  """Generator implementation for network definitions based on DNS."""

  SUPPORTED_TYPES = ['A', 'AAAA']

  def GenerateDefinition(self, config, unused_global_config):
    """Generates a list of all nodes in a network definition.

    This method basically processes all the configuration which is
    hierarchically below "networks" in the "definitions" section in the
    configuration file to generate a list of all nodes in that definition.

    Args:
      config: YAML configuration structure (dictionaries, lists and strings)
        representing the "networks" section in "definitions" of the
        configuration file.
      unused_global_config: YAML configuration structure (dictionaries, lists
        and strings) representing the "global" section of the configuration
        file.

    Returns:
      Tuples of IPNetwork objects and string comments representing all the nodes
      in one definition.

    Raises:
      DefinateConfigError: The configuration is not well formed.
      DnsGeneratorError: There is a problem generating the output.
    """
    nodes = []
    yaml_structure = {
        'names': ['str'],
        'types': ['str'],
        }
    for network in config:
      self._yaml_validator.CheckConfiguration(network, yaml_structure)
      for typ in network['types']:
        if typ not in self.SUPPORTED_TYPES:
          raise DnsGeneratorError('Unsupported DNS type found: %s' % typ)
      for name in network['names']:
        try:
          addr_list = socket.getaddrinfo(name, None)
        except socket.gaierror:
          raise DnsGeneratorError('Hostname not found: %s' % name)
        for family, _, _, _, sockaddr in addr_list:
          ip_addr = None
          if family == socket.AF_INET and 'A' in network['types']:
            # sockaddr = (address, port)
            ip_addr = ipaddr.IPv4Network(sockaddr[0])
          elif family == socket.AF_INET6 and 'AAAA' in network['types']:
            # sockaddr = (address, port, flow info, scope id)
            ip_addr = ipaddr.IPv6Network(sockaddr[0])
          else:
            logging.debug('Skipping unknown AF \'%d\' for: %s', family, name)
          if ip_addr:
            nodes.append((ip_addr, name))
    return nodes
<|endoftext|>


<commit_before># Copyright 2013 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for the GCI Organization Admins to propose winners for their orgs."""

from google.appengine.ext import db

from django import forms
from django.utils.translation import ugettext

from soc.logic import exceptions

from soc.modules.gci.views.base import GCIRequestHandler

from soc.views.helper import lists
from soc.views.helper import url_patterns

from soc.modules.gci.logic import organization as org_logic
from soc.modules.gci.logic import org_score as org_score_logic
from soc.modules.gci.models import organization as organization_model
from soc.modules.gci.models import profile as profile_model
from soc.modules.gci.models import profile
from soc.modules.gci.templates import org_list
from soc.modules.gci.views import forms as gci_forms
from soc.modules.gci.views.helper import url_patterns as gci_url_patterns
from soc.modules.gci.views.helper import url_names

DEF_WINNER_MORE_THAN_ONCE_ERROR = ugettext(
    'You cannot set the same winner more than once')

class ProposeWinnersForm(gci_forms.GCIModelForm):
  """Django form to propose the grand prize winners."""

  EMPTY_CHOICE = 'EMPTY_CHOICE'

  class Meta:
    model = None

  def __init__(self, request_data, *args):
    super(ProposeWinnersForm, self).__init__(*args)
    self.request_data = request_data

    choices = [(ProposeWinnersForm.EMPTY_CHOICE, '---')]

    possible_winners = org_score_logic.getPossibleWinners(
        request_data.organization)

    for possible_winner in possible_winners:
      choices.append(self._getChoiceOption(possible_winner))

    self.fields.get('first_proposed_winner').choices = choices
    self.fields.get('second_proposed_winner').choices = choices
    self.fields.get('backup_proposed_winner').choices = choices

    # check if at least one grand prize winner is already set
    if len(self.request_data.organization.proposed_winners) > 0:
      self.fields.get('first_proposed_winner').initial = \
          self.request_data.organization.proposed_winners[0].name()

      # check if both grand prize winners are already set
      if len(self.request_data.organization.proposed_winners) > 1:
         self.fields.get('second_proposed_winner').initial = \
             self.request_data.organization.proposed_winners[1].name()

    # check if backup winner is already set
    if self.request_data.organization.backup_winner:
      self.fields.get('backup_proposed_winner').initial = (
          organization_model.GCIOrganization.backup_winner.
              get_value_for_datastore(self.request_data.organization).name())

  first_proposed_winner = forms.ChoiceField(label='First Grand Prize Winner')
  second_proposed_winner = forms.ChoiceField(label='Second Grand Prize Winner')
  backup_proposed_winner = forms.ChoiceField(label='Backup Grand Prize Winner')

  def clean(self):
    first_proposed_winner = self.cleaned_data.get(
        'first_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)
    second_proposed_winner = self.cleaned_data.get(
        'second_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)
    backup_proposed_winner = self.cleaned_data.get(
        'backup_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    # TODO (daniel): the logic below should be simplified
    key_names = set([
        first_proposed_winner, second_proposed_winner, backup_proposed_winner])

    if len(key_names) == 3:
      # there are three different key_names, so everything is OK
      pass
    elif len(key_names) == 2:
      # it is not OK, because at least one of the fields is duplicated
      self._errors['__all__'] = DEF_WINNER_MORE_THAN_ONCE_ERROR
    elif list(key_names)[0] != ProposeWinnersForm.EMPTY_CHOICE:
      # it is not OK, when there is one choice which is not empty
      self._errors['__all__'] = DEF_WINNER_MORE_THAN_ONCE_ERROR

  def _getChoiceOption(self, student):
    return (student.key().name(), self._formatPossibleWinner(student))

  def _formatPossibleWinner(self, student):
    return '%s' % student.name()

class ProposeWinnersPage(GCIRequestHandler):
  """Page to propose winners by organization admins"""

  def templatePath(self):
    return 'v2/modules/gci/propose_winners/base.html'

  def djangoURLPatterns(self):
    return [
        gci_url_patterns.url(r'propose_winners/%s$' % url_patterns.ORG, self,
            name=url_names.GCI_ORG_PROPOSE_WINNERS),
    ]

  def checkAccess(self):
    self.check.isOrgAdmin()
    if not self.data.timeline.allReviewsStopped():
      raise exceptions.AccessViolation(
          'This page may be accessed when the review period is over')

  def context(self):
    form = ProposeWinnersForm(
        self.data, self.data.POST or None)
    context = {
        'page_name': 'Propose winners for %s' % (self.data.organization.name),
        'forms': [form]
    }

    return context

  def post(self):
    """Handles POST requests."""
    form = ProposeWinnersForm(self.data, self.data.POST)

    if not form.is_valid():
      # TODO(nathaniel): problematic self-call.
      return self.get()

    first_key_name = self.data.POST.get(
        'first_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    second_key_name = self.data.POST.get(
        'second_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    backup_key_name = self.data.POST.get(
        'backup_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    proposed_winners = self._getProposedWinnersList(
        first_key_name, second_key_name)

    backup_winner = self._getBackupWinner(backup_key_name)

    def txn():
      organization = organization_model.GCIOrganization.get(
          self.data.organization.key())
      organization.proposed_winners = proposed_winners
      organization.backup_winner = backup_winner
      organization.put()

    db.run_in_transaction(txn)

    self.redirect.organization()
    return self.redirect.to(url_names.GCI_ORG_PROPOSE_WINNERS)

  def _getBackupWinner(self, backup_key_name):
    """Returns the GCIProfile entity belonging to the backup winner chosen
    by the organization.

    Args:
      backup_key_name: the key name proposed by the organization

    Returns:
      the GCIProfile entity associated with the specified argument or None
      if it does not point to any existing profile
    """
    parent_key = self._getUserKey(backup_key_name)

    profile = None
    if parent_key:
      profile = profile_model.GCIProfile.get_by_key_name(
          backup_key_name, parent_key)

    return profile

  def _getProposedWinnersList(self, first_key_name, second_key_name):
    """Returns the list which contains the keys of the GCIProfile entities
    belonging to students proposed by the organization.

    Args:
      first_key_name: the first key name proposed by the organization.
      second_key_name: the second key name proposed by the organization.

    Returns:
      a list with the keys of GCIProfile entity that correspond to
      the specified arguments.
    """
    proposed_winners = []

    parent_key = self._getUserKey(first_key_name)
    if parent_key:
      profile = profile_model.GCIProfile.get_by_key_name(
          first_key_name, parent_key)
      if profile:
        proposed_winners.append(profile.key())

    parent_key = self._getUserKey(second_key_name)
    if parent_key:
      profile = profile_model.GCIProfile.get_by_key_name(
          second_key_name, parent_key)
      if profile:
        proposed_winners.append(profile.key())

    return proposed_winners

  # TODO(daniel): discuss if this should go to the logic?
  def _getUserKey(self, key_name):
    """Returns the db.Key instance of the User entity that belong to a parent
    of the specified profile's key name.

    Args:
      key_name: the specified Profile's key name

    Returns:
      db.Key instance or None if the key_name is invalid
    """
    parts = key_name.split('/')
    if len(parts) != 3:
      return None

    # TODO(daniel): check if the element has correct format to be a link_id
    return db.Key.from_path('User', parts[2])

class OrganizationsForProposeWinnersList(org_list.OrgList):
  """Lists all organizations for which the current user may propose the Grand
  Prize Winner and the row action takes their to ProposeWinnersPage for
  the corresponding organization.
  """

  def _getDescription(self):
    return ugettext('Choose an organization for which to propose the '
        'Grand Prize Winners.')

  def _getRedirect(self):
    def redirect(e, *args):
      # TODO(nathaniel): make this .organization call unnecessary.
      self.data.redirect.organization(organization=e)

      return self.data.redirect.urlOf(url_names.GCI_ORG_PROPOSE_WINNERS)
    return redirect

  def _getListConfig(self):
    """Returns ListConfiguration object for the list.
    """
    r = self.data.redirect

    list_config = lists.ListConfiguration()
    list_config.addColumn('name', 'Name',
        lambda e, *args: e.name.strip())
    list_config.addSimpleColumn('link_id', 'Link ID', hidden=True)
    list_config.setRowAction(self._getRedirect())
    return list_config

  def _getQuery(self):
    """Returns Query object to fetch entities for the list.
    """
    return org_logic.queryForOrgAdminAndStatus(
        self.data.profile, ['new', 'active'])

class ChooseOrganizationForProposeWinnersPage(GCIRequestHandler):
  """View with a list of organizations. When a user clicks on one of them,
  he or she is moved to the propose winner page for this organization.
  """

  def templatePath(self):
    return 'v2/modules/gci/org_list/base.html'

  def djangoURLPatterns(self):
    return [
        gci_url_patterns.url(
            r'org_choose_for_propose_winners/%s$' % url_patterns.PROGRAM, self,
            name=url_names.GCI_ORG_CHOOSE_FOR_PROPOSE_WINNNERS),
    ]

  def checkAccess(self):
    pass

  def jsonContext(self):
    list_content = OrganizationsForProposeWinnersList(
        self.request, self.data).getListData()

    if not list_content:
      raise exceptions.AccessViolation(
          'You do not have access to this data')
    return list_content.content()

  def context(self):
    return {
        'page_name': "Choose an organization for which to display scores.",
        'org_list': OrganizationsForProposeWinnersList(
            self.request, self.data),
        #'program_select': ProgramSelect(self.data, 'gci_accepted_orgs'),
    }
<commit_msg>Propose winners form recognizes students by the string repr of keys.<commit_after># Copyright 2013 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for the GCI Organization Admins to propose winners for their orgs."""

from google.appengine.ext import db

from django import forms
from django.utils.translation import ugettext

from soc.logic import exceptions

from soc.modules.gci.views.base import GCIRequestHandler

from soc.views.helper import lists
from soc.views.helper import url_patterns

from soc.modules.gci.logic import organization as org_logic
from soc.modules.gci.logic import org_score as org_score_logic
from soc.modules.gci.models import organization as organization_model
from soc.modules.gci.models import profile as profile_model
from soc.modules.gci.models import profile
from soc.modules.gci.templates import org_list
from soc.modules.gci.views import forms as gci_forms
from soc.modules.gci.views.helper import url_patterns as gci_url_patterns
from soc.modules.gci.views.helper import url_names

DEF_WINNER_MORE_THAN_ONCE_ERROR = ugettext(
    'You cannot set the same winner more than once')

class ProposeWinnersForm(gci_forms.GCIModelForm):
  """Django form to propose the grand prize winners."""

  EMPTY_CHOICE = 'EMPTY_CHOICE'

  class Meta:
    model = None

  def __init__(self, request_data, *args):
    super(ProposeWinnersForm, self).__init__(*args)
    self.request_data = request_data

    choices = [(ProposeWinnersForm.EMPTY_CHOICE, '---')]

    possible_winners = org_score_logic.getPossibleWinners(
        request_data.organization)

    for possible_winner in possible_winners:
      choices.append(self._getChoiceOption(possible_winner))

    self.fields.get('first_proposed_winner').choices = choices
    self.fields.get('second_proposed_winner').choices = choices
    self.fields.get('backup_proposed_winner').choices = choices

    # check if at least one grand prize winner is already set
    if len(self.request_data.organization.proposed_winners) > 0:
      self.fields.get('first_proposed_winner').initial = \
          str(self.request_data.organization.proposed_winners[0])

      # check if both grand prize winners are already set
      if len(self.request_data.organization.proposed_winners) > 1:
         self.fields.get('second_proposed_winner').initial = \
             str(self.request_data.organization.proposed_winners[1])

    # check if backup winner is already set
    if self.request_data.organization.backup_winner:
      self.fields.get('backup_proposed_winner').initial = str(
          organization_model.GCIOrganization.backup_winner.
              get_value_for_datastore(self.request_data.organization))

  first_proposed_winner = forms.ChoiceField(label='First Grand Prize Winner')
  second_proposed_winner = forms.ChoiceField(label='Second Grand Prize Winner')
  backup_proposed_winner = forms.ChoiceField(label='Backup Grand Prize Winner')

  def clean(self):
    first_proposed_winner = self.cleaned_data.get(
        'first_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)
    second_proposed_winner = self.cleaned_data.get(
        'second_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)
    backup_proposed_winner = self.cleaned_data.get(
        'backup_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    # TODO (daniel): the logic below should be simplified
    key_names = set([
        first_proposed_winner, second_proposed_winner, backup_proposed_winner])

    if len(key_names) == 3:
      # there are three different key_names, so everything is OK
      pass
    elif len(key_names) == 2:
      # it is not OK, because at least one of the fields is duplicated
      self._errors['__all__'] = DEF_WINNER_MORE_THAN_ONCE_ERROR
    elif list(key_names)[0] != ProposeWinnersForm.EMPTY_CHOICE:
      # it is not OK, when there is one choice which is not empty
      self._errors['__all__'] = DEF_WINNER_MORE_THAN_ONCE_ERROR

  def _getChoiceOption(self, student):
    return (str(student.key()), self._formatPossibleWinner(student))

  def _formatPossibleWinner(self, student):
    return '%s' % student.name()

class ProposeWinnersPage(GCIRequestHandler):
  """Page to propose winners by organization admins"""

  def templatePath(self):
    return 'v2/modules/gci/propose_winners/base.html'

  def djangoURLPatterns(self):
    return [
        gci_url_patterns.url(r'propose_winners/%s$' % url_patterns.ORG, self,
            name=url_names.GCI_ORG_PROPOSE_WINNERS),
    ]

  def checkAccess(self):
    self.check.isOrgAdmin()
    if not self.data.timeline.allReviewsStopped():
      raise exceptions.AccessViolation(
          'This page may be accessed when the review period is over')

  def context(self):
    form = ProposeWinnersForm(
        self.data, self.data.POST or None)
    context = {
        'page_name': 'Propose winners for %s' % (self.data.organization.name),
        'forms': [form]
    }

    return context

  def post(self):
    """Handles POST requests."""
    form = ProposeWinnersForm(self.data, self.data.POST)

    if not form.is_valid():
      # TODO(nathaniel): problematic self-call.
      return self.get()

    first_key_str = self.data.POST.get(
        'first_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    second_key_str = self.data.POST.get(
        'second_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    backup_key_str = self.data.POST.get(
        'backup_proposed_winner', ProposeWinnersForm.EMPTY_CHOICE)

    proposed_winners = self._getProposedWinnersList(
        first_key_str, second_key_str)

    backup_winner = self._getBackupWinner(backup_key_str)

    def txn():
      organization = organization_model.GCIOrganization.get(
          self.data.organization.key())
      organization.proposed_winners = proposed_winners
      organization.backup_winner = backup_winner
      organization.put()

    db.run_in_transaction(txn)

    self.redirect.organization()
    return self.redirect.to(url_names.GCI_ORG_PROPOSE_WINNERS)

  def _getProfileByKeyStr(self, key_str):
    """Returns the GCIProfile entity based on the specified string
    representation of db.Key.
    """
    try:
      key = db.Key(key_str)
    except db.BadKeyError:
      return None

    return profile_model.GCIProfile.get(key)

  def _getBackupWinner(self, backup_key_str):
    """Returns the GCIProfile entity belonging to the backup winner chosen
    by the organization.

    Args:
      backup_key_str: the string representation of the key associated with
          the profile proposed by the organization.

    Returns:
      the GCIProfile entity associated with the specified argument or None
      if it does not point to any existing profile
    """
    return self._getProfileByKeyStr(backup_key_str)

  def _getProposedWinnersList(self, first_key_str, second_key_str):
    """Returns the list which contains the keys of the GCIProfile entities
    belonging to students proposed by the organization.

    Args:
      first_key_str: the string representation of the first key associated
          with the profile proposed by the organization.
      second_key_str: the string representation of the second key associated
          with the profile proposed by the organization.

    Returns:
      a list with the keys of GCIProfile entity that correspond to
      the specified arguments.
    """
    proposed_winners = []

    profile = self._getProfileByKeyStr(first_key_str)
    if profile:
      proposed_winners.append(profile.key())

    profile = self._getProfileByKeyStr(second_key_str)
    if profile:
      proposed_winners.append(profile.key())

    return proposed_winners

class OrganizationsForProposeWinnersList(org_list.OrgList):
  """Lists all organizations for which the current user may propose the Grand
  Prize Winner and the row action takes their to ProposeWinnersPage for
  the corresponding organization.
  """

  def _getDescription(self):
    return ugettext('Choose an organization for which to propose the '
        'Grand Prize Winners.')

  def _getRedirect(self):
    def redirect(e, *args):
      # TODO(nathaniel): make this .organization call unnecessary.
      self.data.redirect.organization(organization=e)

      return self.data.redirect.urlOf(url_names.GCI_ORG_PROPOSE_WINNERS)
    return redirect

  def _getListConfig(self):
    """Returns ListConfiguration object for the list.
    """
    r = self.data.redirect

    list_config = lists.ListConfiguration()
    list_config.addColumn('name', 'Name',
        lambda e, *args: e.name.strip())
    list_config.addSimpleColumn('link_id', 'Link ID', hidden=True)
    list_config.setRowAction(self._getRedirect())
    return list_config

  def _getQuery(self):
    """Returns Query object to fetch entities for the list.
    """
    return org_logic.queryForOrgAdminAndStatus(
        self.data.profile, ['new', 'active'])

class ChooseOrganizationForProposeWinnersPage(GCIRequestHandler):
  """View with a list of organizations. When a user clicks on one of them,
  he or she is moved to the propose winner page for this organization.
  """

  def templatePath(self):
    return 'v2/modules/gci/org_list/base.html'

  def djangoURLPatterns(self):
    return [
        gci_url_patterns.url(
            r'org_choose_for_propose_winners/%s$' % url_patterns.PROGRAM, self,
            name=url_names.GCI_ORG_CHOOSE_FOR_PROPOSE_WINNNERS),
    ]

  def checkAccess(self):
    pass

  def jsonContext(self):
    list_content = OrganizationsForProposeWinnersList(
        self.request, self.data).getListData()

    if not list_content:
      raise exceptions.AccessViolation(
          'You do not have access to this data')
    return list_content.content()

  def context(self):
    return {
        'page_name': "Choose an organization for which to display scores.",
        'org_list': OrganizationsForProposeWinnersList(
            self.request, self.data),
        #'program_select': ProgramSelect(self.data, 'gci_accepted_orgs'),
    }
<|endoftext|>


<commit_before>import math

import torch.nn.functional as F
from torch import nn, torch

def _make_conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1,
               bias=False):
    conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size,
                     stride=stride, padding=padding, bias=bias)
    bn = nn.BatchNorm2d(out_planes)
    relu = nn.ReLU(inplace=True)
    return nn.Sequential(conv, bn, relu)

class Block(nn.Module):
    def __init__(self, in_planes, out_planes, pool_method, stride):
        super(Block, self).__init__()
        self.branches = nn.ModuleList([
            nn.Sequential(
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0),
                _make_conv(out_planes, out_planes, stride=stride)
            ),
            nn.Sequential(
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0),
                _make_conv(out_planes, out_planes),
                _make_conv(out_planes, out_planes, stride=stride))
        ])

        if pool_method == 'Avg':
            assert stride == 1
            self.branches.append(
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0))
            self.branches.append(nn.Sequential(
                nn.AvgPool2d(kernel_size=3, stride=1, padding=1),
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0)))
        else:
            self.branches.append(
                nn.MaxPool2d(kernel_size=3, stride=stride, padding=1))

    def forward(self, x):
        return torch.cat([b(x) for b in self.branches], 1)

class Inception(nn.Module):
    def __init__(self, num_classes=0, num_features=256, norm=False, dropout=0):
        super(Inception, self).__init__()
        self.num_classes = num_classes
        self.num_features = num_features
        self.norm = norm
        self.dropout = dropout

        self.conv1 = _make_conv(3, 32)
        self.conv2 = _make_conv(32, 32)
        self.conv3 = _make_conv(32, 32)
        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2, padding=1)
        self.in_planes = 32
        self.inception4a = self._make_inception(64, 'Avg', 1)
        self.inception4b = self._make_inception(64, 'Max', 2)
        self.inception5a = self._make_inception(128, 'Avg', 1)
        self.inception5b = self._make_inception(128, 'Max', 2)
        self.inception6a = self._make_inception(256, 'Avg', 1)
        self.inception6b = self._make_inception(256, 'Max', 2)
        self.global_pool = nn.AvgPool2d(kernel_size=(9, 4))
        self.feat = nn.Linear(self.in_planes, self.num_features)
        if self.dropout > 0:
            self.drop = nn.Dropout(self.dropout)
        if self.num_classes > 0:
            self.classifier = nn.Linear(self.num_features, self.num_classes)

        self._reset_params()

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.pool3(x)
        x = self.inception4a(x)
        x = self.inception4b(x)
        x = self.inception5a(x)
        x = self.inception5b(x)
        x = self.inception6a(x)
        x = self.inception6b(x)
        x = self.global_pool(x)
        x = x.view(x.size(0), -1)
        x = self.feat(x)
        if self.norm:
            x = x / x.norm(2, 1).expand_as(x)
        else:
            x = F.relu(x)
        if self.dropout > 0:
            x = self.drop(x)
        if self.num_classes > 0:
            x = self.classifier(x)
        return x

    def _make_inception(self, out_planes, pool_method, stride):
        block = Block(self.in_planes, out_planes, pool_method, stride)
        self.in_planes = (out_planes * 4 if pool_method == 'Avg' else
                          out_planes * 2 + self.in_planes)
        return block

    def _reset_params(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.weight.data.normal_(0, 0.001)
                if m.bias is not None:
                    m.bias.data.zero_()
<commit_msg>Add BN back<commit_after>import math

import torch.nn.functional as F
from torch import nn, torch

def _make_conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1,
               bias=False):
    conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size,
                     stride=stride, padding=padding, bias=bias)
    bn = nn.BatchNorm2d(out_planes)
    relu = nn.ReLU(inplace=True)
    return nn.Sequential(conv, bn, relu)

class Block(nn.Module):
    def __init__(self, in_planes, out_planes, pool_method, stride):
        super(Block, self).__init__()
        self.branches = nn.ModuleList([
            nn.Sequential(
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0),
                _make_conv(out_planes, out_planes, stride=stride)
            ),
            nn.Sequential(
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0),
                _make_conv(out_planes, out_planes),
                _make_conv(out_planes, out_planes, stride=stride))
        ])

        if pool_method == 'Avg':
            assert stride == 1
            self.branches.append(
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0))
            self.branches.append(nn.Sequential(
                nn.AvgPool2d(kernel_size=3, stride=1, padding=1),
                _make_conv(in_planes, out_planes, kernel_size=1, padding=0)))
        else:
            self.branches.append(
                nn.MaxPool2d(kernel_size=3, stride=stride, padding=1))

    def forward(self, x):
        return torch.cat([b(x) for b in self.branches], 1)

class Inception(nn.Module):
    def __init__(self, num_classes=0, num_features=256, norm=False, dropout=0):
        super(Inception, self).__init__()
        self.num_classes = num_classes
        self.num_features = num_features
        self.norm = norm
        self.dropout = dropout

        self.conv1 = _make_conv(3, 32)
        self.conv2 = _make_conv(32, 32)
        self.conv3 = _make_conv(32, 32)
        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2, padding=1)
        self.in_planes = 32
        self.inception4a = self._make_inception(64, 'Avg', 1)
        self.inception4b = self._make_inception(64, 'Max', 2)
        self.inception5a = self._make_inception(128, 'Avg', 1)
        self.inception5b = self._make_inception(128, 'Max', 2)
        self.inception6a = self._make_inception(256, 'Avg', 1)
        self.inception6b = self._make_inception(256, 'Max', 2)
        self.global_pool = nn.AvgPool2d(kernel_size=(9, 4))
        self.feat = nn.Linear(self.in_planes, self.num_features)
        self.feat_bn = nn.BatchNorm1d(self.num_features)
        if self.dropout > 0:
            self.drop = nn.Dropout(self.dropout)
        if self.num_classes > 0:
            self.classifier = nn.Linear(self.num_features, self.num_classes)

        self._reset_params()

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.pool3(x)
        x = self.inception4a(x)
        x = self.inception4b(x)
        x = self.inception5a(x)
        x = self.inception5b(x)
        x = self.inception6a(x)
        x = self.inception6b(x)
        x = self.global_pool(x)
        x = x.view(x.size(0), -1)
        x = self.feat(x)
        x = self.feat_bn(x)
        if self.norm:
            x = x / x.norm(2, 1).expand_as(x)
        else:
            x = F.relu(x)
        if self.dropout > 0:
            x = self.drop(x)
        if self.num_classes > 0:
            x = self.classifier(x)
        return x

    def _make_inception(self, out_planes, pool_method, stride):
        block = Block(self.in_planes, out_planes, pool_method, stride)
        self.in_planes = (out_planes * 4 if pool_method == 'Avg' else
                          out_planes * 2 + self.in_planes)
        return block

    def _reset_params(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.weight.data.normal_(0, 0.001)
                if m.bias is not None:
                    m.bias.data.zero_()
<|endoftext|>