<commit_before># Copyright (C) 2005-2006 Jelmer Vernooij <jelmer@samba.org>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""Access to stored Subversion basis trees."""

from bzrlib.inventory import Inventory

from bzrlib import osutils, urlutils
from bzrlib.trace import mutter
from bzrlib.revisiontree import RevisionTree

import os
import md5
from cStringIO import StringIO
import urllib

import constants
import core, wc

class SvnRevisionTree(RevisionTree):
    """A tree that existed in a historical Subversion revision."""
    def __init__(self, repository, revision_id):
        self._repository = repository
        self._revision_id = revision_id
        (self.branch_path, self.revnum, mapping) = repository.lookup_revision_id(revision_id)
        self._inventory = Inventory()
        self.id_map = repository.get_fileid_map(self.revnum, self.branch_path,
                                                mapping)
        editor = TreeBuildEditor(self)
        self.file_data = {}
        root_repos = repository.transport.get_svn_repos_root()
        reporter = repository.transport.do_switch(
                self.revnum, True,
                urlutils.join(root_repos, self.branch_path), editor)
        reporter.set_path("", 0, True, None)
        reporter.finish_report()

    def get_file_lines(self, file_id):
        return osutils.split_lines(self.file_data[file_id])

class TreeBuildEditor:
    """Builds a tree given Subversion tree transform calls."""
    def __init__(self, tree):
        self.tree = tree
        self.repository = tree._repository
        self.last_revnum = {}
        self.dir_revnum = {}
        self.dir_ignores = {}

    def set_target_revision(self, revnum):
        self.revnum = revnum

    def open_root(self, revnum):
        file_id, revision_id = self.tree.id_map[""]
        ie = self.tree._inventory.add_path("", 'directory', file_id)
        ie.revision = revision_id
        self.tree._inventory.revision_id = revision_id
        return DirectoryTreeEditor(self.tree, file_id)

    def close(self):
        pass

    def abort(self):
        pass

class DirectoryTreeEditor:
    def __init__(self, tree, file_id):
        self.tree = tree
        self.file_id = file_id
        self.dir_ignores = None

    def add_directory(self, path, copyfrom_path=None, copyfrom_revnum=-1):
        path = path.decode("utf-8")
        file_id, revision_id = self.tree.id_map[path]
        ie = self.tree._inventory.add_path(path, 'directory', file_id)
        ie.revision = revision_id
        return DirectoryTreeEditor(self.editor, file_id)

    def change_prop(self, name, value):
        from mapping import (SVN_PROP_BZR_ANCESTRY,
                        SVN_PROP_BZR_PREFIX, SVN_PROP_BZR_REVISION_INFO,
                        SVN_PROP_BZR_FILEIDS, SVN_PROP_BZR_REVISION_ID,
                        SVN_PROP_BZR_BRANCHING_SCHEME, SVN_PROP_BZR_MERGE)

        if name == constants.PROP_ENTRY_COMMITTED_REV:
            self.dir_revnum = int(value)
        elif name == constants.PROP_IGNORE:
            self.dir_ignores = value
        elif name.startswith(SVN_PROP_BZR_ANCESTRY):
            if self.file_id != self.tree._inventory.root.file_id:
                mutter('%r set on non-root dir!' % name)
                return
        elif name in (SVN_PROP_BZR_FILEIDS, SVN_PROP_BZR_BRANCHING_SCHEME):
            if self.file_id != self.tree._inventory.root.file_id:
                mutter('%r set on non-root dir!' % name)
                return
        elif name in (SVN_PROP_ENTRY_COMMITTED_DATE,
                      SVN_PROP_ENTRY_LAST_AUTHOR,
                      SVN_PROP_ENTRY_LOCK_TOKEN,
                      SVN_PROP_ENTRY_UUID,
                      SVN_PROP_EXECUTABLE):
            pass
        elif name.startswith(constants.PROP_WC_PREFIX):
            pass
        elif (name == SVN_PROP_BZR_REVISION_INFO or
              name.startswith(SVN_PROP_BZR_REVISION_ID)):
            pass
        elif name == SVN_PROP_BZR_MERGE:
            pass
        elif (name.startswith(constants.PROP_PREFIX) or
              name.startswith(SVN_PROP_BZR_PREFIX)):
            mutter('unsupported dir property %r' % name)

    def add_file(self, path, copyfrom_path=None, copyfrom_revnum=-1):
        path = path.decode("utf-8")
        return FileTreeEditor(self.tree, path)

    def close(self):
        if (self.dir_ignores is not None and
            self.file_id in self.tree._inventory):
            self.tree._inventory[self.file_id].ignores = self.dir_ignores

class FileTreeEditor:
    def __init__(self, tree, path):
        self.tree = tree
        self.path = path
        self.is_executable = False
        self.is_symlink = False
        self.last_file_rev = None

    def change_prop(self, name, value):
        from mapping import SVN_PROP_BZR_PREFIX

        if name == constants.PROP_EXECUTABLE:
            self.is_executable = (value != None)
        elif name == constants.PROP_SPECIAL:
            self.is_symlink = (value != None)
        elif name == constants.PROP_ENTRY_COMMITTED_REV:
            self.last_file_rev = int(value)
        elif name in (constants.PROP_ENTRY_COMMITTED_DATE,
                      constants.PROP_ENTRY_LAST_AUTHOR,
                      constants.PROP_ENTRY_LOCK_TOKEN,
                      constants.PROP_ENTRY_UUID,
                      constants.PROP_MIME_TYPE):
            pass
        elif name.startswith(constants.PROP_WC_PREFIX):
            pass
        elif (name.startswith(constants.PROP_PREFIX) or
              name.startswith(SVN_PROP_BZR_PREFIX)):
            mutter('unsupported file property %r' % name)

    def close(self, checksum=None):
        file_id, revision_id = self.tree.id_map[self.path]
        if self.is_symlink:
            ie = self.tree._inventory.add_path(path, 'symlink', file_id)
        else:
            ie = self.tree._inventory.add_path(path, 'file', file_id)
        ie.revision = revision_id

        if self.file_stream:
            self.file_stream.seek(0)
            file_data = self.file_stream.read()
        else:
            file_data = ""

        actual_checksum = md5.new(file_data).hexdigest()
        assert(checksum is None or checksum == actual_checksum,
                "checksum mismatch: %r != %r" % (checksum, actual_checksum))

        if self.is_symlink:
            ie.symlink_target = file_data[len("link "):]
            ie.text_sha1 = None
            ie.text_size = None
            ie.text_id = None
            ie.executable = False
        else:
            ie.text_sha1 = osutils.sha_string(file_data)
            ie.text_size = len(file_data)
            self.tree.file_data[file_id] = file_data
            ie.executable = self.is_executable

        self.file_stream = None

    def apply_textdelta(self, file_id, base_checksum):
        self.file_stream = StringIO()
        return apply_txdelta_handler(StringIO(""), self.file_stream)

class SvnBasisTree(RevisionTree):
    """Optimized version of SvnRevisionTree."""
    def __init__(self, workingtree):
        self.workingtree = workingtree
        self._revision_id = workingtree.branch.generate_revision_id(
                                      workingtree.base_revnum)
        self.id_map = workingtree.branch.repository.get_fileid_map(
                workingtree.base_revnum,
                workingtree.branch.get_branch_path(workingtree.base_revnum),
                workingtree.branch.mapping)
        self._inventory = Inventory(root_id=None)
        self._repository = workingtree.branch.repository

        def add_file_to_inv(relpath, id, revid, adm):
            (delta_props, props) = adm.get_prop_diffs(self.workingtree.abspath(relpath))
            if props.has_key(constants.PROP_SPECIAL):
                ie = self._inventory.add_path(relpath, 'symlink', id)
                ie.symlink_target = open(self._abspath(relpath)).read()[len("link "):]
                ie.text_sha1 = None
                ie.text_size = None
                ie.text_id = None
                ie.executable = False
            else:
                ie = self._inventory.add_path(relpath, 'file', id)
                data = osutils.fingerprint_file(open(self._abspath(relpath)))
                ie.text_sha1 = data['sha1']
                ie.text_size = data['size']
                ie.executable = props.has_key(constants.PROP_EXECUTABLE)
            ie.revision = revid
            return ie

        def find_ids(entry):
            relpath = urllib.unquote(entry.url[len(entry.repos):].strip("/"))
            if entry.schedule in (wc.SCHEDULE_NORMAL,
                                  wc.SCHEDULE_DELETE,
                                  wc.SCHEDULE_REPLACE):
                return self.id_map[workingtree.branch.unprefix(relpath)]
            return (None, None)

        def add_dir_to_inv(relpath, adm, parent_id):
            entries = adm.entries_read(False)
            entry = entries[""]
            (id, revid) = find_ids(entry)
            if id == None:
                return

            # First handle directory itself
            ie = self._inventory.add_path(relpath, 'directory', id)
            ie.revision = revid
            if relpath == "":
                self._inventory.revision_id = revid

            for name in entries:
                if name == "":
                    continue

                subrelpath = os.path.join(relpath, name)

                entry = entries[name]
                assert entry

                if entry.kind == core.NODE_DIR:
                    subwc = wc.WorkingCopy(adm,
                            self.workingtree.abspath(subrelpath),
                                             False, 0, None)
                    try:
                        add_dir_to_inv(subrelpath, subwc, id)
                    finally:
                        subwc.close()
                else:
                    (subid, subrevid) = find_ids(entry)
                    if subid is not None:
                        add_file_to_inv(subrelpath, subid, subrevid, adm)

        adm = workingtree._get_wc()
        try:
            add_dir_to_inv("", adm, None)
        finally:
            adm.close()

    def _abspath(self, relpath):
        return wc.get_pristine_copy_path(self.workingtree.abspath(relpath))

    def get_file_lines(self, file_id):
        base_copy = self._abspath(self.id2path(file_id))
        return osutils.split_lines(open(base_copy).read())

<commit_msg>Fix tree snapshot creation.<commit_after># Copyright (C) 2005-2006 Jelmer Vernooij <jelmer@samba.org>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""Access to stored Subversion basis trees."""

from bzrlib.inventory import Inventory

from bzrlib import osutils, urlutils
from bzrlib.trace import mutter
from bzrlib.revisiontree import RevisionTree

import os
import md5
from cStringIO import StringIO
import urllib

import constants
import core, wc
from delta import apply_txdelta_handler

class SvnRevisionTree(RevisionTree):
    """A tree that existed in a historical Subversion revision."""
    def __init__(self, repository, revision_id):
        self._repository = repository
        self._revision_id = revision_id
        (self.branch_path, self.revnum, mapping) = repository.lookup_revision_id(revision_id)
        self._inventory = Inventory()
        self.id_map = repository.get_fileid_map(self.revnum, self.branch_path,
                                                mapping)
        editor = TreeBuildEditor(self)
        self.file_data = {}
        root_repos = repository.transport.get_svn_repos_root()
        reporter = repository.transport.do_switch(
                self.revnum, True,
                urlutils.join(root_repos, self.branch_path), editor)
        reporter.set_path("", 0, True, None)
        reporter.finish_report()

    def get_file_lines(self, file_id):
        return osutils.split_lines(self.file_data[file_id])

class TreeBuildEditor:
    """Builds a tree given Subversion tree transform calls."""
    def __init__(self, tree):
        self.tree = tree
        self.repository = tree._repository
        self.last_revnum = {}
        self.dir_revnum = {}
        self.dir_ignores = {}

    def set_target_revision(self, revnum):
        self.revnum = revnum

    def open_root(self, revnum):
        file_id, revision_id = self.tree.id_map[""]
        ie = self.tree._inventory.add_path("", 'directory', file_id)
        ie.revision = revision_id
        self.tree._inventory.revision_id = revision_id
        return DirectoryTreeEditor(self.tree, file_id)

    def close(self):
        pass

    def abort(self):
        pass

class DirectoryTreeEditor:
    def __init__(self, tree, file_id):
        self.tree = tree
        self.file_id = file_id
        self.dir_ignores = None

    def add_directory(self, path, copyfrom_path=None, copyfrom_revnum=-1):
        path = path.decode("utf-8")
        file_id, revision_id = self.tree.id_map[path]
        ie = self.tree._inventory.add_path(path, 'directory', file_id)
        ie.revision = revision_id
        return DirectoryTreeEditor(self.editor, file_id)

    def change_prop(self, name, value):
        from mapping import (SVN_PROP_BZR_ANCESTRY,
                        SVN_PROP_BZR_PREFIX, SVN_PROP_BZR_REVISION_INFO,
                        SVN_PROP_BZR_FILEIDS, SVN_PROP_BZR_REVISION_ID,
                        SVN_PROP_BZR_BRANCHING_SCHEME, SVN_PROP_BZR_MERGE)

        if name == constants.PROP_ENTRY_COMMITTED_REV:
            self.dir_revnum = int(value)
        elif name == constants.PROP_IGNORE:
            self.dir_ignores = value
        elif name.startswith(SVN_PROP_BZR_ANCESTRY):
            if self.file_id != self.tree._inventory.root.file_id:
                mutter('%r set on non-root dir!' % name)
                return
        elif name in (SVN_PROP_BZR_FILEIDS, SVN_PROP_BZR_BRANCHING_SCHEME):
            if self.file_id != self.tree._inventory.root.file_id:
                mutter('%r set on non-root dir!' % name)
                return
        elif name in (SVN_PROP_ENTRY_COMMITTED_DATE,
                      SVN_PROP_ENTRY_LAST_AUTHOR,
                      SVN_PROP_ENTRY_LOCK_TOKEN,
                      SVN_PROP_ENTRY_UUID,
                      SVN_PROP_EXECUTABLE):
            pass
        elif name.startswith(constants.PROP_WC_PREFIX):
            pass
        elif (name == SVN_PROP_BZR_REVISION_INFO or
              name.startswith(SVN_PROP_BZR_REVISION_ID)):
            pass
        elif name == SVN_PROP_BZR_MERGE:
            pass
        elif (name.startswith(constants.PROP_PREFIX) or
              name.startswith(SVN_PROP_BZR_PREFIX)):
            mutter('unsupported dir property %r' % name)

    def add_file(self, path, copyfrom_path=None, copyfrom_revnum=-1):
        path = path.decode("utf-8")
        return FileTreeEditor(self.tree, path)

    def close(self):
        if (self.dir_ignores is not None and
            self.file_id in self.tree._inventory):
            self.tree._inventory[self.file_id].ignores = self.dir_ignores

class FileTreeEditor:
    def __init__(self, tree, path):
        self.tree = tree
        self.path = path
        self.is_executable = False
        self.is_symlink = False
        self.last_file_rev = None

    def change_prop(self, name, value):
        from mapping import SVN_PROP_BZR_PREFIX

        if name == constants.PROP_EXECUTABLE:
            self.is_executable = (value != None)
        elif name == constants.PROP_SPECIAL:
            self.is_symlink = (value != None)
        elif name == constants.PROP_ENTRY_COMMITTED_REV:
            self.last_file_rev = int(value)
        elif name in (constants.PROP_ENTRY_COMMITTED_DATE,
                      constants.PROP_ENTRY_LAST_AUTHOR,
                      constants.PROP_ENTRY_LOCK_TOKEN,
                      constants.PROP_ENTRY_UUID,
                      constants.PROP_MIME_TYPE):
            pass
        elif name.startswith(constants.PROP_WC_PREFIX):
            pass
        elif (name.startswith(constants.PROP_PREFIX) or
              name.startswith(SVN_PROP_BZR_PREFIX)):
            mutter('unsupported file property %r' % name)

    def close(self, checksum=None):
        file_id, revision_id = self.tree.id_map[self.path]
        if self.is_symlink:
            ie = self.tree._inventory.add_path(self.path, 'symlink', file_id)
        else:
            ie = self.tree._inventory.add_path(self.path, 'file', file_id)
        ie.revision = revision_id

        if self.file_stream:
            self.file_stream.seek(0)
            file_data = self.file_stream.read()
        else:
            file_data = ""

        actual_checksum = md5.new(file_data).hexdigest()
        assert(checksum is None or checksum == actual_checksum,
                "checksum mismatch: %r != %r" % (checksum, actual_checksum))

        if self.is_symlink:
            ie.symlink_target = file_data[len("link "):]
            ie.text_sha1 = None
            ie.text_size = None
            ie.text_id = None
            ie.executable = False
        else:
            ie.text_sha1 = osutils.sha_string(file_data)
            ie.text_size = len(file_data)
            self.tree.file_data[file_id] = file_data
            ie.executable = self.is_executable

        self.file_stream = None

    def apply_textdelta(self, base_checksum=None):
        self.file_stream = StringIO()
        return apply_txdelta_handler(StringIO(""), self.file_stream)

class SvnBasisTree(RevisionTree):
    """Optimized version of SvnRevisionTree."""
    def __init__(self, workingtree):
        self.workingtree = workingtree
        self._revision_id = workingtree.branch.generate_revision_id(
                                      workingtree.base_revnum)
        self.id_map = workingtree.branch.repository.get_fileid_map(
                workingtree.base_revnum,
                workingtree.branch.get_branch_path(workingtree.base_revnum),
                workingtree.branch.mapping)
        self._inventory = Inventory(root_id=None)
        self._repository = workingtree.branch.repository

        def add_file_to_inv(relpath, id, revid, adm):
            (delta_props, props) = adm.get_prop_diffs(self.workingtree.abspath(relpath))
            if props.has_key(constants.PROP_SPECIAL):
                ie = self._inventory.add_path(relpath, 'symlink', id)
                ie.symlink_target = open(self._abspath(relpath)).read()[len("link "):]
                ie.text_sha1 = None
                ie.text_size = None
                ie.text_id = None
                ie.executable = False
            else:
                ie = self._inventory.add_path(relpath, 'file', id)
                data = osutils.fingerprint_file(open(self._abspath(relpath)))
                ie.text_sha1 = data['sha1']
                ie.text_size = data['size']
                ie.executable = props.has_key(constants.PROP_EXECUTABLE)
            ie.revision = revid
            return ie

        def find_ids(entry):
            relpath = urllib.unquote(entry.url[len(entry.repos):].strip("/"))
            if entry.schedule in (wc.SCHEDULE_NORMAL,
                                  wc.SCHEDULE_DELETE,
                                  wc.SCHEDULE_REPLACE):
                return self.id_map[workingtree.branch.unprefix(relpath)]
            return (None, None)

        def add_dir_to_inv(relpath, adm, parent_id):
            entries = adm.entries_read(False)
            entry = entries[""]
            (id, revid) = find_ids(entry)
            if id == None:
                return

            # First handle directory itself
            ie = self._inventory.add_path(relpath, 'directory', id)
            ie.revision = revid
            if relpath == "":
                self._inventory.revision_id = revid

            for name in entries:
                if name == "":
                    continue

                subrelpath = os.path.join(relpath, name)

                entry = entries[name]
                assert entry

                if entry.kind == core.NODE_DIR:
                    subwc = wc.WorkingCopy(adm,
                            self.workingtree.abspath(subrelpath),
                                             False, 0, None)
                    try:
                        add_dir_to_inv(subrelpath, subwc, id)
                    finally:
                        subwc.close()
                else:
                    (subid, subrevid) = find_ids(entry)
                    if subid is not None:
                        add_file_to_inv(subrelpath, subid, subrevid, adm)

        adm = workingtree._get_wc()
        try:
            add_dir_to_inv("", adm, None)
        finally:
            adm.close()

    def _abspath(self, relpath):
        return wc.get_pristine_copy_path(self.workingtree.abspath(relpath))

    def get_file_lines(self, file_id):
        base_copy = self._abspath(self.id2path(file_id))
        return osutils.split_lines(open(base_copy).read())

<|endoftext|>


<commit_before>__author__ = 'Evgenii'
<commit_msg>Delete tttt.py<commit_after><|endoftext|>


<commit_before># Copyright 2010 Gregory L. Rosenblatt

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from storage import *

class TypeError(Exception): pass
def typeErr(ctx, msg): raise TypeError(ctx, msg)
def isTyped(val): return (isinstance(val, tuple) and len(val)>0 and
                          isinstance(getTy(val), Type))
def typed(ty, val): return (ty, val)
def getTy(val): return val[0]
def getVal(val): return val[1]
class Type:
    def contains(self, ty): return self is ty
    def checkTy(self, val):
        if not self.contains(getTy(val)):
            typeErr(None, "expected '%s', given '%s'"%
                    (self.desc(), getTy(val).desc()))
    def checkIndex(self, idx, msg, exact=False):
        ni = self.numIndices()
        if ni is None and not exact: return
        if (exact and (idx != ni)) or idx<0 or idx>self.numIndices():
            typeErr(None, msg+" '%d'; %s has %d elements"%
                    (idx, self.desc(), ni))
    def size(self): raise NotImplementedError
    def numIndices(self): return 0
    def index(self, idx): raise NotImplementedError
    def unpack(self, mem, offset): raise NotImplementedError
    def pack(self, mem, offset, val): raise NotImplementedError
    def unpackEl(self, agg, idx):
        self.checkTy(agg); elt, off = self.index(idx)
        return elt.unpack(getVal(agg), off)
    def packEl(self, agg, idx, val):
        self.checkTy(agg); elt, off = self.index(idx);
        elt.pack(getVal(agg), off, val)
    def desc(self): raise NotImplementedError
    def __repr__(self): return '<%s %s>'%(self.__class__.__name__, self.desc())
################################################################
# unboxed types
class UnboxedType(Type):
    def unpack(self, mem, offs): return typed(self, self._unpack(mem, offs))
    def _unpack(self, mem, offs): raise NotImplementedError
    def pack(self, mem, offs, val):
        self.checkTy(val); self._pack(mem, offs, getVal(val))
    def _pack(self, mem, offset, val): raise NotImplementedError
class AtomicUnboxedType(UnboxedType):
    def size(self): return 1 # fixed size due to python
    def _unpack(self, mem, offset): return mem_read(mem, offset)
    def _pack(self, mem, offset, val): mem_write(mem, offset, val)
class ScalarType(AtomicUnboxedType):
    def __init__(self, name, pred=None): self.name = name; self.pred = pred
    def new(self, val):
        if self.pred is not None and not self.pred(val):
            typeErr(None, "invalid scalar '%r'"%val)
        return typed(self, val)
    def desc(self): return str(self.name)
class PyType(ScalarType):
    def __init__(self, name, pyty):
        super().__init__(name, (lambda x: isinstance(x, pyty)))
def cachedType(cls):
    cls._cache = {}
    def makeType(*args):
        ty = cls._cache.get(args)
        if ty is None: ty = cls(*args); cls._cache[args] = ty
        return ty
    return makeType
class PtrType(AtomicUnboxedType):
    def __init__(self, elt): self.elt = elt
    def contains(self, ty):
        return type(ty) is type(self) and self.elt.contains(ty.elt)
    def desc(self): return '&%s'%self.elt
ptrType = cachedType(PtrType)
class AggUnboxedType(UnboxedType):
    def _unpack(self, mem, offset): return mem_offset(mem, offset)
    def _pack(self, mem, offset, val):
        mem_copy(mem_offset(mem, offset), val, self.size())
class ArrayType(AggUnboxedType):
    def __init__(self, elt, cnt=None): self.elt = elt; self.cnt = cnt
    def contains(self, ty):
        return (type(ty) is type(self) and self.elt.contains(ty.elt)
                and (ty.cnt == self.cnt or
                    (ty.cnt is not None and ty.cnt > self.cnt)))
    def size(self):
        if self.cnt is not None: return self.cnt*self.elt.size()
        else: return None
    def numIndices(self): return self.cnt
    def index(self, idx): # None-length array implies dynamic extent
        assert self.cnt is None or idx>=0 and idx<self.cnt, (idx, self.desc())
        return self.elt, idx*self.elt.size()
    def desc(self):
        if self.cnt is None: pref = '';
        else: pref = '%d * '%self.cnt
        return '#[%s]'%(pref+self.elt.desc())
arrayType = cachedType(ArrayType)
def struct_index(struct, idx):
    struct.checkIndex(idx, 'invalid index')
    return struct.elts[idx], sum(elt.size() for elt in struct.elts[:idx])
class StructType(AggUnboxedType):
    def __init__(self, elts): self.elts = elts
    def contains(self, ty):
        return (type(ty) is type(self) and
                all(t1.contains(t2) for t1, t2 in zip(self.elts, ty.elts)))
    def size(self): return sum(elt.size() for elt in self.elts())
    def numIndices(self): return len(self.elts)
    def index(self, idx): return struct_index(self, idx)
    def desc(self): return '#{%s}'%' '.join(lyt.desc() for lyt in self.ellyts)
structType = cachedType(StructType)
################################################################
# boxed types
class BoxedType(Type):
    def size(self): return 1
    def unpack(self, mem, offset):
        box = mem_read(mem, offset); self.checkTy(box); return box
    def pack(self, mem, offset, box):
        self.checkTy(box); mem_write(mem, offset, box)
class AnyType(BoxedType):
    def contains(self, ty): return isinstance(ty, BoxedType)
    def desc(self): return 'Any'
anyTy = AnyType()
class NodeType(BoxedType):
    def __init__(self, name, elts):
        self.name = name; self.elts = elts;
        if elts is not None: self.eltSize = sum(elt.size() for elt in elts)
    def checkValid(self):
        if self.elts is None:
            typeErr(None, "attempted to use undefined tag: '%s'"%self.name)
    def numIndices(self): self.checkValid(); return len(self.elts)
    def index(self, idx): self.checkValid(); return struct_index(self, idx)
    def new(self, *args):
        self.checkIndex(len(args), 'invalid number of constructor args', True)
        mem = mem_alloc(self.eltSize)
        offset = 0
        for elt, arg in zip(self.elts, args):
            elt.pack(mem, offset, arg); offset += elt.size()
        return typed(self, mem)
    def desc(self): return str(self.name)
def isNode(v): return isTyped(v) and anyTy.contains(getTy(v))
class ProcType(BoxedType):
    def __init__(self, inTy, outTy): self.inTy = inTy; self.outTy = outTy
    def new(self, proc): return typed(self, proc)
    def appliedTy(self, remainingApps):
        ty = self; argts = []
        while remainingApps != 0:
            if not isinstance(ty, ProcType): break
            argts.append(ty.inTy); ty = ty.outTy; remainingApps -= 1
        return ty, argts, remainingApps
    def desc(self): return '%s -> %s'%(self.inTy.desc(), self.outTy.desc())
def isProc(v): return isTyped(v) and isinstance(getTy(v), ProcType)
procType = cachedType(ProcType)
def curryProcType(paramts, rett):
    for paramt in paramts: rett = procType(paramt, rett)
    return rett
class UnionType(BoxedType):
    def __init__(self, elts): self.elts = elts
    def contains(self, ty):
        return ((isinstance(ty, UnionType) and
                 all(self.contains(elt) for elt in ty.elts))
                or any(elt.contains(ty) for elt in self.elts))
    def desc(self): return '(%s)'%'|'.join(elt.desc() for elt in self.elts)
class VarType(BoxedType):
    def __init__(self, name, constraint=anyTy):
        self.name = name; self.constraint = constraint
    def contains(self, ty): # todo: separate containment for types vs. tags
        return ((isinstance(ty, PolyType) and
                 self.constraint.contains(ty.constraint))
                or self.constraint.contains(ty))
    def desc(self): return str(self.name)
class ParametricType(BoxedType):
    def __init__(self, name, params, elt):
        self.name = name; self.params = params; self.elt = elt
    def contains(self, ty): pass
    def desc(self):
        return '(%s)'%' '.join([str(self.name), ' '.join(self.params)])
<commit_msg>subtyping in progress<commit_after># Copyright 2010 Gregory L. Rosenblatt

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from storage import *

class TypeError(Exception): pass
def typeErr(ctx, msg): raise TypeError(ctx, msg)
def isTyped(val): return (isinstance(val, tuple) and len(val)>0 and
                          isinstance(getTy(val), Type))
def typed(ty, val): return (ty, val)
def getTy(val): return val[0]
def getVal(val): return val[1]
class Type:
    def contains(self, ty, tenv=None): return self is ty
    def checkTy(self, val):
        if not self.contains(getTy(val)):
            typeErr(None, "expected '%s', given '%s'"%
                    (self.desc(), getTy(val).desc()))
    def checkIndex(self, idx, msg, exact=False):
        ni = self.numIndices()
        if ni is None and not exact: return
        if (exact and (idx != ni)) or idx<0 or idx>self.numIndices():
            typeErr(None, msg+" '%d'; %s has %d elements"%
                    (idx, self.desc(), ni))
    def size(self): raise NotImplementedError
    def numIndices(self): return 0
    def index(self, idx): raise NotImplementedError
    def unpack(self, mem, offset): raise NotImplementedError
    def pack(self, mem, offset, val): raise NotImplementedError
    def unpackEl(self, agg, idx):
        self.checkTy(agg); elt, off = self.index(idx)
        return elt.unpack(getVal(agg), off)
    def packEl(self, agg, idx, val):
        self.checkTy(agg); elt, off = self.index(idx);
        elt.pack(getVal(agg), off, val)
    def desc(self): raise NotImplementedError
    def __str__(self): return self.desc()
    def __repr__(self): return '<%s %s>'%(self.__class__.__name__, self.desc())
################################################################
# unboxed types
class UnboxedType(Type):
    def unpack(self, mem, offs): return typed(self, self._unpack(mem, offs))
    def _unpack(self, mem, offs): raise NotImplementedError
    def pack(self, mem, offs, val):
        self.checkTy(val); self._pack(mem, offs, getVal(val))
    def _pack(self, mem, offset, val): raise NotImplementedError
class AtomicUnboxedType(UnboxedType):
    def size(self): return 1 # fixed size due to python
    def _unpack(self, mem, offset): return mem_read(mem, offset)
    def _pack(self, mem, offset, val): mem_write(mem, offset, val)
class ScalarType(AtomicUnboxedType):
    def __init__(self, name, pred=None): self.name = name; self.pred = pred
    def new(self, val):
        if self.pred is not None and not self.pred(val):
            typeErr(None, "invalid scalar '%r'"%val)
        return typed(self, val)
    def desc(self): return str(self.name)
class PyType(ScalarType):
    def __init__(self, name, pyty):
        super().__init__(name, (lambda x: isinstance(x, pyty)))
def cachedType(cls):
    cls._cache = {}
    def makeType(*args):
        ty = cls._cache.get(args)
        if ty is None: ty = cls(*args); cls._cache[args] = ty
        return ty
    return makeType
class PtrType(AtomicUnboxedType):
    def __init__(self, elt): self.elt = elt
    def contains(self, ty, tenv=None):
        return type(ty) is type(self) and self.elt.contains(ty.elt, tenv)
    def desc(self): return '&%s'%self.elt
ptrType = cachedType(PtrType)
class AggUnboxedType(UnboxedType):
    def _unpack(self, mem, offset): return mem_offset(mem, offset)
    def _pack(self, mem, offset, val):
        mem_copy(mem_offset(mem, offset), val, self.size())
class ArrayType(AggUnboxedType):
    def __init__(self, elt, cnt=None): self.elt = elt; self.cnt = cnt
    def contains(self, ty, tenv=None):
        return (type(ty) is type(self) and self.elt.contains(ty.elt, tenv)
                and (ty.cnt == self.cnt or
                    (ty.cnt is not None and ty.cnt > self.cnt)))
    def size(self):
        if self.cnt is not None: return self.cnt*self.elt.size()
        else: return None
    def numIndices(self): return self.cnt
    def index(self, idx): # None-length array implies dynamic extent
        assert self.cnt is None or idx>=0 and idx<self.cnt, (idx, self.desc())
        return self.elt, idx*self.elt.size()
    def desc(self):
        if self.cnt is None: pref = '';
        else: pref = '%d * '%self.cnt
        return '#[%s]'%(pref+self.elt.desc())
arrayType = cachedType(ArrayType)
def struct_index(struct, idx):
    struct.checkIndex(idx, 'invalid index')
    return struct.elts[idx], sum(elt.size() for elt in struct.elts[:idx])
class StructType(AggUnboxedType):
    def __init__(self, elts): self.elts = elts
    def contains(self, ty, tenv=None):
        return (type(ty) is type(self) and
                all(t1.contains(t2, tenv)
                    for t1, t2 in zip(self.elts, ty.elts)))
    def size(self): return sum(elt.size() for elt in self.elts())
    def numIndices(self): return len(self.elts)
    def index(self, idx): return struct_index(self, idx)
    def desc(self): return '#{%s}'%' '.join(lyt.desc() for lyt in self.ellyts)
structType = cachedType(StructType)
################################################################
# boxed types
class BoxedType(Type):
    def size(self): return 1
    def unpack(self, mem, offset):
        box = mem_read(mem, offset); self.checkTy(box); return box
    def pack(self, mem, offset, box):
        self.checkTy(box); mem_write(mem, offset, box)
class AnyType(BoxedType):
    def contains(self, ty, tenv=None): return isinstance(ty, BoxedType)
    def desc(self): return 'Any'
anyTy = AnyType()
class NodeType(BoxedType):
    def __init__(self, name, elts):
        self.name = name; self.elts = elts;
        if elts is not None: self.eltSize = sum(elt.size() for elt in elts)
    def contains(self, ty, tenv=None):
        if isinstance(ty, RefinedType):
            return (ty.elt is self and
                    all(elt.contains(arg, tenv)
                        for elt, arg in zip(self.elts, ty.args)))
        return ty is self
    def checkValid(self):
        if self.elts is None:
            typeErr(None, "attempted to use undefined tag: '%s'"%self.name)
    def numIndices(self): self.checkValid(); return len(self.elts)
    def index(self, idx): self.checkValid(); return struct_index(self, idx)
    def new(self, *args):
        self.checkIndex(len(args), 'invalid number of constructor args', True)
        mem = mem_alloc(self.eltSize)
        offset = 0
        for elt, arg in zip(self.elts, args):
            elt.pack(mem, offset, arg); offset += elt.size()
        return typed(self, mem)
    def desc(self): return str(self.name)
    def refine(self, args):
        assert len(args) == len(self.elts)
        for arg, elt in zip(args, self.elts):
            if arg is None: arg = elt
            elif not elt.contains(arg):
                typeErr(None, 'cannot refine node type')
        return RefinedType(self, args)
def isNode(v): return isTyped(v) and anyTy.contains(getTy(v))
class ProcType(BoxedType):
    def __init__(self, inTy, outTy): self.inTy = inTy; self.outTy = outTy
    def contains(self, ty, tenv=None):
        return (isProc(ty) and self.inTy.contains(ty.inTy, tenv) and
                self.outTy.contains(ty.outTy, tenv))
    def new(self, proc): return typed(self, proc)
    def appliedTy(self, remainingApps):
        ty = self; argts = []
        while remainingApps != 0:
            if not isinstance(ty, ProcType): break
            argts.append(ty.inTy); ty = ty.outTy; remainingApps -= 1
        return ty, argts, remainingApps
    def desc(self): return '%s -> %s'%(self.inTy.desc(), self.outTy.desc())
    def refine(self, rIn, rOut):
        rpt = procType(rIn, rOut)
        if self.contains(rpt): return rpt
        typeErr(None, 'cannot refine proc type')
def isProc(v): return isTyped(v) and isinstance(getTy(v), ProcType)
procType = cachedType(ProcType)
def curryProcType(paramts, rett):
    for paramt in paramts: rett = procType(paramt, rett)
    return rett
class UnionType(BoxedType): # todo: refinement produces union of refinements
    def __init__(self, elts): self.elts = elts
    def contains(self, ty, tenv=None):
        return ((isinstance(ty, UnionType) and
                 all(self.contains(elt, tenv) for elt in ty.elts))
                 or any(elt.contains(ty, tenv) for elt in self.elts))
    def desc(self): return '(%s)'%'|'.join(elt.desc() for elt in self.elts)
def intersectCns(cns, ty, tenv):
    if cns[0].contains(ty, tenv): cns[0] = ty; return True # todo: reduce ty
    return False
class ParamVarType(BoxedType): # only makes sense within parametric context
    def __init__(self, name): self.name = name
    def contains(self, ty, tenv=None):
        assert tenv is not None; constraint = tenv.get(self.name)
        return intersectCns(constraint, ty, tenv)
   def desc(self): return str(self.name)
class ConstraintType(BoxedType):
    def __init__(self, name, constraint=anyTy):
        self.name = name; self.constraint = constraint
    def contains(self, ty, tenv=None):
        return self.constraint.contains(ty, tenv)
    def desc(self): return '%s<:%s'%(str(self.name), self.constraint)
class ParametricType(BoxedType):
    def __init__(self, name, constraints, elt):
        self.name = name; self.constraints = constraints; self.elt = elt
    def contains(self, ty, tenv=None): # todo
        if isinstance(ty, ParametricType): return ty is self
        elif isinstance(ty, RefinedType):
            if isinstance(ty, ParametricType):
                return (ty.elt is self and
                        all(cns.contains(arg, tenv)
                            for cns, arg in zip(self.constraints, ty.args)))
        assert tenv is not None; tenv = Env(tenv)
        for cns in self.constraints: tenv.add(cns.name, [cns.constraint])
        return self.elt.contains(ty, tenv)
    def desc(self):
        return '(%s => %s)'%(' '.join([str(self.name), ' '.join(self.params)]), self.elt.desc())
class RefinedType(BoxedType):
    def __init__(self, elt, args): self.elt = elt; self.args = args
    def contains(self, ty, tenv):
        if isinstance(ty, RefinedType):
            return (self.elt is ty.elt and
                    all(arg.contains(targ, tenv)
                        for arg, targ in zip(self.args, ty.args)))
        elif isinstance(self.elt, ParametricType):
            assert tenv is not None; tenv = Env(tenv)
            for cns, arg in zip(self.elt.constraints, self.args):
                tenv.add(cns.name, [arg])
            return self.elt.elt.contains(ty, tenv)
        return False
    def desc(self):
        return '(%s)'%' '.join([self.elt.desc(), ' '.join(self.args)])
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *
from django.conf import settings

from django.contrib import admin
admin.autodiscover()

import pages.urls

urlpatterns = patterns('',
    (r'^$', 'p3.views.root'),
    (r'^pycon3/__assopy-dev$', 'conference.views.genro_wrapper'),
    (r'^admin/(.*)', admin.site.root),
    (r'^blog/', include('microblog.urls')),
    (r'^i18n/', include('django.conf.urls.i18n')),
    (r'^pycon3/', include('pages.urls')),
    (r'^conference/', include('conference.urls')),
)

if settings.DEBUG:
    import os.path
    args = []
    for k, path in settings.STATIC_DIRS.items():
        args.append((
            r'^static/%s/(?P<path>.*)$' % k,
            'django.views.static.serve',
            {'document_root': os.path.join(path, k), 'show_indexes': True}
        ))
    urlpatterns += patterns('', *args)

urlpatterns = urlpatterns
<commit_msg>fixato l'url di assopy-dev<commit_after>from django.conf.urls.defaults import *
from django.conf import settings

from django.contrib import admin
admin.autodiscover()

import pages.urls

urlpatterns = patterns('',
    (r'^$', 'p3.views.root'),
    (r'^pycon3/__assopy-dev/$', 'conference.views.genro_wrapper'),
    (r'^admin/(.*)', admin.site.root),
    (r'^blog/', include('microblog.urls')),
    (r'^i18n/', include('django.conf.urls.i18n')),
    (r'^pycon3/', include('pages.urls')),
    (r'^conference/', include('conference.urls')),
)

if settings.DEBUG:
    import os.path
    args = []
    for k, path in settings.STATIC_DIRS.items():
        args.append((
            r'^static/%s/(?P<path>.*)$' % k,
            'django.views.static.serve',
            {'document_root': os.path.join(path, k), 'show_indexes': True}
        ))
    urlpatterns += patterns('', *args)

urlpatterns = urlpatterns
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *
from django.contrib import admin

from unalog2.settings import UNALOG_ROOT

admin.autodiscover()

urlpatterns = patterns('',
    # NOTE: don't enable this for real.
    (r'^s/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': "%s/base/media" % UNALOG_ROOT}),

    url(r'^admin/(.*)', admin.site.root),

    url(r'^login/$', 'django.contrib.auth.views.login',
        {'template_name': 'login.html'}, name='login'),

    )

urlpatterns += patterns('unalog2.base.views',
    # Site-wide
    url(r'^$', 'index', name='index'),
    url(r'^feed/$', 'feed', name='feed'),
    url(r'^logout/$', 'logout_view', name='logout'),
    url(r'^register/$', 'register_view', name='register'),
    url(r'^indexing.js$', 'indexing_js', name='indexing_js'),
    url(r'^bookmarklet/$', 'bookmarklet', name='bookmarklet'),
    url(r'^about/$', 'about', name='about'),
    url(r'^contact/$', 'contact', name='contact'),

    # Entries
    url(r'^entry/(?P<entry_id>[0-9]+)$', 'entry', name='entry'),
    url(r'^entry/new', 'entry_new', name='entry_new'),
    url(r'^entry/(?P<entry_id>[0-9]+)/edit/$', 'entry_edit',
        name='entry_edit'),
    url(r'^entry/(?P<entry_id>[0-9]+)/delete/$', 'entry_delete',
        name='entry_delete'),

    # Tags (site-wide)
    url(r'^tags/$', 'tags', name='tags'),
    url(r'^tag/(?P<tag_name>[\w:._-]+)/$', 'tag', name='tag'),
    url(r'^tag/(?P<tag_name>[\w:._-]+)/feed/$', 'tag_feed',
        name='tag_feed'),

    # Users
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/$', 'user', name='user'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/feed/$', 'user_feed',
        name='user_feed'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/tags/$', 'user_tags',
        name='user_tags'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[\w:._-]+)/$',
        'user_tag', name='user_tag'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[\w:._-]+)/feed/$',
        'user_tag_feed', name='user_tag_feed'),

    # Filters
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/filters/$', 'user_filters',
        name='user_filters'),

    # Legacy url pattern - redirect.  No name assigned to discourage use.
    url(r'^person/(?P<user_name>.*)/$', 'person'),

    # Groups...
    url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/$', 'group', name='group'),
    url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/feed/$', 'group_feed',
        name='group_feed'),
    # Punt on these for now.
    #url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/tags/$', 'group_tags',
    #    name='group_tags'),
    #url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[a-zA-Z0-9:._-]+)/$',
    #    'group_tag', name='group_tag'),
    #url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[a-zA-Z0-9:._-]+)/feed/$',
    #    'group_tag_feed', name='group_tag_feed'),

    # URLs
    url(r'^url/(?P<md5sum>[a-f0-9]{32})/$', 'url', name='url'),
    url(r'^url/(?P<md5sum>[a-f0-9]{32})/feed/$', 'url_feed', name='url_feed'),

    # Search
    url(r'^search/$', 'search', name='search'),
    url(r'^search/feed/$', 'search_feed', name='search_feed'),

    )
<commit_msg>Making tag name pattern slightly more accomodating with "+", need to cross-check against form<commit_after>from django.conf.urls.defaults import *
from django.contrib import admin

from unalog2.settings import UNALOG_ROOT

admin.autodiscover()

urlpatterns = patterns('',
    # NOTE: don't enable this for real.
    (r'^s/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': "%s/base/media" % UNALOG_ROOT}),

    url(r'^admin/(.*)', admin.site.root),

    url(r'^login/$', 'django.contrib.auth.views.login',
        {'template_name': 'login.html'}, name='login'),

    )

urlpatterns += patterns('unalog2.base.views',
    # Site-wide
    url(r'^$', 'index', name='index'),
    url(r'^feed/$', 'feed', name='feed'),
    url(r'^logout/$', 'logout_view', name='logout'),
    url(r'^register/$', 'register_view', name='register'),
    url(r'^indexing.js$', 'indexing_js', name='indexing_js'),
    url(r'^bookmarklet/$', 'bookmarklet', name='bookmarklet'),
    url(r'^about/$', 'about', name='about'),
    url(r'^contact/$', 'contact', name='contact'),

    # Entries
    url(r'^entry/(?P<entry_id>[0-9]+)$', 'entry', name='entry'),
    url(r'^entry/new', 'entry_new', name='entry_new'),
    url(r'^entry/(?P<entry_id>[0-9]+)/edit/$', 'entry_edit',
        name='entry_edit'),
    url(r'^entry/(?P<entry_id>[0-9]+)/delete/$', 'entry_delete',
        name='entry_delete'),

    # Tags (site-wide)
    url(r'^tags/$', 'tags', name='tags'),
    url(r'^tag/(?P<tag_name>[+\w:._-]+)/$', 'tag', name='tag'),
    url(r'^tag/(?P<tag_name>[+\w:._-]+)/feed/$', 'tag_feed',
        name='tag_feed'),

    # Users
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/$', 'user', name='user'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/feed/$', 'user_feed',
        name='user_feed'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/tags/$', 'user_tags',
        name='user_tags'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[+\w:._-]+)/$',
        'user_tag', name='user_tag'),
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[+\w:._-]+)/feed/$',
        'user_tag_feed', name='user_tag_feed'),

    # Filters
    url(r'^user/(?P<user_name>[a-zA-Z0-9._]+)/filters/$', 'user_filters',
        name='user_filters'),

    # Legacy url pattern - redirect.  No name assigned to discourage use.
    url(r'^person/(?P<user_name>.*)/$', 'person'),

    # Groups...
    url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/$', 'group', name='group'),
    url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/feed/$', 'group_feed',
        name='group_feed'),
    # Punt on these for now.
    #url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/tags/$', 'group_tags',
    #    name='group_tags'),
    #url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[a-zA-Z0-9:._-]+)/$',
    #    'group_tag', name='group_tag'),
    #url(r'^group/(?P<group_name>[a-zA-Z0-9._]+)/tag/(?P<tag_name>[a-zA-Z0-9:._-]+)/feed/$',
    #    'group_tag_feed', name='group_tag_feed'),

    # URLs
    url(r'^url/(?P<md5sum>[a-f0-9]{32})/$', 'url', name='url'),
    url(r'^url/(?P<md5sum>[a-f0-9]{32})/feed/$', 'url_feed', name='url_feed'),

    # Search
    url(r'^search/$', 'search', name='search'),
    url(r'^search/feed/$', 'search_feed', name='search_feed'),

    )
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *

urlpatterns = patterns('',
    # Example:
    # (r'^labtracker/', include('labtracker.foo.urls')),

    # Uncomment this for admin:
     (r'^admin/', include('django.contrib.admin.urls')),
)
<commit_msg>Added the databrowse and the IssueTracker url entries <commit_after>from django.conf.urls.defaults import *
from django.contrib import databrowse
from labtracker.Machine.models import *
#from labtracker.LabtrackerCore.models import *

#databrowse.site.register(labtracker.Machine.model.Machine)
#databrowse.site.register(Item)
databrowse.site.register(Machine)

urlpatterns = patterns('',
    # Example:
    # (r'^labtracker/', include('labtracker.foo.urls')),

    # Uncomment this for admin:
     (r'^admin/', include('django.contrib.admin.urls')),
     (r'^databrowse/(.*)', databrowse.site.root),
     (r'^issue/', include('labtracker.IssueTracker.urls')),
)
<|endoftext|>


<commit_before>from collections import defaultdict
import smtplib

import requests

from config import MAIL_SERVER
from config import MAIL_PORT
from config import MAIL_USERNAME
from config import MAIL_PASSWORD
from config import DEFAULT_MAIL_SENDER
from config import ENABLE_SLACK
from config import SLACK_API_KEY
from config import SLACK_CHANNEL
from config import MULTIPLE_ACCOUNT_WARNING_MAIL_RECIPIENT

def notify_slack(contact=None, opportunity=None, rdo=None):
    """
    Send a notification about a donation to Slack.
    """
    reason = ""
    if opportunity:
        if opportunity.encouraged_by:
            reason = f" (encouraged by {opportunity.encouraged_by})"
        message = f"*{contact.name}* ({contact.email}) pledged *${opportunity.amount}*{reason}"

    if rdo:
        if rdo.encouraged_by:
            reason = f" (encouraged by {rdo.encouraged_by})"
        message = f"*{contact.name}* ({contact.email}) pledged *${rdo.amount}*{reason} [{rdo.installment_period}]"

    if not ENABLE_SLACK:
        print(message)
        return

    payload = {
        "token": SLACK_API_KEY,
        "channel": SLACK_CHANNEL,
        "text": message,
        "username": "moneybot",
        "icon_emoji": ":moneybag:",
    }
    url = "https://slack.com/api/chat.postMessage"
    try:
        requests.get(url, params=payload)
    except Exception as e:
        print("Failed to send Slack notification: {}".format(e))

def send_multiple_account_warning(contact):
    """
    Send the warnings about multiple accounts.
    """

    body = f"""
    Multiple accounts were found matching the email address <{contact.email}>
    while inserting a transaction.

    The transaction was attached to the first match found. You may want to
    move the transaction to the proper account if the one chosen is not
    correct. You may also want to delete or otherwise correct the duplicate
    account(s).
    """

    send_email(
        recipient=MULTIPLE_ACCOUNT_WARNING_MAIL_RECIPIENT,
        subject=f"Multiple accounts found for {contact.email}",
        body=body,
    )

def clean(form=None):
    result = defaultdict(lambda: None)
    for k, v in form.items():
        if v is None or v == "None":
            result[k] = None
            continue
        if v is True or v == "True":
            result[k] = True
            continue
        if v is False or v == "False":
            result[k] = False
            continue
        try:
            result[k] = int(v)
            continue
        except ValueError:
            try:
                result[k] = float(v)
                continue
            except ValueError:
                result[k] = v
    return result

def send_email(recipient, subject, body, sender=None):

    if sender is None:
        FROM = DEFAULT_MAIL_SENDER
    else:
        FROM = sender

    TO = recipient if type(recipient) is list else [recipient]
    SUBJECT = subject
    TEXT = body

    # Prepare actual message
    message = """\From: %s\nTo: %s\nSubject: %s\n\n%s
    """ % (
        FROM,
        ", ".join(TO),
        SUBJECT,
        TEXT,
    )
    try:
        server = smtplib.SMTP(MAIL_SERVER, MAIL_PORT)
        server.ehlo()
        server.starttls()
        server.login(MAIL_USERNAME, MAIL_PASSWORD)
        server.sendmail(FROM, TO, message)
        server.close()
        print("successfully sent the mail")
    except Exception as e:
        print("failed to send mail: {}".format(e))
<commit_msg>convert print to logging<commit_after>from collections import defaultdict
import smtplib
import logging

import requests

from config import MAIL_SERVER
from config import MAIL_PORT
from config import MAIL_USERNAME
from config import MAIL_PASSWORD
from config import DEFAULT_MAIL_SENDER
from config import ENABLE_SLACK
from config import SLACK_API_KEY
from config import SLACK_CHANNEL
from config import MULTIPLE_ACCOUNT_WARNING_MAIL_RECIPIENT

def notify_slack(contact=None, opportunity=None, rdo=None):
    """
    Send a notification about a donation to Slack.
    """
    reason = ""
    if opportunity:
        if opportunity.encouraged_by:
            reason = f" (encouraged by {opportunity.encouraged_by})"
        message = f"*{contact.name}* ({contact.email}) pledged *${opportunity.amount}*{reason}"

    if rdo:
        if rdo.encouraged_by:
            reason = f" (encouraged by {rdo.encouraged_by})"
        message = f"*{contact.name}* ({contact.email}) pledged *${rdo.amount}*{reason} [{rdo.installment_period}]"

    logging.info(message)

    if not ENABLE_SLACK:
        return

    payload = {
        "token": SLACK_API_KEY,
        "channel": SLACK_CHANNEL,
        "text": message,
        "username": "moneybot",
        "icon_emoji": ":moneybag:",
    }
    url = "https://slack.com/api/chat.postMessage"
    try:
        requests.get(url, params=payload)
    except Exception as e:
        logging.error("Failed to send Slack notification: {}".format(e))

def send_multiple_account_warning(contact):
    """
    Send the warnings about multiple accounts.
    """

    body = f"""
    Multiple accounts were found matching the email address <{contact.email}>
    while inserting a transaction.

    The transaction was attached to the first match found. You may want to
    move the transaction to the proper account if the one chosen is not
    correct. You may also want to delete or otherwise correct the duplicate
    account(s).
    """

    send_email(
        recipient=MULTIPLE_ACCOUNT_WARNING_MAIL_RECIPIENT,
        subject=f"Multiple accounts found for {contact.email}",
        body=body,
    )

def clean(form=None):
    result = defaultdict(lambda: None)
    for k, v in form.items():
        if v is None or v == "None":
            result[k] = None
            continue
        if v is True or v == "True":
            result[k] = True
            continue
        if v is False or v == "False":
            result[k] = False
            continue
        try:
            result[k] = int(v)
            continue
        except ValueError:
            try:
                result[k] = float(v)
                continue
            except ValueError:
                result[k] = v
    return result

def send_email(recipient, subject, body, sender=None):

    if sender is None:
        FROM = DEFAULT_MAIL_SENDER
    else:
        FROM = sender

    TO = recipient if type(recipient) is list else [recipient]
    SUBJECT = subject
    TEXT = body

    # Prepare actual message
    message = """\From: %s\nTo: %s\nSubject: %s\n\n%s
    """ % (
        FROM,
        ", ".join(TO),
        SUBJECT,
        TEXT,
    )
    try:
        server = smtplib.SMTP(MAIL_SERVER, MAIL_PORT)
        server.ehlo()
        server.starttls()
        server.login(MAIL_USERNAME, MAIL_PASSWORD)
        server.sendmail(FROM, TO, message)
        server.close()
        logging.debug("successfully sent the mail")
    except Exception as e:
        logging.error("failed to send mail: {}".format(e))
<|endoftext|>


<commit_before>def gethostname(mdsip):
	'mdsip: the IP address of MDS'
	import socket
	s=socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	s.connect((mdsip, 12345))
	hostname=s.getsockname()[0]
	s.close()
	return hostname

class Object:
	def __init__(self, d=None):
		if isinstance(d, dict):
			self.__dict__=d

def message2object(message):
	"receive a PB message, returns its guid and a object describing the message"
	import guid as Guid, messages_pb2 as msg
	if not hasattr(message, 'ListFields'):
		return message
	fields=message.ListFields()
	rst=Object()
	for f in fields:
		name=f[0].name
		value=f[1]
		if isinstance(value, msg.Guid):
			value=Guid.toStr(value)
		elif hasattr(value, 'ListFields'):
			value=message2object(value, '')
		elif hasattr(value, '_values'):
			value=[message2object(x) for x in value._values]
		else:
			pass  #should be a native value like str, int, float, ...
		setattr(rst, name, value)
	return rst

def object2message(object, message):
	import guid as Guid, messages_pb2 as msg
	d = object if isinstance(object, dict) else object.__dict__
	for key in d:
		if key.startswith('_'):
			continue
		value=d[key]
		if isinstance(value, list):
			mfield=getattr(message, key)
			appender = (lambda x : mfield.append(x)) if hasattr(mfield, 'append') \
				  else (lambda x : object2message(x, mfield.add()))
			for item in value:
				appender(item)
		else:
			try:
				if key == 'guid':
					value=Guid.fromStr(value)
					Guid.assign(message.guid, value)
				else:
					setattr(message, key, value)
			except:
				print 'exception occured'
				pass

def setupLogging(configure):
	import logging, os, os.path
	str2logginglevel={\
		'debug'		: logging.DEBUG,\
		'info'		: logging.INFO,\
		'warning'	: logging.WARNING,\
		'warn'		: logging.WARNING,\
		'error'		: logging.ERROR,\
		'fatal'		: logging.CRITICAL,\
	}
	loglevel=configure['logging-level']
	assert loglevel in str2logginglevel, 'invalid logging level "{0}"'.format(level)
	loglevel=str2logginglevel[loglevel]

	logfile=configure['logging-file']
	if logfile=='stdout':
		logfile=None
	else:
		assert not logfile[-1] in ['/', '\\'], 'logging-file must NOT be a directory'
		directory=os.path.dirname(logfile)
		if not os.path.isdir(directory):
			os.makedirs(directory)
	logging.basicConfig(filename=logfile, level=loglevel)

class Pool:
	def __init__(self, constructor, destructor=None):
		self.pool={}
		self.ctor=constructor
		self.dtor=destructor
	def get(self, *key):
		if key in self.pool:
			return self.pool[key]
		value=self.ctor(*key)
		self.pool[key]=value
		return value
	def dispose(self):
		if self.dtor:
			for value in self.pool.itervalues():
				self.dtor(value)
		self.pool.clear()

def testPool():
	class Stub:
		def __init__(self,a,b,c,d):
			self.key=(a,b,c,d)
			print 'constructing stub', self.key
		def close(self):
			print 'destructing stub', self.key
	pool=Pool(Stub)
	pool.get(10,20,3,2)
	pool.get(32,1.2312,56,32)
	pool.get('asdf', 'jkl;', 32, 3.13)
	pool.dispose()

	class Socket:
		def __init__(self,endpoint):
			self.key=endpoint
			print 'constructing socket', self.key
		def close(self):
			print 'destructing socket', self.key
	pool=Pool(lambda *args : Socket(args), Socket.close)
	pool.get(10,20)
	pool.get(32,1.2312)
	pool.get('asdf', 'jkl;')
	pool.dispose()

if __name__ == '__main__':
	#print gethostname('localhost.localdomain')
	testPool()
<commit_msg>make the destructor argument of Pool to accept a str<commit_after>def gethostname(mdsip):
	'mdsip: the IP address of MDS'
	import socket
	s=socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	s.connect((mdsip, 12345))
	hostname=s.getsockname()[0]
	s.close()
	return hostname

class Object:
	def __init__(self, d=None):
		if isinstance(d, dict):
			self.__dict__=d

def message2object(message):
	"receive a PB message, returns its guid and a object describing the message"
	import guid as Guid, messages_pb2 as msg
	if not hasattr(message, 'ListFields'):
		return message
	fields=message.ListFields()
	rst=Object()
	for f in fields:
		name=f[0].name
		value=f[1]
		if isinstance(value, msg.Guid):
			value=Guid.toStr(value)
		elif hasattr(value, 'ListFields'):
			value=message2object(value, '')
		elif hasattr(value, '_values'):
			value=[message2object(x) for x in value._values]
		else:
			pass  #should be a native value like str, int, float, ...
		setattr(rst, name, value)
	return rst

def object2message(object, message):
	import guid as Guid, messages_pb2 as msg
	d = object if isinstance(object, dict) else object.__dict__
	for key in d:
		if key.startswith('_'):
			continue
		value=d[key]
		if isinstance(value, list):
			mfield=getattr(message, key)
			appender = (lambda x : mfield.append(x)) if hasattr(mfield, 'append') \
				  else (lambda x : object2message(x, mfield.add()))
			for item in value:
				appender(item)
		else:
			try:
				if key == 'guid':
					value=Guid.fromStr(value)
					Guid.assign(message.guid, value)
				else:
					setattr(message, key, value)
			except:
				print 'exception occured'
				pass

def setupLogging(configure):
	import logging, os, os.path
	str2logginglevel={\
		'debug'		: logging.DEBUG,\
		'info'		: logging.INFO,\
		'warning'	: logging.WARNING,\
		'warn'		: logging.WARNING,\
		'error'		: logging.ERROR,\
		'fatal'		: logging.CRITICAL,\
	}
	loglevel=configure['logging-level']
	assert loglevel in str2logginglevel, 'invalid logging level "{0}"'.format(level)
	loglevel=str2logginglevel[loglevel]

	logfile=configure['logging-file']
	if logfile=='stdout':
		logfile=None
	else:
		assert not logfile[-1] in ['/', '\\'], 'logging-file must NOT be a directory'
		directory=os.path.dirname(logfile)
		if not os.path.isdir(directory):
			os.makedirs(directory)
	logging.basicConfig(filename=logfile, level=loglevel)

class Pool:
	def __init__(self, constructor, destructor=None):
		self.pool={}
		self.ctor=constructor
		if isinstance(destructor, str):
			destructor=getattr(constructor, destructor)
		self.dtor=destructor
	def get(self, *key):
		if key in self.pool:
			return self.pool[key]
		value=self.ctor(*key)
		self.pool[key]=value
		return value
	def dispose(self):
		if self.dtor:
			for value in self.pool.itervalues():
				self.dtor(value)
		self.pool.clear()

def testPool():
	class Stub:
		def __init__(self,a,b,c,d):
			self.key=(a,b,c,d)
			print 'constructing stub', self.key
		def close(self):
			print 'destructing stub', self.key
	pool=Pool(Stub)
	pool.get(10,20,3,2)
	pool.get(32,1.2312,56,32)
	pool.get('asdf', 'jkl;', 32, 3.13)
	pool.dispose()

	class Socket:
		def __init__(self,endpoint):
			self.key=endpoint
			print 'constructing socket', self.key
		def close(self):
			print 'destructing socket', self.key
	pool=Pool(lambda *args : Socket(args), Socket.close)
	pool.get(10,20)
	pool.get(32,1.2312)
	pool.get('asdf', 'jkl;')
	pool.dispose()

if __name__ == '__main__':
	#print gethostname('localhost.localdomain')
	testPool()
<|endoftext|>


<commit_before>import numpy
from theano import function

class StateComputer(object):
    """Convenient interface to theano function for computing states/cells
    of a trained SequenceGenerator model.

    Expects a Model instance as argument. This model must also be created
    from a SequenceGenerator.cost application. Furthermore, expects a dict
    that maps from characters to indices.
    Extracts the model's inputs and auxiliary variables into attributes
    and creates theano function from inputs to auxiliary vars.

    Finally, provides a method that wraps the output of the theano function in
    a dict keyed by auxiliary variable labels.
    """

    def __init__(self, cost_model, map_char_to_ind):
        raw_state_vars = filter(self._relevant, cost_model.auxiliary_variables)
        self.state_variables = sorted(raw_state_vars, key=lambda var: var.name)
        self.state_var_names = [var.name for var in self.state_variables]
        self.inputs = sorted(cost_model.inputs, key=lambda var: var.name)
        self.func = function(self.inputs, self.state_variables)
        self.map_char_to_ind = map_char_to_ind

    def _relevant(self, aux_var):
        not_final_value = "final_value" not in aux_var.name
        cell_or_state = "states" in aux_var.name or "cells" in aux_var.name
        return cell_or_state and not_final_value

    def read_single_sequence(self, sequence):
        """Combines list of aux var values (output from theano func) with their
        corresponding labels.

        New and improved with (I think) more convenient interface.
        - as input provide a list (or something array-like, i.e. that can be
        converted to a numpy array) of either characters or their representing
        integers. If a list of characters is provided, the conversion is done
        in this method. NOTE that a string is more or less a list of characters
        for the purpose of this method =)
        - the theano function expects a 2D input where the first dimension is
        batch_size. This "fake" dimension is added below, so don't wrap the
        input sequence yourself.
        - because the cost was originally defined with a mask, the theano
        function needs one, as well. This mask is constructed below so you
        don't have to provide it yourself.
        """
        if all(isinstance(entry, str) for entry in sequence):
            converted_sequence = numpy.array([[self.map_char_to_ind[char] for char in sequence]], dtype="int32")
        elif all(isinstance(entry, int) for entry in sequence):
            converted_sequence = numpy.array([sequence], dtype="int32")
        else:
            raise ValueError("Some or all sequence elements have invalid type (should be str or int)!")
        mask = numpy.ones(converted_sequence.shape, dtype="int8")
        return dict(zip(self.state_var_names, self.func(converted_sequence, mask)))
<commit_msg>PEP8: line length under 90 chars<commit_after>import numpy
from theano import function

class StateComputer(object):
    """Convenient interface to theano function for computing states/cells
    of a trained SequenceGenerator model.

    Expects a Model instance as argument. This model must also be created
    from a SequenceGenerator.cost application. Furthermore, expects a dict
    that maps from characters to indices.
    Extracts the model's inputs and auxiliary variables into attributes
    and creates theano function from inputs to auxiliary vars.

    Finally, provides a method that wraps the output of the theano function in
    a dict keyed by auxiliary variable labels.
    """

    def __init__(self, cost_model, map_char_to_ind):
        raw_state_vars = filter(self._relevant, cost_model.auxiliary_variables)
        self.state_variables = sorted(raw_state_vars, key=lambda var: var.name)
        self.state_var_names = [var.name for var in self.state_variables]
        self.inputs = sorted(cost_model.inputs, key=lambda var: var.name)
        self.func = function(self.inputs, self.state_variables)
        self.map_char_to_ind = map_char_to_ind

    def _relevant(self, aux_var):
        not_final_value = "final_value" not in aux_var.name
        cell_or_state = "states" in aux_var.name or "cells" in aux_var.name
        return cell_or_state and not_final_value

    def read_single_sequence(self, sequence):
        """Combines list of aux var values (output from theano func) with their
        corresponding labels.

        New and improved with (I think) more convenient interface.
        - as input provide a list (or something array-like, i.e. that can be
        converted to a numpy array) of either characters or their representing
        integers. If a list of characters is provided, the conversion is done
        in this method. NOTE that a string is more or less a list of characters
        for the purpose of this method =)
        - the theano function expects a 2D input where the first dimension is
        batch_size. This "fake" dimension is added below, so don't wrap the
        input sequence yourself.
        - because the cost was originally defined with a mask, the theano
        function needs one, as well. This mask is constructed below so you
        don't have to provide it yourself.
        """
        if all(isinstance(entry, str) for entry in sequence):
            indices = [self.map_char_to_ind[char] for char in sequence]
            converted_sequence = numpy.array([indices], dtype="int32")
        elif all(isinstance(entry, int) for entry in sequence):
            converted_sequence = numpy.array([sequence], dtype="int32")
        else:
            raise ValueError("Some or all sequence elements have invalid type "
                             "(should be str or int)!")
        mask = numpy.ones(converted_sequence.shape, dtype="int8")
        return dict(zip(self.state_var_names, self.func(converted_sequence, mask)))
<|endoftext|>


<commit_before>import os.path
import logging

import gym
import numpy as np
import stable_baselines
from stable_baselines.common.vec_env import DummyVecEnv, VecEnv

import reward_net
import util

def maybe_load_env(env_or_str, vectorize=False):
    """
    Params:
    env_or_str (str or gym.Env): The Env or its string id in Gym.
    vectorize (bool): If True, then vectorize the environment before returning,
      if it isn't already vectorized.

    Return:
    env (gym.Env) -- Either the original argument if it was an Env or an
      instantiated gym Env if it was a string.
    id (str) -- The environment's id.
    """
    if isinstance(env_or_str, str):
        env = gym.make(env_or_str)
    else:
        env = env_or_str

    if not is_vec_env(env) and vectorize:
        env = DummyVecEnv([lambda: env])

    return env

def is_vec_env(env):
    return isinstance(env, VecEnv)

def get_env_id(env):
    env = maybe_load_env(env)
    if is_vec_env(env):
        env = env.envs[0]
    return env.spec.id

def make_blank_policy(env, policy_class=stable_baselines.PPO1,
        init_tensorboard=False, policy_network_class="MlpPolicy",
        **kwargs):
    """
    Instantiates a policy for the provided environment.

    Params:
    env (str or Env): The Env or its string id in Gym.
    policy_network_class (stable_baselines.BasePolicy): A policy network
      constructor from the stable_baselines module.
    policy_class (stable_baselines.BaseRLModel subclass): A policy constructor
      from the stable_baselines module.
    init_tensorboard (bool): If True, then initialize the policy to make
      TensorBoard summary writes.

    Return:
    policy (stable_baselines.BaseRLModel)
    """
    env = util.maybe_load_env(env)
    return policy_class(policy_network_class, env, verbose=1,
            tensorboard_log=_get_tb_log_dir(env, init_tensorboard),
            **kwargs)

def get_or_train_policy(env, force_train=False, timesteps=500000,
        never_overwrite=False, policy_class=stable_baselines.PPO1,
        init_tensorboard=False, **kwargs):
    """
    Returns a policy trained on the given environment, maybe pretrained.

    If a policy for the environment hasn't been trained and pickled before,
    then first train and pickle it in saved_models/. Otherwise, load that
    pickled policy.

    When looking for trained policies, we first check for an expert policy
    in expert_models/ and then check in saved_models/ for a policy trained
    by this method.

    Params:
    env (str or Env): The Env that this policy is meant to act in, or the
      string name of the Gym environment.
    force_train (bool): If True, then always train a policy, ignoring any
      saved policies.
    timesteps (int): The number of training timesteps if we decide to train
      a policy.
    never_overwrite (bool): It True, then don't pickle a policy if it means
      overwriting another pickle.
    policy_class (stable_baselines.BaseRLModel class): A policy constructor
      from the stable_baselines module.
    init_tensorboard (bool): Whether to initialize Tensorboard logging for
      this policy.
    **kwargs: Additional options for initializing the BaseRLModel class.

    Return:
    policy (stable_baselines.BaseRLModel)
    """
    env = util.maybe_load_env(env)

    if not force_train:
        # Try expert first.
        expert_policy = load_expert_policy(env, policy_class,
                init_tensorboard, **kwargs)
        if expert_policy is not None:
            return expert_policy

        # Try trained.
        trained_policy = load_trained_policy(env, policy_class,
                init_tensorboard, **kwargs)
        if trained_policy is not None:
            return trained_policy

        logging.info("Didn't find a pickled policy. Training...")
    else:
        logging.info("force_train=True. Training...")

    # Learn and pickle a policy
    policy = make_blank_policy(env, policy_class, init_tensorboard, **kwargs)
    policy.learn(timesteps)
    save_trained_policy(policy, never_overwrite)

    return policy

def save_trained_policy(policy, never_overwrite=False):
    """
    Save a trained policy to saved_models/.
    """
    path = os.path.join("saved_models", _policy_filename(policy.__class__))
    if never_overwrite and os.path.exists(path):
        logging.info(("Avoided saving policy pickle to {} because "
            "overwrite is disabled and that file already exists."
                ).format(path))
    else:
        policy.save(path)
        logging.info("Saved pickle to {}!".path)

def load_trained_policy(env, **kwargs):
    """
    Load a trained policy from saved_models/.
    """
    return load_policy(env, basedir="saved_models", **kwargs)

def load_expert_policy(env, **kwargs):
    """
    Load an expert policy from expert_models/.
    """
    return load_policy(env, basedir="expert_models", **kwargs)

# TODO: It's probably a good idea to just use the model/policy semantics
# from stable_baselines, even if I disagree with their naming scheme.
#
# ie, rename policy_class=> policy_model_class
#     rename policy_network_class => policy (matches policy_class.__init__ arg)
def load_policy(env, policy_class=stable_baselines.PPO1, basedir="",
        init_tensorboard=False, policy_network_class=None, **kwargs):
    """
    Load a pickled policy and return it.

    Params:
    env (str or Env): The Env that this policy is meant to act in, or the
      string name of the Gym environment.
    policy_class (stable_baselines.BaseRLModel class): A policy constructor
      from the stable_baselines module.
    init_tensorboard (bool): Whether to initialize Tensorboard logging for
      this policy.
    base_dir (str): The directory of the pickled file.
    policy_network_class (stable_baselines.BasePolicy): A policy network
      constructor. Unless we are using a custom BasePolicy (not builtin to
      stable_baselines), this is automatically infered, and so we can leave
      this argument as None.
    **kwargs: Additional options for initializing the BaseRLModel class.
    """
    path = os.path.join(basedir, _policy_filename(policy_class, env))
    exists = os.path.exists(path)
    if exists:
        env = maybe_load_env(env)
        if policy_network_class is not None:
            kwargs["policy"] = policy_network_class

        policy = policy_class.load(path, env,
                tensorboard_log=_get_tb_log_dir(env, init_tensorboard),
                **kwargs)
        logging.info("loaded policy from '{}'".format(path))
        return policy
    else:
        logging.info("couldn't find policy at '{}'".format(path))
        return None

def _policy_filename(policy_class, env):
    """
    Returns the .pkl filename that the policy instantiated with policy_class
    and trained on env should be saved to.
    """
    return "{}_{}.pkl".format(policy_class.__name__, get_env_id(env))

def rollout_action_probability(policy, rollout_obs, rollout_act):
    """
    Find the batch probability of observation, action pairs under a given
    policy.

    Params:
    policy (stable_baselines.BaseRLModel): The policy.
    rollout_obs (array) -- A numpy array with shape
      `[n_timesteps] + env.observation_space.shape`.
    rollout_act (array) -- A numpy array with shape
      `[n_timesteps] + env.action_space.shape`.

    Return:
    rollout_prob (array) -- A numpy array with shape `[n_timesteps]`. The
      `i`th entry is the action probability of action `rollout_act[i]` when
      observing `rollout_obs[i]`.
    """

    # TODO: Only tested this on Cartpole (which has discrete actions). No
    # idea how this works in a different action space.
    act_prob = policy.action_probability(rollout_obs)
    if rollout_act.ndim == 1:
        # Expand None dimension so that we can use take_along_axis.
        rollout_act = rollout_act[:, np.newaxis]

    rollout_prob = np.take_along_axis(act_prob, rollout_act, axis=-1)
    rollout_prob = np.squeeze(rollout_prob, axis=1)

    n_timesteps = len(rollout_obs)
    assert len(rollout_obs) == len(rollout_act)
    assert rollout_prob.shape == (n_timesteps,)
    return rollout_prob

def rollout_generate(policy, env, *, n_timesteps=None, n_episodes=None):
    """
    Generate old_obs-action-new_obs-reward tuples from a policy and an
    environment.

    Params:
    policy (stable_baselines.BaseRLModel) -- A stable_baselines Model, trained
      on the gym environment.
    env (VecEnv or Env or str) -- The environment(s) to interact with.
    n_timesteps (int) -- The number of obs-action-obs-reward tuples to collect.
      Set exactly one of `n_timesteps` and `n_episodes`, or this function will
      error.
    n_episodes (int) -- The number of episodes to finish before returning
      collected tuples. Tuples from parallel episodes underway when the final
      episode is finished will also be returned.
      Set exactly one of `n_timesteps` and `n_episodes`, or this function will
      error.

    Return:
    rollout_obs_old (array) -- A numpy array with shape
      `[n_timesteps] + env.observation_space.shape`. The ith observation in this
      array is the observation seen with the agent chooses action
      `rollout_act[i]`.
    rollout_act (array) -- A numpy array with shape
      `[n_timesteps] + env.action_space.shape`.
    rollout_obs_new (array) -- A numpy array with shape
      `[n_timesteps] + env.observation_space.shape`. The ith observation in this
      array is from the transition state after the agent chooses action
      `rollout_act[i]`.
    rollout_rewards (array) -- A numpy array with shape `[n_timesteps]`. The
      reward received on the ith timestep is `rollout_rewards[i]`.
    """
    env = util.maybe_load_env(env, vectorize=True)
    policy.set_env(env)  # This checks that env and policy are compatbile.
    assert is_vec_env(env)

    # Validate end condition arguments and initialize end conditions.
    if n_timesteps is not None and n_episodes is not None:
        raise ValueError("n_timesteps and n_episodes were both set")
    elif n_timesteps is not None:
        assert n_timesteps > 0
        end_cond = "timesteps"
    elif n_episodes is not None:
        assert n_episodes > 0
        end_cond = "episodes"
        episodes_elapsed = 0
    else:
        raise ValueError("Set at least one of n_timesteps and n_episodes")

    # Implements end-condition logic.
    def rollout_done():
        if end_cond == "timesteps":
            return len(rollout_obs_new) >= n_timesteps
        elif end_cond == "episodes":
            return episodes_elapsed >= n_episodes
        else:
            raise RuntimeError(end_cond)

    # Collect rollout tuples.
    rollout_obs_old = []
    rollout_act = []
    rollout_obs_new = []
    rollout_rew = []
    obs = env.reset()
    while not rollout_done():
        # Current state.
        rollout_obs_old.extend(obs)

        # Current action.
        act, _ = policy.predict(obs)
        rollout_act.extend(act)

        # Transition state and rewards.
        obs, rew, done, _ = env.step(act)
        rollout_obs_new.extend(obs)
        rollout_rew.extend(rew)

        # Track episodes
        if end_cond == "episodes":
            episodes_elapsed += np.sum(done)

        if np.any(done):
            logging.debug("new episode!")

    # Convert results to numpy arrays. (Possibly truncate).
    rollout_obs_new = np.atleast_1d(rollout_obs_new)
    rollout_obs_old = np.atleast_1d(rollout_obs_old)
    rollout_act = np.atleast_1d(rollout_act)
    rollout_rew = np.atleast_1d(rollout_rew)
    if end_cond == "timesteps":
        n_steps = n_timesteps

        # Truncate because we want exactly n_timesteps.
        rollout_obs_new = rollout_obs_new[:n_timesteps]
        rollout_obs_old = rollout_obs_old[:n_timesteps]
        rollout_act = rollout_act[:n_timesteps]
        rollout_rew = rollout_rew[:n_timesteps]
    elif end_cond == "episodes":
        n_steps = len(rollout_obs_new)

    # Sanity checks.
    exp_obs = (n_steps,) + env.observation_space.shape
    exp_act = (n_steps,) + env.action_space.shape
    assert rollout_obs_new.shape == exp_obs
    assert rollout_obs_old.shape == exp_obs
    assert np.all(rollout_obs_new[:-1] == rollout_obs_old[1:])
    assert rollout_act.shape == exp_act
    assert rollout_rew.shape == (n_steps,)

    return rollout_obs_old, rollout_act, rollout_obs_new, rollout_rew

def rollout_total_reward(policy, env, **kwargs):
    """
    Get the undiscounted reward after rolling out `n_timestep` steps in
    of the policy. With large n_timesteps, this can be a decent metric
    for policy performance.

    Params:
    policy (stable_baselines.BaseRLModel) -- A stable_baselines Model, trained
      on the gym environment.
    env (VecEnv or Env or str) -- The environment(s) to interact with.
    n_timesteps (int) -- The number of rewards to collect.
    n_episodes (int) -- The number of episodes to finish before we stop
      collecting rewards. Reward from parallel episodes that are underway when
      the final episode is finished is also included in the reward total.

    Return:
    total_reward (int) -- The undiscounted reward from `n_timesteps` consecutive
      actions in `env`.
    """
    _, _, _, rew = rollout_generate(policy, env, **kwargs)
    return np.sum(rew)

def _get_tb_log_dir(env, init_tensorboard):
    if init_tensorboard:
        return "./output/{}/".format(get_env_id(env))
    else:
        return None
<commit_msg>Fix a VecEnv assertion error in rollout_generate<commit_after>import os.path
import logging

import gym
import numpy as np
import stable_baselines
from stable_baselines.common.vec_env import DummyVecEnv, VecEnv

import reward_net
import util

def maybe_load_env(env_or_str, vectorize=False):
    """
    Params:
    env_or_str (str or gym.Env): The Env or its string id in Gym.
    vectorize (bool): If True, then vectorize the environment before returning,
      if it isn't already vectorized.

    Return:
    env (gym.Env) -- Either the original argument if it was an Env or an
      instantiated gym Env if it was a string.
    id (str) -- The environment's id.
    """
    if isinstance(env_or_str, str):
        env = gym.make(env_or_str)
    else:
        env = env_or_str

    if not is_vec_env(env) and vectorize:
        env = DummyVecEnv([lambda: env])

    return env

def is_vec_env(env):
    return isinstance(env, VecEnv)

def get_env_id(env):
    env = maybe_load_env(env)
    if is_vec_env(env):
        env = env.envs[0]
    return env.spec.id

def make_blank_policy(env, policy_class=stable_baselines.PPO1,
        init_tensorboard=False, policy_network_class="MlpPolicy",
        **kwargs):
    """
    Instantiates a policy for the provided environment.

    Params:
    env (str or Env): The Env or its string id in Gym.
    policy_network_class (stable_baselines.BasePolicy): A policy network
      constructor from the stable_baselines module.
    policy_class (stable_baselines.BaseRLModel subclass): A policy constructor
      from the stable_baselines module.
    init_tensorboard (bool): If True, then initialize the policy to make
      TensorBoard summary writes.

    Return:
    policy (stable_baselines.BaseRLModel)
    """
    env = util.maybe_load_env(env)
    return policy_class(policy_network_class, env, verbose=1,
            tensorboard_log=_get_tb_log_dir(env, init_tensorboard),
            **kwargs)

def get_or_train_policy(env, force_train=False, timesteps=500000,
        never_overwrite=False, policy_class=stable_baselines.PPO1,
        init_tensorboard=False, **kwargs):
    """
    Returns a policy trained on the given environment, maybe pretrained.

    If a policy for the environment hasn't been trained and pickled before,
    then first train and pickle it in saved_models/. Otherwise, load that
    pickled policy.

    When looking for trained policies, we first check for an expert policy
    in expert_models/ and then check in saved_models/ for a policy trained
    by this method.

    Params:
    env (str or Env): The Env that this policy is meant to act in, or the
      string name of the Gym environment.
    force_train (bool): If True, then always train a policy, ignoring any
      saved policies.
    timesteps (int): The number of training timesteps if we decide to train
      a policy.
    never_overwrite (bool): It True, then don't pickle a policy if it means
      overwriting another pickle.
    policy_class (stable_baselines.BaseRLModel class): A policy constructor
      from the stable_baselines module.
    init_tensorboard (bool): Whether to initialize Tensorboard logging for
      this policy.
    **kwargs: Additional options for initializing the BaseRLModel class.

    Return:
    policy (stable_baselines.BaseRLModel)
    """
    env = util.maybe_load_env(env)

    if not force_train:
        # Try expert first.
        expert_policy = load_expert_policy(env, policy_class,
                init_tensorboard, **kwargs)
        if expert_policy is not None:
            return expert_policy

        # Try trained.
        trained_policy = load_trained_policy(env, policy_class,
                init_tensorboard, **kwargs)
        if trained_policy is not None:
            return trained_policy

        logging.info("Didn't find a pickled policy. Training...")
    else:
        logging.info("force_train=True. Training...")

    # Learn and pickle a policy
    policy = make_blank_policy(env, policy_class, init_tensorboard, **kwargs)
    policy.learn(timesteps)
    save_trained_policy(policy, never_overwrite)

    return policy

def save_trained_policy(policy, never_overwrite=False):
    """
    Save a trained policy to saved_models/.
    """
    path = os.path.join("saved_models", _policy_filename(policy.__class__))
    if never_overwrite and os.path.exists(path):
        logging.info(("Avoided saving policy pickle to {} because "
            "overwrite is disabled and that file already exists."
                ).format(path))
    else:
        policy.save(path)
        logging.info("Saved pickle to {}!".path)

def load_trained_policy(env, **kwargs):
    """
    Load a trained policy from saved_models/.
    """
    return load_policy(env, basedir="saved_models", **kwargs)

def load_expert_policy(env, **kwargs):
    """
    Load an expert policy from expert_models/.
    """
    return load_policy(env, basedir="expert_models", **kwargs)

# TODO: It's probably a good idea to just use the model/policy semantics
# from stable_baselines, even if I disagree with their naming scheme.
#
# ie, rename policy_class=> policy_model_class
#     rename policy_network_class => policy (matches policy_class.__init__ arg)
def load_policy(env, policy_class=stable_baselines.PPO1, basedir="",
        init_tensorboard=False, policy_network_class=None, **kwargs):
    """
    Load a pickled policy and return it.

    Params:
    env (str or Env): The Env that this policy is meant to act in, or the
      string name of the Gym environment.
    policy_class (stable_baselines.BaseRLModel class): A policy constructor
      from the stable_baselines module.
    init_tensorboard (bool): Whether to initialize Tensorboard logging for
      this policy.
    base_dir (str): The directory of the pickled file.
    policy_network_class (stable_baselines.BasePolicy): A policy network
      constructor. Unless we are using a custom BasePolicy (not builtin to
      stable_baselines), this is automatically infered, and so we can leave
      this argument as None.
    **kwargs: Additional options for initializing the BaseRLModel class.
    """
    path = os.path.join(basedir, _policy_filename(policy_class, env))
    exists = os.path.exists(path)
    if exists:
        env = maybe_load_env(env)
        if policy_network_class is not None:
            kwargs["policy"] = policy_network_class

        policy = policy_class.load(path, env,
                tensorboard_log=_get_tb_log_dir(env, init_tensorboard),
                **kwargs)
        logging.info("loaded policy from '{}'".format(path))
        return policy
    else:
        logging.info("couldn't find policy at '{}'".format(path))
        return None

def _policy_filename(policy_class, env):
    """
    Returns the .pkl filename that the policy instantiated with policy_class
    and trained on env should be saved to.
    """
    return "{}_{}.pkl".format(policy_class.__name__, get_env_id(env))

def rollout_action_probability(policy, rollout_obs, rollout_act):
    """
    Find the batch probability of observation, action pairs under a given
    policy.

    Params:
    policy (stable_baselines.BaseRLModel): The policy.
    rollout_obs (array) -- A numpy array with shape
      `[n_timesteps] + env.observation_space.shape`.
    rollout_act (array) -- A numpy array with shape
      `[n_timesteps] + env.action_space.shape`.

    Return:
    rollout_prob (array) -- A numpy array with shape `[n_timesteps]`. The
      `i`th entry is the action probability of action `rollout_act[i]` when
      observing `rollout_obs[i]`.
    """

    # TODO: Only tested this on Cartpole (which has discrete actions). No
    # idea how this works in a different action space.
    act_prob = policy.action_probability(rollout_obs)
    if rollout_act.ndim == 1:
        # Expand None dimension so that we can use take_along_axis.
        rollout_act = rollout_act[:, np.newaxis]

    rollout_prob = np.take_along_axis(act_prob, rollout_act, axis=-1)
    rollout_prob = np.squeeze(rollout_prob, axis=1)

    n_timesteps = len(rollout_obs)
    assert len(rollout_obs) == len(rollout_act)
    assert rollout_prob.shape == (n_timesteps,)
    return rollout_prob

def rollout_generate(policy, env, *, n_timesteps=None, n_episodes=None,
        truncate_timesteps=False):
    """
    Generate old_obs-action-new_obs-reward tuples from a policy and an
    environment.

    Params:
    policy (stable_baselines.BaseRLModel) -- A stable_baselines Model, trained
      on the gym environment.
    env (VecEnv or Env or str) -- The environment(s) to interact with.
    n_timesteps (int) -- The number of obs-action-obs-reward tuples to collect.
      The `truncate_timesteps` parameter chooses whether to discard extra
      tuples.
      Set exactly one of `n_timesteps` and `n_episodes`, or this function will
      error.
    n_episodes (int) -- The number of episodes to finish before returning
      collected tuples. Tuples from parallel episodes underway when the final
      episode is finished will also be returned.
      Set exactly one of `n_timesteps` and `n_episodes`, or this function will
      error.
    truncate_timesteps (bool) -- If True, then discard any tuples, ensuring that
      exactly `n_timesteps` are returned. Otherwise, return every collected
      tuple.

    Return:
    rollout_obs_old (array) -- A numpy array with shape
      `[n_timesteps] + env.observation_space.shape`. The ith observation in this
      array is the observation seen with the agent chooses action
      `rollout_act[i]`.
    rollout_act (array) -- A numpy array with shape
      `[n_timesteps] + env.action_space.shape`.
    rollout_obs_new (array) -- A numpy array with shape
      `[n_timesteps] + env.observation_space.shape`. The ith observation in this
      array is from the transition state after the agent chooses action
      `rollout_act[i]`.
    rollout_rewards (array) -- A numpy array with shape `[n_timesteps]`. The
      reward received on the ith timestep is `rollout_rewards[i]`.
    """
    env = util.maybe_load_env(env, vectorize=True)
    policy.set_env(env)  # This checks that env and policy are compatbile.
    assert is_vec_env(env)

    # Validate end condition arguments and initialize end conditions.
    if n_timesteps is not None and n_episodes is not None:
        raise ValueError("n_timesteps and n_episodes were both set")
    elif n_timesteps is not None:
        assert n_timesteps > 0
        end_cond = "timesteps"
    elif n_episodes is not None:
        assert n_episodes > 0
        end_cond = "episodes"
        episodes_elapsed = 0
    else:
        raise ValueError("Set at least one of n_timesteps and n_episodes")

    # Implements end-condition logic.
    def rollout_done():
        if end_cond == "timesteps":
            return len(rollout_obs_new) >= n_timesteps
        elif end_cond == "episodes":
            return episodes_elapsed >= n_episodes
        else:
            raise RuntimeError(end_cond)

    # Collect rollout tuples.
    rollout_obs_old = []
    rollout_act = []
    rollout_obs_new = []
    rollout_rew = []
    obs = env.reset()
    while not rollout_done():
        # Current state.
        rollout_obs_old.extend(obs)

        # Current action.
        act, _ = policy.predict(obs)
        rollout_act.extend(act)

        # Transition state and rewards.
        obs, rew, done, _ = env.step(act)
        rollout_obs_new.extend(obs)
        rollout_rew.extend(rew)

        # Track episodes
        if end_cond == "episodes":
            episodes_elapsed += np.sum(done)

        if np.any(done):
            logging.debug("new episode!")

    # Convert results to numpy arrays. (Possibly truncate).
    rollout_obs_new = np.atleast_1d(rollout_obs_new)
    rollout_obs_old = np.atleast_1d(rollout_obs_old)
    rollout_act = np.atleast_1d(rollout_act)
    rollout_rew = np.atleast_1d(rollout_rew)
    if end_cond == "timesteps" and truncate_timesteps:
        n_steps = n_timesteps

        # Truncate because we want exactly n_timesteps.
        rollout_obs_new = rollout_obs_new[:n_timesteps]
        rollout_obs_old = rollout_obs_old[:n_timesteps]
        rollout_act = rollout_act[:n_timesteps]
        rollout_rew = rollout_rew[:n_timesteps]
    else:
        n_steps = len(rollout_obs_new)

    # Sanity checks.
    exp_obs = (n_steps,) + env.observation_space.shape
    exp_act = (n_steps,) + env.action_space.shape
    n_envs = env.num_envs
    assert rollout_obs_new.shape == exp_obs
    assert rollout_obs_old.shape == exp_obs
    if not truncate_timesteps or n_timesteps % n_envs == 0:
        assert np.all(rollout_obs_new[:-n_envs] == rollout_obs_old[n_envs:])
    assert rollout_act.shape == exp_act
    assert rollout_rew.shape == (n_steps,)

    return rollout_obs_old, rollout_act, rollout_obs_new, rollout_rew

def rollout_total_reward(policy, env, **kwargs):
    """
    Get the undiscounted reward after rolling out `n_timestep` steps in
    of the policy. With large n_timesteps, this can be a decent metric
    for policy performance.

    Params:
    policy (stable_baselines.BaseRLModel) -- A stable_baselines Model, trained
      on the gym environment.
    env (VecEnv or Env or str) -- The environment(s) to interact with.
    n_timesteps (int) -- The number of rewards to collect.
    n_episodes (int) -- The number of episodes to finish before we stop
      collecting rewards. Reward from parallel episodes that are underway when
      the final episode is finished is also included in the reward total.

    Return:
    total_reward (int) -- The undiscounted reward from `n_timesteps` consecutive
      actions in `env`.
    """
    _, _, _, rew = rollout_generate(policy, env, **kwargs)
    return np.sum(rew)

def _get_tb_log_dir(env, init_tensorboard):
    if init_tensorboard:
        return "./output/{}/".format(get_env_id(env))
    else:
        return None
<|endoftext|>