<commit_before># Copyright (C) 2006-2007 Jelmer Vernooij <jelmer@samba.org>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""Generation of file-ids."""

from bzrlib import ui
from bzrlib.errors import NotBranchError, RevisionNotPresent
from bzrlib.knit import KnitVersionedFile
from bzrlib.revision import NULL_REVISION
from bzrlib.trace import mutter

import urllib

import changes
from mapping import escape_svn_path

def get_local_changes(paths, branch, mapping, generate_revid,
                      get_children=None):
    new_paths = {}
    for p in sorted(paths.keys()):
        if not changes.path_is_child(branch, p):
            continue
        data = paths[p]
        new_p = mapping.scheme.unprefix(p)[1]
        if data[1] is not None:
            try:
                (cbp, crp) = mapping.scheme.unprefix(data[1])

                # Branch copy
                if (crp == "" and new_p == ""):
                    data = ('M', None, None)
                else:
                    data = (data[0], crp, generate_revid(
                                  data[2], cbp, mapping))
            except NotBranchError:
                # Copied from outside of a known branch
                # Make it look like the files were added in this revision
                if get_children is not None:
                    for c in get_children(data[1], data[2]):
                        mutter('oops: %r child %r' % (data[1], c))
                        new_paths[(new_p+"/"+c[len(data[1]):].strip("/")).strip("/")] = (data[0], None, -1)
                data = (data[0], None, -1)

        new_paths[new_p] = data
    return new_paths

FILEIDMAP_VERSION = 1

def simple_apply_changes(new_file_id, changes, find_children=None):
    """Simple function that can apply file id changes.

    Does not track renames. """
    map = {}
    for p in sorted(changes.keys()):
        data = changes[p]

        if data[0] in ('A', 'R'):
            inv_p = p.decode("utf-8")
            map[inv_p] = new_file_id(inv_p)

            if data[1] is not None:
                mutter('%r copied from %r:%s' % (inv_p, data[1], data[2]))
                if find_children is not None:
                    for c in find_children(data[1], data[2]):
                        inv_c = c.decode("utf-8")
                        path = c.replace(data[1].decode("utf-8"), inv_p+"/", 1).replace(u"//", u"/")
                        map[path] = new_file_id(path)
                        mutter('added mapping %r -> %r' % (path, map[path]))

    return map

class FileIdMap(object):
    """File id store.

    Keeps a map

    revnum -> branch -> path -> fileid
    """
    def __init__(self, apply_changes_fn, repos):
        self.apply_changes_fn = apply_changes_fn
        self.repos = repos

    def apply_changes(self, uuid, revnum, branch, global_changes,
                      renames, mapping, find_children=None):
        """Change file id map to incorporate specified changes.

        :param uuid: UUID of repository changes happen in
        :param revnum: Revno for revision in which changes happened
        :param branch: Branch path where changes happened
        :param global_changes: Dict with global changes that happened
        :param renames: List of renames (known file ids for particular paths)
        :param mapping: Mapping
        """
        changes = get_local_changes(global_changes, branch, mapping,
                    self.repos.generate_revision_id, find_children)
        if find_children is not None:
            def get_children(path, revid):
                (bp, revnum, mapping) = self.repos.lookup_revision_id(revid)
                for p in find_children(bp+"/"+path, revnum):
                    yield mapping.unprefix(bp, p)
        else:
            get_children = None

        def new_file_id(x):
            return mapping.generate_file_id(uuid, revnum, branch, x)

        idmap = self.apply_changes_fn(new_file_id, changes, get_children)
        idmap.update(renames)
        return (idmap, changes)

    def get_map(self, uuid, revnum, branch, mapping):
        """Make sure the map is up to date until revnum."""
        # First, find the last cached map
        if revnum == 0:
            assert branch == ""
            return {"": (mapping.generate_file_id(uuid, 0, "", u""),
              self.repos.generate_revision_id(0, "", mapping))}

        todo = []
        next_parent_revs = []
        if mapping.scheme.is_branch(""):
            map = {u"": (mapping.generate_file_id(uuid, 0, "", u""), NULL_REVISION)}
        else:
            map = {}

        quickrevidmap = {}

        # No history -> empty map
        for (bp, paths, rev, revprops, changed_fileprops) in self.repos.iter_reverse_branch_changes(branch, revnum, mapping):
            revid = self.repos.generate_revision_id(rev, bp, mapping, changed_fileprops, revprops)
            quickrevidmap[revid] = (rev, bp)
            todo.append((revid, paths, revprops, changed_fileprops))

        pb = ui.ui_factory.nested_progress_bar()

        try:
            i = 1
            for (revid, global_changes, revprops, changed_fileprops) in reversed(todo):
                expensive = False
                def log_find_children(path, revnum):
                    expensive = True
                    return self.repos._log.find_children(path, revnum)

                (revnum, branch) = quickrevidmap[revid]
                (idmap, changes) = self.actual.apply_changes(self.repos.uuid,
                        revnum, branch, global_changes,
                        mapping.import_fileid_map(revprops, changed_fileprops),
                        mapping, log_find_children)
                pb.update('generating file id map', i, len(todo))

                parent_revs = next_parent_revs

                self.actual.update_map(map, revid, idmap, changes)

                next_parent_revs = [revid]
                i += 1
        finally:
            pb.finished()
        return map

    def update_map(self, map, revid, idmap, changes):
        for p in changes:
            if changes[p][0] == 'M' and not idmap.has_key(p):
                idmap[p] = map[p][0]

        map.update(dict([(x, (str(idmap[x]), revid)) for x in idmap]))

        # Mark all parent paths as changed
        for p in idmap:
            parts = p.split("/")
            for j in range(1, len(parts)+1):
                parent = "/".join(parts[0:len(parts)-j])
                assert map.has_key(parent), "Parent item %s of %s doesn't exist in map" % (parent, p)
                if map[parent][1] == revid:
                    break
                map[parent] = map[parent][0], revid

class CachingFileIdMap(object):
    """A file id map that uses a cache."""
    def __init__(self, cache_transport, actual):
        self.idmap_knit = KnitVersionedFile("fileidmap-v%d" % FILEIDMAP_VERSION, cache_transport, create=True)
        self.actual = actual
        self.apply_changes = actual.apply_changes
        self.repos = actual.repos

    def save(self, revid, parent_revids, _map):
        mutter('saving file id map for %r' % revid)

        self.idmap_knit.add_lines_with_ghosts(revid, parent_revids,
                ["%s\t%s\t%s\n" % (urllib.quote(filename), urllib.quote(_map[filename][0]),
                                        urllib.quote(_map[filename][1])) for filename in sorted(_map.keys())])

    def load(self, revid):
        map = {}
        for line in self.idmap_knit.get_lines(revid):
            (filename, id, create_revid) = line.rstrip("\n").split("\t", 3)
            map[urllib.unquote(filename)] = (urllib.unquote(id), urllib.unquote(create_revid))
            assert isinstance(map[urllib.unquote(filename)][0], str)

        return map

    def get_map(self, uuid, revnum, branch, mapping):
        """Make sure the map is up to date until revnum."""
        # First, find the last cached map
        if revnum == 0:
            assert branch == ""
            return {"": (mapping.generate_file_id(uuid, 0, "", u""),
              self.repos.generate_revision_id(0, "", mapping))}

        todo = []
        next_parent_revs = []
        quickrevidmap = {}

        # No history -> empty map
        for (bp, paths, rev, revprops, changed_fileprops) in self.repos.iter_reverse_branch_changes(branch, revnum, mapping):
            revid = self.repos.generate_revision_id(rev, bp, mapping,
                        changed_fileprops, revprops)
            quickrevidmap[revid] = (rev, bp)
            try:
                map = self.load(revid)
                # found the nearest cached map
                next_parent_revs = [revid]
                break
            except RevisionNotPresent:
                todo.append((revid, paths, revprops, changed_fileprops))

        # target revision was present
        if len(todo) == 0:
            return map

        if len(next_parent_revs) == 0:
            if mapping.scheme.is_branch(""):
                map = {u"": (mapping.generate_file_id(uuid, 0, "", u""), NULL_REVISION)}
            else:
                map = {}

        pb = ui.ui_factory.nested_progress_bar()

        try:
            i = 1
            for (revid, global_changes, revprops, changed_fileprops) in reversed(todo):
                expensive = False
                def log_find_children(path, revnum):
                    expensive = True
                    return self.repos._log.find_children(path, revnum)

                (revnum, branch) = quickrevidmap[revid]
                (idmap, changes) = self.actual.apply_changes(self.repos.uuid,
                        revnum, branch, global_changes,
                        mapping.import_fileid_map(revprops, changed_fileprops),
                        mapping, log_find_children)
                pb.update('generating file id map', i, len(todo))

                parent_revs = next_parent_revs

                self.actual.update_map(map, revid, idmap, changes)

                saved = False
                if i % 500 == 0 or expensive:
                    self.save(revid, parent_revs, map)
                    saved = True
                next_parent_revs = [revid]
                i += 1
        finally:
            pb.finished()
        if not saved:
            self.save(revid, parent_revs, map)
        return map

<commit_msg>Fix compatibility after KnitVersionedFile changes.<commit_after># Copyright (C) 2006-2007 Jelmer Vernooij <jelmer@samba.org>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""Generation of file-ids."""

from bzrlib import ui
from bzrlib.errors import NotBranchError, RevisionNotPresent
from bzrlib.knit import make_file_knit
from bzrlib.revision import NULL_REVISION
from bzrlib.trace import mutter

import urllib

import changes
from mapping import escape_svn_path

def get_local_changes(paths, branch, mapping, generate_revid,
                      get_children=None):
    new_paths = {}
    for p in sorted(paths.keys()):
        if not changes.path_is_child(branch, p):
            continue
        data = paths[p]
        new_p = mapping.scheme.unprefix(p)[1]
        if data[1] is not None:
            try:
                (cbp, crp) = mapping.scheme.unprefix(data[1])

                # Branch copy
                if (crp == "" and new_p == ""):
                    data = ('M', None, None)
                else:
                    data = (data[0], crp, generate_revid(
                                  data[2], cbp, mapping))
            except NotBranchError:
                # Copied from outside of a known branch
                # Make it look like the files were added in this revision
                if get_children is not None:
                    for c in get_children(data[1], data[2]):
                        mutter('oops: %r child %r' % (data[1], c))
                        new_paths[(new_p+"/"+c[len(data[1]):].strip("/")).strip("/")] = (data[0], None, -1)
                data = (data[0], None, -1)

        new_paths[new_p] = data
    return new_paths

FILEIDMAP_VERSION = 1

def simple_apply_changes(new_file_id, changes, find_children=None):
    """Simple function that can apply file id changes.

    Does not track renames. """
    map = {}
    for p in sorted(changes.keys()):
        data = changes[p]

        if data[0] in ('A', 'R'):
            inv_p = p.decode("utf-8")
            map[inv_p] = new_file_id(inv_p)

            if data[1] is not None:
                mutter('%r copied from %r:%s' % (inv_p, data[1], data[2]))
                if find_children is not None:
                    for c in find_children(data[1], data[2]):
                        inv_c = c.decode("utf-8")
                        path = c.replace(data[1].decode("utf-8"), inv_p+"/", 1).replace(u"//", u"/")
                        map[path] = new_file_id(path)
                        mutter('added mapping %r -> %r' % (path, map[path]))

    return map

class FileIdMap(object):
    """File id store.

    Keeps a map

    revnum -> branch -> path -> fileid
    """
    def __init__(self, apply_changes_fn, repos):
        self.apply_changes_fn = apply_changes_fn
        self.repos = repos

    def apply_changes(self, uuid, revnum, branch, global_changes,
                      renames, mapping, find_children=None):
        """Change file id map to incorporate specified changes.

        :param uuid: UUID of repository changes happen in
        :param revnum: Revno for revision in which changes happened
        :param branch: Branch path where changes happened
        :param global_changes: Dict with global changes that happened
        :param renames: List of renames (known file ids for particular paths)
        :param mapping: Mapping
        """
        changes = get_local_changes(global_changes, branch, mapping,
                    self.repos.generate_revision_id, find_children)
        if find_children is not None:
            def get_children(path, revid):
                (bp, revnum, mapping) = self.repos.lookup_revision_id(revid)
                for p in find_children(bp+"/"+path, revnum):
                    yield mapping.unprefix(bp, p)
        else:
            get_children = None

        def new_file_id(x):
            return mapping.generate_file_id(uuid, revnum, branch, x)

        idmap = self.apply_changes_fn(new_file_id, changes, get_children)
        idmap.update(renames)
        return (idmap, changes)

    def get_map(self, uuid, revnum, branch, mapping):
        """Make sure the map is up to date until revnum."""
        # First, find the last cached map
        if revnum == 0:
            assert branch == ""
            return {"": (mapping.generate_file_id(uuid, 0, "", u""),
              self.repos.generate_revision_id(0, "", mapping))}

        todo = []
        next_parent_revs = []
        if mapping.scheme.is_branch(""):
            map = {u"": (mapping.generate_file_id(uuid, 0, "", u""), NULL_REVISION)}
        else:
            map = {}

        quickrevidmap = {}

        # No history -> empty map
        for (bp, paths, rev, revprops, changed_fileprops) in self.repos.iter_reverse_branch_changes(branch, revnum, mapping):
            revid = self.repos.generate_revision_id(rev, bp, mapping, changed_fileprops, revprops)
            quickrevidmap[revid] = (rev, bp)
            todo.append((revid, paths, revprops, changed_fileprops))

        pb = ui.ui_factory.nested_progress_bar()

        try:
            i = 1
            for (revid, global_changes, revprops, changed_fileprops) in reversed(todo):
                expensive = False
                def log_find_children(path, revnum):
                    expensive = True
                    return self.repos._log.find_children(path, revnum)

                (revnum, branch) = quickrevidmap[revid]
                (idmap, changes) = self.actual.apply_changes(self.repos.uuid,
                        revnum, branch, global_changes,
                        mapping.import_fileid_map(revprops, changed_fileprops),
                        mapping, log_find_children)
                pb.update('generating file id map', i, len(todo))

                parent_revs = next_parent_revs

                self.actual.update_map(map, revid, idmap, changes)

                next_parent_revs = [revid]
                i += 1
        finally:
            pb.finished()
        return map

    def update_map(self, map, revid, idmap, changes):
        for p in changes:
            if changes[p][0] == 'M' and not idmap.has_key(p):
                idmap[p] = map[p][0]

        map.update(dict([(x, (str(idmap[x]), revid)) for x in idmap]))

        # Mark all parent paths as changed
        for p in idmap:
            parts = p.split("/")
            for j in range(1, len(parts)+1):
                parent = "/".join(parts[0:len(parts)-j])
                assert map.has_key(parent), "Parent item %s of %s doesn't exist in map" % (parent, p)
                if map[parent][1] == revid:
                    break
                map[parent] = map[parent][0], revid

class CachingFileIdMap(object):
    """A file id map that uses a cache."""
    def __init__(self, cache_transport, actual):
        self.idmap_knit = make_file_knit("fileidmap-v%d" % FILEIDMAP_VERSION, cache_transport, 0644, create=True)
        self.actual = actual
        self.apply_changes = actual.apply_changes
        self.repos = actual.repos

    def save(self, revid, parent_revids, _map):
        mutter('saving file id map for %r' % revid)

        self.idmap_knit.add_lines_with_ghosts(revid, parent_revids,
                ["%s\t%s\t%s\n" % (urllib.quote(filename), urllib.quote(_map[filename][0]),
                                        urllib.quote(_map[filename][1])) for filename in sorted(_map.keys())])

    def load(self, revid):
        map = {}
        for line in self.idmap_knit.get_lines(revid):
            (filename, id, create_revid) = line.rstrip("\n").split("\t", 3)
            map[urllib.unquote(filename)] = (urllib.unquote(id), urllib.unquote(create_revid))
            assert isinstance(map[urllib.unquote(filename)][0], str)

        return map

    def get_map(self, uuid, revnum, branch, mapping):
        """Make sure the map is up to date until revnum."""
        # First, find the last cached map
        if revnum == 0:
            assert branch == ""
            return {"": (mapping.generate_file_id(uuid, 0, "", u""),
              self.repos.generate_revision_id(0, "", mapping))}

        todo = []
        next_parent_revs = []
        quickrevidmap = {}

        # No history -> empty map
        for (bp, paths, rev, revprops, changed_fileprops) in self.repos.iter_reverse_branch_changes(branch, revnum, mapping):
            revid = self.repos.generate_revision_id(rev, bp, mapping,
                        changed_fileprops, revprops)
            quickrevidmap[revid] = (rev, bp)
            try:
                map = self.load(revid)
                # found the nearest cached map
                next_parent_revs = [revid]
                break
            except RevisionNotPresent:
                todo.append((revid, paths, revprops, changed_fileprops))

        # target revision was present
        if len(todo) == 0:
            return map

        if len(next_parent_revs) == 0:
            if mapping.scheme.is_branch(""):
                map = {u"": (mapping.generate_file_id(uuid, 0, "", u""), NULL_REVISION)}
            else:
                map = {}

        pb = ui.ui_factory.nested_progress_bar()

        try:
            i = 1
            for (revid, global_changes, revprops, changed_fileprops) in reversed(todo):
                expensive = False
                def log_find_children(path, revnum):
                    expensive = True
                    return self.repos._log.find_children(path, revnum)

                (revnum, branch) = quickrevidmap[revid]
                (idmap, changes) = self.actual.apply_changes(self.repos.uuid,
                        revnum, branch, global_changes,
                        mapping.import_fileid_map(revprops, changed_fileprops),
                        mapping, log_find_children)
                pb.update('generating file id map', i, len(todo))

                parent_revs = next_parent_revs

                self.actual.update_map(map, revid, idmap, changes)

                saved = False
                if i % 500 == 0 or expensive:
                    self.save(revid, parent_revs, map)
                    saved = True
                next_parent_revs = [revid]
                i += 1
        finally:
            pb.finished()
        if not saved:
            self.save(revid, parent_revs, map)
        return map

<|endoftext|>


<commit_before>from materials import alphaMaterials
from numpy import arange, zeros

def genericVerticalFlip(cls):
    rotation = arange(16, dtype='uint8')
    if hasattr(cls, "Up") and hasattr(cls, "Down"):
        rotation[cls.Up] = cls.Down
        rotation[cls.Down] = cls.Up

    return rotation

def genericRotation(cls):
    rotation = arange(16, dtype='uint8')
    rotation[cls.North] = cls.West
    rotation[cls.West] = cls.South
    rotation[cls.South] = cls.East
    rotation[cls.East] = cls.North
    return rotation

def genericEastWestFlip(cls):
    rotation = arange(16, dtype='uint8')
    rotation[cls.West] = cls.East
    rotation[cls.East] = cls.West
    return rotation

def genericNorthSouthFlip(cls):
    rotation = arange(16, dtype='uint8')
    rotation[cls.South] = cls.North
    rotation[cls.North] = cls.South
    return rotation

rotationClasses = []

def genericFlipRotation(cls):
    cls.rotateLeft = genericRotation(cls)

    cls.flipVertical = genericVerticalFlip(cls)
    cls.flipEastWest = genericEastWestFlip(cls)
    cls.flipNorthSouth = genericNorthSouthFlip(cls)
    rotationClasses.append(cls)

class Torch:
    blocktypes = [
        alphaMaterials.Torch.ID,
        alphaMaterials.RedstoneTorchOn.ID,
        alphaMaterials.RedstoneTorchOff.ID,
    ]

    South = 1
    North = 2
    West = 3
    East = 4

genericFlipRotation(Torch)

class Ladder:
    blocktypes = [alphaMaterials.Ladder.ID]

    East = 2
    West = 3
    North = 4
    South = 5
genericFlipRotation(Ladder)

class Stair:
    blocktypes = [b.ID for b in alphaMaterials.AllStairs]

    South = 0
    North = 1
    West = 2
    East = 3
genericFlipRotation(Stair)

class WallSign:
    blocktypes = [alphaMaterials.WallSign.ID]

    East = 2
    West = 3
    North = 4
    South = 5
genericFlipRotation(WallSign)

class FurnaceDispenserChest:
    blocktypes = [
        alphaMaterials.Furnace.ID,
        alphaMaterials.LitFurnace.ID,
        alphaMaterials.Dispenser.ID,
        alphaMaterials.Chest.ID,
    ]
    East = 2
    West = 3
    North = 4
    South = 5
genericFlipRotation(FurnaceDispenserChest)

class Pumpkin:
    blocktypes = [
        alphaMaterials.Pumpkin.ID,
        alphaMaterials.JackOLantern.ID,
    ]

    East = 0
    South = 1
    West = 2
    North = 3
genericFlipRotation(Pumpkin)

class Rail:
    blocktypes = [alphaMaterials.Rail.ID]

    EastWest = 0
    NorthSouth = 1
    South = 2
    North = 3
    East = 4
    West = 5

    Northeast = 6
    Southeast = 7
    Southwest = 8
    Northwest = 9

def generic8wayRotation(cls):

    cls.rotateLeft = genericRotation(cls)
    cls.rotateLeft[cls.Northeast] = cls.Northwest
    cls.rotateLeft[cls.Southeast] = cls.Northeast
    cls.rotateLeft[cls.Southwest] = cls.Southeast
    cls.rotateLeft[cls.Northwest] = cls.Southwest

    cls.flipEastWest = genericEastWestFlip(cls)
    cls.flipEastWest[cls.Northeast] = cls.Northwest
    cls.flipEastWest[cls.Northwest] = cls.Northeast
    cls.flipEastWest[cls.Southwest] = cls.Southeast
    cls.flipEastWest[cls.Southeast] = cls.Southwest

    cls.flipNorthSouth = genericNorthSouthFlip(cls)
    cls.flipNorthSouth[cls.Northeast] = cls.Southeast
    cls.flipNorthSouth[cls.Southeast] = cls.Northeast
    cls.flipNorthSouth[cls.Southwest] = cls.Northwest
    cls.flipNorthSouth[cls.Northwest] = cls.Southwest
    rotationClasses.append(cls)

generic8wayRotation(Rail)
Rail.rotateLeft[Rail.NorthSouth] = Rail.EastWest
Rail.rotateLeft[Rail.EastWest] = Rail.NorthSouth

def applyBit(apply):
    def _applyBit(class_or_array):
        if hasattr(class_or_array, "rotateLeft"):
            for a in (class_or_array.flipEastWest,
                      class_or_array.flipNorthSouth,
                      class_or_array.rotateLeft):
                apply(a)
        else:
            array = class_or_array
            apply(array)

    return _applyBit

@applyBit
def applyBit8(array):
    array[8:16] = array[0:8] | 0x8

@applyBit
def applyBit4(array):
    array[4:8] = array[0:4] | 0x4
    array[12:16] = array[8:12] | 0x4

@applyBit
def applyBits48(array):
    array[4:8] = array[0:4] | 0x4
    array[8:16] = array[0:8] | 0x8

applyThrownBit = applyBit8

class PoweredDetectorRail(Rail):
    blocktypes = [alphaMaterials.PoweredRail.ID, alphaMaterials.DetectorRail.ID]
PoweredDetectorRail.rotateLeft = genericRotation(PoweredDetectorRail)

PoweredDetectorRail.rotateLeft[PoweredDetectorRail.NorthSouth] = PoweredDetectorRail.EastWest
PoweredDetectorRail.rotateLeft[PoweredDetectorRail.EastWest] = PoweredDetectorRail.NorthSouth

PoweredDetectorRail.flipEastWest = genericEastWestFlip(PoweredDetectorRail)
PoweredDetectorRail.flipNorthSouth = genericNorthSouthFlip(PoweredDetectorRail)
applyThrownBit(PoweredDetectorRail)
rotationClasses.append(PoweredDetectorRail)

class Lever:
    blocktypes = [alphaMaterials.Lever.ID]
    ThrownBit = 0x8
    South = 1
    North = 2
    West = 3
    East = 4
    EastWest = 5
    NorthSouth = 6
Lever.rotateLeft = genericRotation(Lever)
Lever.rotateLeft[Lever.NorthSouth] = Lever.EastWest
Lever.rotateLeft[Lever.EastWest] = Lever.NorthSouth
Lever.flipEastWest = genericEastWestFlip(Lever)
Lever.flipNorthSouth = genericNorthSouthFlip(Lever)
applyThrownBit(Lever)
rotationClasses.append(Lever)

class Button:
    blocktypes = [alphaMaterials.Button.ID]
    PressedBit = 0x8
    South = 1
    North = 2
    West = 3
    East = 4
Button.rotateLeft = genericRotation(Button)
Button.flipEastWest = genericEastWestFlip(Button)
Button.flipNorthSouth = genericNorthSouthFlip(Button)
applyThrownBit(Button)
rotationClasses.append(Button)

class SignPost:
    blocktypes = [alphaMaterials.Sign.ID]
    #west is 0, increasing clockwise

    rotateLeft = arange(16, dtype='uint8')
    rotateLeft -= 4
    rotateLeft &= 0xf

    flipEastWest = arange(16, dtype='uint8')
    flipNorthSouth = arange(16, dtype='uint8')
    pass

rotationClasses.append(SignPost)

class Bed:
    blocktypes = [alphaMaterials.Bed.ID]
    West = 0
    North = 1
    East = 2
    South = 3

genericFlipRotation(Bed)
applyBit8(Bed)
applyBit4(Bed)

class Door:
    blocktypes = [
        alphaMaterials.IronDoor.ID,
        alphaMaterials.WoodenDoor.ID,
    ]
    TopHalfBit = 0x8
    SwungCCWBit = 0x4

    Northeast = 0
    Southeast = 1
    Southwest = 2
    Northwest = 3

    rotateLeft = arange(16, dtype='uint8')

Door.rotateLeft[Door.Northeast] = Door.Northwest
Door.rotateLeft[Door.Southeast] = Door.Northeast
Door.rotateLeft[Door.Southwest] = Door.Southeast
Door.rotateLeft[Door.Northwest] = Door.Southwest

applyBit4(Door.rotateLeft)

#when flipping horizontally, swing the doors so they at least look the same

Door.flipEastWest = arange(16, dtype='uint8')
Door.flipEastWest[Door.Northeast] = Door.Northwest
Door.flipEastWest[Door.Northwest] = Door.Northeast
Door.flipEastWest[Door.Southwest] = Door.Southeast
Door.flipEastWest[Door.Southeast] = Door.Southwest
Door.flipEastWest[4:8] = Door.flipEastWest[0:4]
Door.flipEastWest[0:4] = Door.flipEastWest[4:8] | 0x4
Door.flipEastWest[8:16] = Door.flipEastWest[0:8] | 0x8

Door.flipNorthSouth = arange(16, dtype='uint8')
Door.flipNorthSouth[Door.Northeast] = Door.Southeast
Door.flipNorthSouth[Door.Northwest] = Door.Southwest
Door.flipNorthSouth[Door.Southwest] = Door.Northwest
Door.flipNorthSouth[Door.Southeast] = Door.Northeast
Door.flipNorthSouth[4:8] = Door.flipNorthSouth[0:4]
Door.flipNorthSouth[0:4] = Door.flipNorthSouth[4:8] | 0x4
Door.flipNorthSouth[8:16] = Door.flipNorthSouth[0:8] | 0x8

rotationClasses.append(Door)

class RedstoneRepeater:
    blocktypes = [
        alphaMaterials.RedstoneRepeaterOff.ID,
        alphaMaterials.RedstoneRepeaterOn.ID,

    ]

    East = 0
    South = 1
    West = 2
    North = 3

genericFlipRotation(RedstoneRepeater)

#high bits of the repeater indicate repeater delay, and should be preserved
applyBits48(RedstoneRepeater)

class Trapdoor:
    blocktypes = [alphaMaterials.Trapdoor.ID]

    West = 0
    East = 1
    South = 2
    North = 3

genericFlipRotation(Trapdoor)
applyOpenedBit = applyBit4
applyOpenedBit(Trapdoor)

class PistonBody:
    blocktypes = [alphaMaterials.StickyPiston.ID, alphaMaterials.Piston.ID]

    Down = 0
    Up = 1
    East = 2
    West = 3
    North = 4
    South = 5

genericFlipRotation(PistonBody)
applyPistonBit = applyBit8
applyPistonBit(PistonBody)

class PistonHead(PistonBody):
    blocktypes = [alphaMaterials.PistonHead.ID]
rotationClasses.append(PistonHead)

class Vines:
    blocktypes = [alphaMaterials.Vines.ID]

    WestBit = 1
    NorthBit = 2
    EastBit = 4
    SouthBit = 8

    rotateLeft = arange(16, dtype='uint8')
    flipEastWest = arange(16, dtype='uint8')
    flipNorthSouth = arange(16, dtype='uint8')

"""
Value     Description     Textures
0     Fleshy piece     Pores on all sides
1     Corner piece     Cap texture on top, directions 1 (cloud direction) and 2 (sunrise)
2     Side piece     Cap texture on top and direction 2 (sunrise)
3     Corner piece     Cap texture on top, directions 2 (sunrise) and 3 (cloud origin)
4     Side piece     Cap texture on top and direction 1 (cloud direction)
5     Top piece     Cap texture on top
6     Side piece     Cap texture on top and direction 3 (cloud origin)
7     Corner piece     Cap texture on top, directions 0 (sunset) and 1 (cloud direction)
8     Side piece     Cap texture on top and direction 0 (sunset)
9     Corner piece     Cap texture on top, directions 3 (cloud origin) and 0 (sunset)
10     Stem piece     Stem texture on all four sides, pores on top and bottom
"""

class HugeMushroom:
    blocktypes = [alphaMaterials.HugeRedMushroom.ID, alphaMaterials.HugeBrownMushroom.ID]
    Northeast = 1
    East = 2
    Southeast = 3
    South = 6
    Southwest = 9
    West = 8
    Northwest = 7
    North = 4

generic8wayRotation(HugeMushroom)

#Hmm... Since each bit is a direction, we can rotate by shifting!
Vines.rotateLeft = 0xf & ((Vines.rotateLeft >> 1) | (Vines.rotateLeft << 3))
# Wherever each bit is set, clear it and set the opposite bit
EastWestBits = (Vines.EastBit | Vines.WestBit)
Vines.flipEastWest[(Vines.flipEastWest & EastWestBits) > 0] ^= EastWestBits

NorthSouthBits = (Vines.NorthBit | Vines.SouthBit)
Vines.flipNorthSouth[(Vines.flipNorthSouth & NorthSouthBits) > 0] ^= NorthSouthBits

rotationClasses.append(Vines)

def masterRotationTable(attrname):
    # compute a 256x16 table mapping each possible blocktype/data combination to
    # the resulting data when the block is rotated
    table = zeros((256, 16), dtype='uint8')
    table[:] = arange(16, dtype='uint8')
    for cls in rotationClasses:
        if hasattr(cls, attrname):
            blocktable = getattr(cls, attrname)
            for blocktype in cls.blocktypes:
                table[blocktype] = blocktable

    return table

def rotationTypeTable():
    table = {}
    for cls in rotationClasses:
        for b in cls.blocktypes:
            table[b] = cls

    return table

class BlockRotation:
    rotateLeft = masterRotationTable("rotateLeft")
    flipEastWest = masterRotationTable("flipEastWest")
    flipNorthSouth = masterRotationTable("flipNorthSouth")
    flipVertical = masterRotationTable("flipVertical")
    typeTable = rotationTypeTable()

def SameRotationType(blocktype1, blocktype2):
    #use different default values for typeTable.get() to make it return false when neither blocktype is present
    return BlockRotation.typeTable.get(blocktype1.ID) == BlockRotation.typeTable.get(blocktype2.ID, BlockRotation)

def FlipVertical(blocks, data):
    data[:] = BlockRotation.flipVertical[blocks, data]

def FlipNorthSouth(blocks, data):
    data[:] = BlockRotation.flipNorthSouth[blocks, data]

def FlipEastWest(blocks, data):
    data[:] = BlockRotation.flipEastWest[blocks, data]

def RotateLeft(blocks, data):
    data[:] = BlockRotation.rotateLeft[blocks, data]
<commit_msg>Add rotation for upside down stairs.<commit_after>from materials import alphaMaterials
from numpy import arange, zeros

def genericVerticalFlip(cls):
    rotation = arange(16, dtype='uint8')
    if hasattr(cls, "Up") and hasattr(cls, "Down"):
        rotation[cls.Up] = cls.Down
        rotation[cls.Down] = cls.Up

    if hasattr(cls, "TopNorth") and hasattr(cls, "TopWest") and hasattr(cls, "TopSouth") and hasattr(cls, "TopEast"):
        rotation[cls.North] = cls.TopNorth
        rotation[cls.West] = cls.TopWest
        rotation[cls.South] = cls.TopSouth
        rotation[cls.East] = cls.TopEast
        rotation[cls.TopNorth] = cls.North
        rotation[cls.TopWest] = cls.West
        rotation[cls.TopSouth] = cls.South
        rotation[cls.TopEast] = cls.East

    return rotation

def genericRotation(cls):
    rotation = arange(16, dtype='uint8')
    rotation[cls.North] = cls.West
    rotation[cls.West] = cls.South
    rotation[cls.South] = cls.East
    rotation[cls.East] = cls.North
    if hasattr(cls, "TopNorth") and hasattr(cls, "TopWest") and hasattr(cls, "TopSouth") and hasattr(cls, "TopEast"):
        rotation[cls.TopNorth] = cls.TopWest
        rotation[cls.TopWest] = cls.TopSouth
        rotation[cls.TopSouth] = cls.TopEast
        rotation[cls.TopEast] = cls.TopNorth

    return rotation

def genericEastWestFlip(cls):
    rotation = arange(16, dtype='uint8')
    rotation[cls.West] = cls.East
    rotation[cls.East] = cls.West
    if hasattr(cls, "TopWest") and hasattr(cls, "TopEast"):
        rotation[cls.TopWest] = cls.TopEast
        rotation[cls.TopEast] = cls.TopWest

    return rotation

def genericNorthSouthFlip(cls):
    rotation = arange(16, dtype='uint8')
    rotation[cls.South] = cls.North
    rotation[cls.North] = cls.South
    if hasattr(cls, "TopNorth") and hasattr(cls, "TopSouth"):
        rotation[cls.TopSouth] = cls.TopNorth
        rotation[cls.TopNorth] = cls.TopSouth

    return rotation

rotationClasses = []

def genericFlipRotation(cls):
    cls.rotateLeft = genericRotation(cls)

    cls.flipVertical = genericVerticalFlip(cls)
    cls.flipEastWest = genericEastWestFlip(cls)
    cls.flipNorthSouth = genericNorthSouthFlip(cls)
    rotationClasses.append(cls)

class Torch:
    blocktypes = [
        alphaMaterials.Torch.ID,
        alphaMaterials.RedstoneTorchOn.ID,
        alphaMaterials.RedstoneTorchOff.ID,
    ]

    South = 1
    North = 2
    West = 3
    East = 4

genericFlipRotation(Torch)

class Ladder:
    blocktypes = [alphaMaterials.Ladder.ID]

    East = 2
    West = 3
    North = 4
    South = 5
genericFlipRotation(Ladder)

class Stair:
    blocktypes = [b.ID for b in alphaMaterials.AllStairs]

    South = 0
    North = 1
    West = 2
    East = 3
    TopSouth = 4
    TopNorth = 5
    TopWest = 6
    TopEast = 7
genericFlipRotation(Stair)

class WallSign:
    blocktypes = [alphaMaterials.WallSign.ID]

    East = 2
    West = 3
    North = 4
    South = 5
genericFlipRotation(WallSign)

class FurnaceDispenserChest:
    blocktypes = [
        alphaMaterials.Furnace.ID,
        alphaMaterials.LitFurnace.ID,
        alphaMaterials.Dispenser.ID,
        alphaMaterials.Chest.ID,
    ]
    East = 2
    West = 3
    North = 4
    South = 5
genericFlipRotation(FurnaceDispenserChest)

class Pumpkin:
    blocktypes = [
        alphaMaterials.Pumpkin.ID,
        alphaMaterials.JackOLantern.ID,
    ]

    East = 0
    South = 1
    West = 2
    North = 3
genericFlipRotation(Pumpkin)

class Rail:
    blocktypes = [alphaMaterials.Rail.ID]

    EastWest = 0
    NorthSouth = 1
    South = 2
    North = 3
    East = 4
    West = 5

    Northeast = 6
    Southeast = 7
    Southwest = 8
    Northwest = 9

def generic8wayRotation(cls):

    cls.rotateLeft = genericRotation(cls)
    cls.rotateLeft[cls.Northeast] = cls.Northwest
    cls.rotateLeft[cls.Southeast] = cls.Northeast
    cls.rotateLeft[cls.Southwest] = cls.Southeast
    cls.rotateLeft[cls.Northwest] = cls.Southwest

    cls.flipEastWest = genericEastWestFlip(cls)
    cls.flipEastWest[cls.Northeast] = cls.Northwest
    cls.flipEastWest[cls.Northwest] = cls.Northeast
    cls.flipEastWest[cls.Southwest] = cls.Southeast
    cls.flipEastWest[cls.Southeast] = cls.Southwest

    cls.flipNorthSouth = genericNorthSouthFlip(cls)
    cls.flipNorthSouth[cls.Northeast] = cls.Southeast
    cls.flipNorthSouth[cls.Southeast] = cls.Northeast
    cls.flipNorthSouth[cls.Southwest] = cls.Northwest
    cls.flipNorthSouth[cls.Northwest] = cls.Southwest
    rotationClasses.append(cls)

generic8wayRotation(Rail)
Rail.rotateLeft[Rail.NorthSouth] = Rail.EastWest
Rail.rotateLeft[Rail.EastWest] = Rail.NorthSouth

def applyBit(apply):
    def _applyBit(class_or_array):
        if hasattr(class_or_array, "rotateLeft"):
            for a in (class_or_array.flipEastWest,
                      class_or_array.flipNorthSouth,
                      class_or_array.rotateLeft):
                apply(a)
        else:
            array = class_or_array
            apply(array)

    return _applyBit

@applyBit
def applyBit8(array):
    array[8:16] = array[0:8] | 0x8

@applyBit
def applyBit4(array):
    array[4:8] = array[0:4] | 0x4
    array[12:16] = array[8:12] | 0x4

@applyBit
def applyBits48(array):
    array[4:8] = array[0:4] | 0x4
    array[8:16] = array[0:8] | 0x8

applyThrownBit = applyBit8

class PoweredDetectorRail(Rail):
    blocktypes = [alphaMaterials.PoweredRail.ID, alphaMaterials.DetectorRail.ID]
PoweredDetectorRail.rotateLeft = genericRotation(PoweredDetectorRail)

PoweredDetectorRail.rotateLeft[PoweredDetectorRail.NorthSouth] = PoweredDetectorRail.EastWest
PoweredDetectorRail.rotateLeft[PoweredDetectorRail.EastWest] = PoweredDetectorRail.NorthSouth

PoweredDetectorRail.flipEastWest = genericEastWestFlip(PoweredDetectorRail)
PoweredDetectorRail.flipNorthSouth = genericNorthSouthFlip(PoweredDetectorRail)
applyThrownBit(PoweredDetectorRail)
rotationClasses.append(PoweredDetectorRail)

class Lever:
    blocktypes = [alphaMaterials.Lever.ID]
    ThrownBit = 0x8
    South = 1
    North = 2
    West = 3
    East = 4
    EastWest = 5
    NorthSouth = 6
Lever.rotateLeft = genericRotation(Lever)
Lever.rotateLeft[Lever.NorthSouth] = Lever.EastWest
Lever.rotateLeft[Lever.EastWest] = Lever.NorthSouth
Lever.flipEastWest = genericEastWestFlip(Lever)
Lever.flipNorthSouth = genericNorthSouthFlip(Lever)
applyThrownBit(Lever)
rotationClasses.append(Lever)

class Button:
    blocktypes = [alphaMaterials.Button.ID]
    PressedBit = 0x8
    South = 1
    North = 2
    West = 3
    East = 4
Button.rotateLeft = genericRotation(Button)
Button.flipEastWest = genericEastWestFlip(Button)
Button.flipNorthSouth = genericNorthSouthFlip(Button)
applyThrownBit(Button)
rotationClasses.append(Button)

class SignPost:
    blocktypes = [alphaMaterials.Sign.ID]
    #west is 0, increasing clockwise

    rotateLeft = arange(16, dtype='uint8')
    rotateLeft -= 4
    rotateLeft &= 0xf

    flipEastWest = arange(16, dtype='uint8')
    flipNorthSouth = arange(16, dtype='uint8')
    pass

rotationClasses.append(SignPost)

class Bed:
    blocktypes = [alphaMaterials.Bed.ID]
    West = 0
    North = 1
    East = 2
    South = 3

genericFlipRotation(Bed)
applyBit8(Bed)
applyBit4(Bed)

class Door:
    blocktypes = [
        alphaMaterials.IronDoor.ID,
        alphaMaterials.WoodenDoor.ID,
    ]
    TopHalfBit = 0x8
    SwungCCWBit = 0x4

    Northeast = 0
    Southeast = 1
    Southwest = 2
    Northwest = 3

    rotateLeft = arange(16, dtype='uint8')

Door.rotateLeft[Door.Northeast] = Door.Northwest
Door.rotateLeft[Door.Southeast] = Door.Northeast
Door.rotateLeft[Door.Southwest] = Door.Southeast
Door.rotateLeft[Door.Northwest] = Door.Southwest

applyBit4(Door.rotateLeft)

#when flipping horizontally, swing the doors so they at least look the same

Door.flipEastWest = arange(16, dtype='uint8')
Door.flipEastWest[Door.Northeast] = Door.Northwest
Door.flipEastWest[Door.Northwest] = Door.Northeast
Door.flipEastWest[Door.Southwest] = Door.Southeast
Door.flipEastWest[Door.Southeast] = Door.Southwest
Door.flipEastWest[4:8] = Door.flipEastWest[0:4]
Door.flipEastWest[0:4] = Door.flipEastWest[4:8] | 0x4
Door.flipEastWest[8:16] = Door.flipEastWest[0:8] | 0x8

Door.flipNorthSouth = arange(16, dtype='uint8')
Door.flipNorthSouth[Door.Northeast] = Door.Southeast
Door.flipNorthSouth[Door.Northwest] = Door.Southwest
Door.flipNorthSouth[Door.Southwest] = Door.Northwest
Door.flipNorthSouth[Door.Southeast] = Door.Northeast
Door.flipNorthSouth[4:8] = Door.flipNorthSouth[0:4]
Door.flipNorthSouth[0:4] = Door.flipNorthSouth[4:8] | 0x4
Door.flipNorthSouth[8:16] = Door.flipNorthSouth[0:8] | 0x8

rotationClasses.append(Door)

class RedstoneRepeater:
    blocktypes = [
        alphaMaterials.RedstoneRepeaterOff.ID,
        alphaMaterials.RedstoneRepeaterOn.ID,

    ]

    East = 0
    South = 1
    West = 2
    North = 3

genericFlipRotation(RedstoneRepeater)

#high bits of the repeater indicate repeater delay, and should be preserved
applyBits48(RedstoneRepeater)

class Trapdoor:
    blocktypes = [alphaMaterials.Trapdoor.ID]

    West = 0
    East = 1
    South = 2
    North = 3

genericFlipRotation(Trapdoor)
applyOpenedBit = applyBit4
applyOpenedBit(Trapdoor)

class PistonBody:
    blocktypes = [alphaMaterials.StickyPiston.ID, alphaMaterials.Piston.ID]

    Down = 0
    Up = 1
    East = 2
    West = 3
    North = 4
    South = 5

genericFlipRotation(PistonBody)
applyPistonBit = applyBit8
applyPistonBit(PistonBody)

class PistonHead(PistonBody):
    blocktypes = [alphaMaterials.PistonHead.ID]
rotationClasses.append(PistonHead)

class Vines:
    blocktypes = [alphaMaterials.Vines.ID]

    WestBit = 1
    NorthBit = 2
    EastBit = 4
    SouthBit = 8

    rotateLeft = arange(16, dtype='uint8')
    flipEastWest = arange(16, dtype='uint8')
    flipNorthSouth = arange(16, dtype='uint8')

"""
Value     Description     Textures
0     Fleshy piece     Pores on all sides
1     Corner piece     Cap texture on top, directions 1 (cloud direction) and 2 (sunrise)
2     Side piece     Cap texture on top and direction 2 (sunrise)
3     Corner piece     Cap texture on top, directions 2 (sunrise) and 3 (cloud origin)
4     Side piece     Cap texture on top and direction 1 (cloud direction)
5     Top piece     Cap texture on top
6     Side piece     Cap texture on top and direction 3 (cloud origin)
7     Corner piece     Cap texture on top, directions 0 (sunset) and 1 (cloud direction)
8     Side piece     Cap texture on top and direction 0 (sunset)
9     Corner piece     Cap texture on top, directions 3 (cloud origin) and 0 (sunset)
10     Stem piece     Stem texture on all four sides, pores on top and bottom
"""

class HugeMushroom:
    blocktypes = [alphaMaterials.HugeRedMushroom.ID, alphaMaterials.HugeBrownMushroom.ID]
    Northeast = 1
    East = 2
    Southeast = 3
    South = 6
    Southwest = 9
    West = 8
    Northwest = 7
    North = 4

generic8wayRotation(HugeMushroom)

#Hmm... Since each bit is a direction, we can rotate by shifting!
Vines.rotateLeft = 0xf & ((Vines.rotateLeft >> 1) | (Vines.rotateLeft << 3))
# Wherever each bit is set, clear it and set the opposite bit
EastWestBits = (Vines.EastBit | Vines.WestBit)
Vines.flipEastWest[(Vines.flipEastWest & EastWestBits) > 0] ^= EastWestBits

NorthSouthBits = (Vines.NorthBit | Vines.SouthBit)
Vines.flipNorthSouth[(Vines.flipNorthSouth & NorthSouthBits) > 0] ^= NorthSouthBits

rotationClasses.append(Vines)

def masterRotationTable(attrname):
    # compute a 256x16 table mapping each possible blocktype/data combination to
    # the resulting data when the block is rotated
    table = zeros((256, 16), dtype='uint8')
    table[:] = arange(16, dtype='uint8')
    for cls in rotationClasses:
        if hasattr(cls, attrname):
            blocktable = getattr(cls, attrname)
            for blocktype in cls.blocktypes:
                table[blocktype] = blocktable

    return table

def rotationTypeTable():
    table = {}
    for cls in rotationClasses:
        for b in cls.blocktypes:
            table[b] = cls

    return table

class BlockRotation:
    rotateLeft = masterRotationTable("rotateLeft")
    flipEastWest = masterRotationTable("flipEastWest")
    flipNorthSouth = masterRotationTable("flipNorthSouth")
    flipVertical = masterRotationTable("flipVertical")
    typeTable = rotationTypeTable()

def SameRotationType(blocktype1, blocktype2):
    #use different default values for typeTable.get() to make it return false when neither blocktype is present
    return BlockRotation.typeTable.get(blocktype1.ID) == BlockRotation.typeTable.get(blocktype2.ID, BlockRotation)

def FlipVertical(blocks, data):
    data[:] = BlockRotation.flipVertical[blocks, data]

def FlipNorthSouth(blocks, data):
    data[:] = BlockRotation.flipNorthSouth[blocks, data]

def FlipEastWest(blocks, data):
    data[:] = BlockRotation.flipEastWest[blocks, data]

def RotateLeft(blocks, data):
    data[:] = BlockRotation.rotateLeft[blocks, data]
<|endoftext|>


<commit_before>import argparse
import importlib
import os
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand

from django_excel_to_model.field_tools import get_valid_excel_field_name
from django_excel_to_model.file_readers.csv_reader import CsvFile
from django_excel_to_model.file_readers.data_source_factory import DataSourceFactory
from django_excel_to_model.management.commands.utils.bulk_inserter import BulkInserter
from django_excel_to_model.management.commands.utils.counter import Counter
from django_excel_to_model.models import ExcelImportTask
from django_excel_to_model.reader import ExcelFile, XlsbFile

try:
    from pinax.eventlog.models import log
except ImportError:
    log = None

class MandatoryColumnMissing(Exception):
    pass

class DictTranslator(object):
    # noinspection PyMethodMayBeStatic
    def translate(self, item_dict):
        for ignored in ["__invalid"]:
            if ignored in item_dict:
                del item_dict["__invalid"]
        return item_dict

class ExcelFileFromClassImporter(object):
    def __init__(self, class_instance, sheet_numbered_from_1=1):
        super(ExcelFileFromClassImporter, self).__init__()
        self.model_module = importlib.import_module(class_instance.__module__)
        self.class_instance = class_instance
        self.translator = DictTranslator()
        self.sheet_numbered_from_1 = sheet_numbered_from_1
        self.inserter = BulkInserter(self.class_instance)
        self.mandatory_column_headers = None

    def import_excel(self, full_path, header_row_numbered_from_1=1, first_import_row_numbered_from_1=2, count=1000):
        filename = os.path.basename(full_path)
        data_source = DataSourceFactory(full_path).get_data_source(
            sheet_index_numbered_from_0=self.sheet_numbered_from_1 - 1,
            header_row_start_from_0=header_row_numbered_from_1 - 1)
        count = min(data_source.get_total_rows(), count)
        c = Counter(count)

        self.validate_existence_of_mandatory_columns(data_source)

        column_to_db_field_mapping = self._get_column_to_db_field_mapping(data_source)

        for item_info_dict in data_source.enumerate_mapped(column_to_db_field_mapping,
                                                           start_row_numbered_from_0=first_import_row_numbered_from_1 - 1):
            # print item_info_dict
            self.translator.translate(item_info_dict)
            item_info_dict["data_import_id"] = filename
            self.inserter.insert(item_info_dict)
            # If count = 1, when 1 processed, cnt will become 0
            c.decrease()
            if c.is_equal_or_below(0):
                self.commit_and_log(filename)
                return 0
        self.commit_and_log(filename)
        return -1

    def _get_column_to_db_field_mapping(self, data_source):
        column_to_db_field_mapping = {}
        for column_name in data_source.get_headers():
            if column_name in self.model_module.mapping:
                column_to_db_field_mapping[column_name] = self.model_module.mapping[column_name]
            else:
                converted_field_name = get_valid_excel_field_name(column_name)
                if converted_field_name in self.model_module.mapping:
                    column_to_db_field_mapping[column_name] = \
                        self.model_module.mapping[converted_field_name]
        return column_to_db_field_mapping

    def validate_existence_of_mandatory_columns(self, sheet):
        if self.mandatory_column_headers is not None:
            if not all(spreadsheet_column_header in sheet.get_headers()
                       for spreadsheet_column_header in self.mandatory_column_headers):
                raise MandatoryColumnMissing()

    def commit_and_log(self, filename):
        self.inserter.commit()
        if log is not None:
            log(
                user=None,
                action=filename,
                extra={
                    "filename": filename
                },
                obj=ContentType.objects.get_for_model(ExcelImportTask)
            )

def import_excel_according_to_model(full_path, content_type_id, header_row_numbered_from_1,
                                    first_import_row_numbered_from_1, count=1000):
    content = ContentType.objects.get(pk=content_type_id)
    e = ExcelFileFromClassImporter(content.model_class())
    e.import_excel(full_path, header_row_numbered_from_1, first_import_row_numbered_from_1, count)

class Command(BaseCommand):
    help = 'Import excel according to model info'

    def add_arguments(self, parser):
        parser.add_argument('file-path')
        parser.add_argument('content-type-id')
        parser.add_argument('header_row_numbered_from_1')
        parser.add_argument('start')
        parser.add_argument('count', nargs='?')

    def handle(self, *args, **options):
        parser = argparse.ArgumentParser(description='Import excel file according to model info')

        subparsers = parser.add_subparsers(help='sub-command help')

        # create the parser for the "a" command
        parser_import_excel_according_to_model = subparsers.add_parser(
            'import_excel_according_to_model', help='import_excel_according_to_model help')

        # parser_import_excel_according_to_model.add_argument(
        #     '--content-type-id', type=int, help='content type pk')
        # parser_import_excel_according_to_model.add_argument(
        #     '--start', default=1,
        #     help='start line for importing excel, default=1 (second line)')
        parser_import_excel_according_to_model.add_argument(
            'file-path', nargs=1, help='path of the excel file')
        parser_import_excel_according_to_model.add_argument(
            'content-type-id', nargs=1, help='content id of the model', type=int)
        parser_import_excel_according_to_model.add_argument(
            'header_row_numbered_from_1', nargs=1, help='header row number (start from 1)', type=int)
        parser_import_excel_according_to_model.add_argument(
            'start', nargs=1, help='path of the excel file', type=int)
        parser_import_excel_according_to_model.add_argument(
            'count', nargs="?", help='process line count', default=[1000], type=int)

        arg_result = parser.parse_args()
        print vars(arg_result)["file-path"][0]
        print vars(arg_result)["content-type-id"][0]
        print vars(arg_result)["start"][0]
        print vars(arg_result)["count"]

        file_path = vars(arg_result)["file-path"][0]
        content_type_id = vars(arg_result)["content-type-id"][0]
        header_row_numbered_from_1 = vars(arg_result)["header_row_numbered_from_1"][0]
        first_import_row_numbered_from_1 = vars(arg_result)["start"][0]
        count = vars(arg_result)["count"]

        return import_excel_according_to_model(
            file_path, content_type_id, header_row_numbered_from_1, first_import_row_numbered_from_1, count)
<commit_msg>Added flag to clean table before import.<commit_after>import argparse
import importlib
import os
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand

from django_excel_to_model.field_tools import get_valid_excel_field_name
from django_excel_to_model.file_readers.csv_reader import CsvFile
from django_excel_to_model.file_readers.data_source_factory import DataSourceFactory
from django_excel_to_model.management.commands.utils.bulk_inserter import BulkInserter
from django_excel_to_model.management.commands.utils.counter import Counter
from django_excel_to_model.models import ExcelImportTask
from django_excel_to_model.reader import ExcelFile, XlsbFile

try:
    from pinax.eventlog.models import log
except ImportError:
    log = None

class MandatoryColumnMissing(Exception):
    pass

class DictTranslator(object):
    # noinspection PyMethodMayBeStatic
    def translate(self, item_dict):
        for ignored in ["__invalid"]:
            if ignored in item_dict:
                del item_dict["__invalid"]
        return item_dict

class ExcelFileFromClassImporter(object):
    def __init__(self, class_instance, sheet_numbered_from_1=1):
        super(ExcelFileFromClassImporter, self).__init__()
        self.model_module = importlib.import_module(class_instance.__module__)
        self.class_instance = class_instance
        self.translator = DictTranslator()
        self.sheet_numbered_from_1 = sheet_numbered_from_1
        self.inserter = BulkInserter(self.class_instance)
        self.mandatory_column_headers = None
        self.is_clean_before_import = False

    def set_clean_before_import(self):
        self.is_clean_before_import = True

    def import_excel(self, full_path, header_row_numbered_from_1=1, first_import_row_numbered_from_1=2, count=1000):
        filename = os.path.basename(full_path)
        data_source = DataSourceFactory(full_path).get_data_source(
            sheet_index_numbered_from_0=self.sheet_numbered_from_1 - 1,
            header_row_start_from_0=header_row_numbered_from_1 - 1)
        count = min(data_source.get_total_rows(), count)
        c = Counter(count)

        self.validate_existence_of_mandatory_columns(data_source)

        if self.is_clean_before_import:
            self.class_instance.objects.all().delete()

        column_to_db_field_mapping = self._get_column_to_db_field_mapping(data_source)

        for item_info_dict in data_source.enumerate_mapped(column_to_db_field_mapping,
                                                           start_row_numbered_from_0=first_import_row_numbered_from_1 - 1):
            # print item_info_dict
            self.translator.translate(item_info_dict)
            item_info_dict["data_import_id"] = filename
            self.inserter.insert(item_info_dict)
            # If count = 1, when 1 processed, cnt will become 0
            c.decrease()
            if c.is_equal_or_below(0):
                self.commit_and_log(filename)
                return 0
        self.commit_and_log(filename)
        return -1

    def _get_column_to_db_field_mapping(self, data_source):
        column_to_db_field_mapping = {}
        for column_name in data_source.get_headers():
            if column_name in self.model_module.mapping:
                column_to_db_field_mapping[column_name] = self.model_module.mapping[column_name]
            else:
                converted_field_name = get_valid_excel_field_name(column_name)
                if converted_field_name in self.model_module.mapping:
                    column_to_db_field_mapping[column_name] = \
                        self.model_module.mapping[converted_field_name]
        return column_to_db_field_mapping

    def validate_existence_of_mandatory_columns(self, sheet):
        if self.mandatory_column_headers is not None:
            if not all(spreadsheet_column_header in sheet.get_headers()
                       for spreadsheet_column_header in self.mandatory_column_headers):
                raise MandatoryColumnMissing()

    def commit_and_log(self, filename):
        self.inserter.commit()
        if log is not None:
            log(
                user=None,
                action=filename,
                extra={
                    "filename": filename
                },
                obj=ContentType.objects.get_for_model(ExcelImportTask)
            )

def import_excel_according_to_model(full_path, content_type_id, header_row_numbered_from_1,
                                    first_import_row_numbered_from_1, count=1000):
    content = ContentType.objects.get(pk=content_type_id)
    e = ExcelFileFromClassImporter(content.model_class())
    e.import_excel(full_path, header_row_numbered_from_1, first_import_row_numbered_from_1, count)

class Command(BaseCommand):
    help = 'Import excel according to model info'

    def add_arguments(self, parser):
        parser.add_argument('file-path')
        parser.add_argument('content-type-id')
        parser.add_argument('header_row_numbered_from_1')
        parser.add_argument('start')
        parser.add_argument('count', nargs='?')

    def handle(self, *args, **options):
        parser = argparse.ArgumentParser(description='Import excel file according to model info')

        subparsers = parser.add_subparsers(help='sub-command help')

        # create the parser for the "a" command
        parser_import_excel_according_to_model = subparsers.add_parser(
            'import_excel_according_to_model', help='import_excel_according_to_model help')

        # parser_import_excel_according_to_model.add_argument(
        #     '--content-type-id', type=int, help='content type pk')
        # parser_import_excel_according_to_model.add_argument(
        #     '--start', default=1,
        #     help='start line for importing excel, default=1 (second line)')
        parser_import_excel_according_to_model.add_argument(
            'file-path', nargs=1, help='path of the excel file')
        parser_import_excel_according_to_model.add_argument(
            'content-type-id', nargs=1, help='content id of the model', type=int)
        parser_import_excel_according_to_model.add_argument(
            'header_row_numbered_from_1', nargs=1, help='header row number (start from 1)', type=int)
        parser_import_excel_according_to_model.add_argument(
            'start', nargs=1, help='path of the excel file', type=int)
        parser_import_excel_according_to_model.add_argument(
            'count', nargs="?", help='process line count', default=[1000], type=int)

        arg_result = parser.parse_args()
        print vars(arg_result)["file-path"][0]
        print vars(arg_result)["content-type-id"][0]
        print vars(arg_result)["start"][0]
        print vars(arg_result)["count"]

        file_path = vars(arg_result)["file-path"][0]
        content_type_id = vars(arg_result)["content-type-id"][0]
        header_row_numbered_from_1 = vars(arg_result)["header_row_numbered_from_1"][0]
        first_import_row_numbered_from_1 = vars(arg_result)["start"][0]
        count = vars(arg_result)["count"]

        return import_excel_according_to_model(
            file_path, content_type_id, header_row_numbered_from_1, first_import_row_numbered_from_1, count)
<|endoftext|>


<commit_before>"""
Evaluates the estimated results of the Segmentation dataset against the
ground truth (human annotated data).
"""
import jams
from joblib import Parallel, delayed
import logging
import mir_eval
import numpy as np
import os
import pandas as pd
import six
import sys

# Local stuff
import msaf
from msaf.exceptions import NoReferencesError
import msaf.input_output as io
from msaf import utils

def print_results(results):
    """Print all the results.

    Parameters
    ----------
    results: pd.DataFrame
        Dataframe with all the results
    """
    res = results.mean()
    logging.info(results["HitRate_3F"])
    logging.info(results["track_id"])
    logging.info("Results:\n%s" % res)

def compute_results(ann_inter, est_inter, ann_labels, est_labels, bins,
                    est_file):
    """Compute the results using all the available evaluations.

    Parameters
    ----------
    ann_inter : np.array
        Annotated intervals in seconds.
    est_inter : np.array
        Estimated intervals in seconds.
    ann_labels : np.array
        Annotated labels.
    est_labels : np.array
        Estimated labels
    bins : int
        Number of bins for the information gain.
    est_file : str
        Path to the output file to store results.

    Return
    ------
    results : dict
        Contains the results of all the evaluations for the given file.
        Keys are the following:
            track_id  : Name of the track
            ds_name :   Name of the data set
            HitRate_3F  :   F-measure of hit rate at 3 seconds
            HitRate_3P  :   Precision of hit rate at 3 seconds
            HitRate_3R  :   Recall of hit rate at 3 seconds
            HitRate_0.5F  :   F-measure of hit rate at 0.5 seconds
            HitRate_0.5P  :   Precision of hit rate at 0.5 seconds
            HitRate_0.5R  :   Recall of hit rate at 0.5 seconds
            HitRate_t3F  :   F-measure of hit rate at 3 seconds (trimmed)
            HitRate_t3P  :   Precision of hit rate at 3 seconds (trimmed)
            HitRate_t3F  :   Recall of hit rate at 3 seconds (trimmed)
            HitRate_t0.5F  :   F-measure of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5P  :   Precision of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5R  :   Recall of hit rate at 0.5 seconds (trimmed)
            DevA2E  :   Median deviation of annotation to estimation
            DevE2A  :   Median deviation of estimation to annotation
            D   :   Information gain
            PWF : F-measure of pair-wise frame clustering
            PWP : Precision of pair-wise frame clustering
            PWR : Recall of pair-wise frame clustering
            Sf  : F-measure normalized entropy score
            So  : Oversegmentation normalized entropy score
            Su  : Undersegmentation normalized entropy score
    """
    res = {}

    # --Boundaries-- #
    # Hit Rate
    res["HitRate_3P"], res["HitRate_3R"], res["HitRate_3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=False)
    res["HitRate_0.5P"], res["HitRate_0.5R"], res["HitRate_0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=False)
    res["HitRate_t3P"], res["HitRate_t3R"], res["HitRate_t3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=True)
    res["HitRate_t0.5P"], res["HitRate_t0.5R"], res["HitRate_t0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=True)

    # Information gain
    res["D"] = compute_information_gain(ann_inter, est_inter, est_file,
                                        bins=bins)

    # Median Deviations
    res["DevR2E"], res["DevE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=False)
    res["DevtR2E"], res["DevtE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=True)

    # --Labels-- #
    if est_labels is not None and "-1" in est_labels:
        est_labels = None

    if est_labels is not None and len(est_labels) != 0:
        try:
            # Align labels with intervals
            ann_labels = list(ann_labels)
            est_labels = list(est_labels)
            ann_inter, ann_labels = mir_eval.util.adjust_intervals(ann_inter,
                                                                   ann_labels)
            est_inter, est_labels = mir_eval.util.adjust_intervals(
                est_inter, est_labels, t_min=0, t_max=ann_inter.max())

            # Pair-wise frame clustering
            res["PWP"], res["PWR"], res["PWF"] = mir_eval.segment.pairwise(
                ann_inter, ann_labels, est_inter, est_labels)

            # Normalized Conditional Entropies
            res["So"], res["Su"], res["Sf"] = mir_eval.segment.nce(
                ann_inter, ann_labels, est_inter, est_labels)
        except:
            logging.warning("Labeling evaluation failed in file: %s" %
                            est_file)
            return {}

    # Names
    base = os.path.basename(est_file)
    res["track_id"] = base[:-5]
    res["ds_name"] = base.split("_")[0]

    return res

def compute_gt_results(est_file, ref_file, boundaries_id, labels_id, config,
                       bins=251, annotator_id=0):
    """Computes the results by using the ground truth dataset identified by
    the annotator parameter.

    Return
    ------
    results : dict
        Dictionary of the results (see function compute_results).
    """
    try:
        if config["hier"]:
            ref_times, ref_labels, ref_levels = \
                msaf.io.read_hier_references(
                    ref_file, annotation_id=0,
                    exclude_levels=["segment_salami_function"])
        else:
            jam = jams.load(ref_file, validate=False)
            ann = jam.search(namespace='segment_.*')[annotator_id]
            ref_inter, ref_labels = ann.data.to_interval_values()
    except:
        logging.warning("No references for file: %s" % ref_file)
        return {}

    # Read estimations with correct configuration
    est_inter, est_labels = io.read_estimations(est_file, boundaries_id,
                                                labels_id, **config)
    if len(est_inter) == 0:
        logging.warning("No estimations for file: %s" % est_file)
        return {}

    # Compute the results and return
    logging.info("Evaluating %s" % os.path.basename(est_file))
    if config["hier"]:
        # Hierarchical
        assert len(est_inter) == len(est_labels), "Same number of levels " \
            "are required in the boundaries and labels for the hierarchical " \
            "evaluation."
        est_times = []
        est_labels = []

        # Sort based on how many segments per level
        est_inter = sorted(est_inter, key=lambda level: len(level))

        for inter in est_inter:
            est_times.append(msaf.utils.intervals_to_times(inter))
            # Add fake labels (hierarchical eval does not use labels --yet--)
            est_labels.append(np.ones(len(est_times[-1]) - 1) * -1)

        # Align the times
        utils.align_end_hierarchies(est_times, ref_times, thres=1)

        # To intervals
        est_hier = [utils.times_to_intervals(times) for times in est_times]
        ref_hier = [utils.times_to_intervals(times) for times in ref_times]

        # Compute evaluations
        res = {}
        res["t_recall10"], res["t_precision10"], res["t_measure10"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=10)
        res["t_recall15"], res["t_precision15"], res["t_measure15"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=15)

        res["track_id"] = os.path.basename(est_file)[:-5]
        return res
    else:
        # Flat
        return compute_results(ref_inter, est_inter, ref_labels, est_labels,
                               bins, est_file)

def compute_information_gain(ann_inter, est_inter, est_file, bins):
    """Computes the information gain of the est_file from the annotated
    intervals and the estimated intervals."""
    ann_times = utils.intervals_to_times(ann_inter)
    est_times = utils.intervals_to_times(est_inter)
    try:
        D = mir_eval.beat.information_gain(ann_times, est_times, bins=bins)
    except:
        logging.warning("Couldn't compute the Information Gain for file "
                        "%s" % est_file)
        D = 0
    return D

def process_track(file_struct, boundaries_id, labels_id, config,
                  annotator_id=0):
    """Processes a single track.

    Parameters
    ----------
    file_struct : object (FileStruct) or str
        File struct or full path of the audio file to be evaluated.
    boundaries_id : str
        Identifier of the boundaries algorithm.
    labels_id : str
        Identifier of the labels algorithm.
    config : dict
        Configuration of the algorithms to be evaluated.
    annotator_id : int
        Number identifiying the annotator.

    Returns
    -------
    one_res : dict
        Dictionary of the results (see function compute_results).
    """
    # Convert to file_struct if string is passed
    if isinstance(file_struct, six.string_types):
        file_struct = io.FileStruct(file_struct)

    est_file = file_struct.est_file
    ref_file = file_struct.ref_file

    # Sanity check
    assert os.path.basename(est_file)[:-4] == \
        os.path.basename(ref_file)[:-4], "File names are different %s --- %s" \
        % (os.path.basename(est_file)[:-4], os.path.basename(ref_file)[:-4])

    if not os.path.isfile(ref_file):
        raise NoReferencesError("Reference file %s does not exis. You must "
                                "have annotated references to run "
                                "evaluations." % ref_file)

    # TODO: Better exception handling
    try:
        one_res = compute_gt_results(est_file, ref_file, boundaries_id,
                                     labels_id, config,
                                     annotator_id=annotator_id)
    except:
        logging.warning("Could not compute evaluations for %s. Error: %s" %
                        (est_file, sys.exc_info()[1]))
        one_res = []

    return one_res

def get_results_file_name(boundaries_id, labels_id, config, ds_name,
                          annotator_id):
    """Based on the config and the dataset, get the file name to store the
    results."""
    if ds_name == "*":
        ds_name = "All"
    utils.ensure_dir(msaf.results_dir)
    file_name = os.path.join(msaf.results_dir, "results_%s" % ds_name)
    file_name += "_boundsE%s_labelsE%s" % (boundaries_id, labels_id)
    file_name += "_annotatorE%d" % (annotator_id)
    sorted_keys = sorted(config.keys(), key=str.lower)
    for key in sorted_keys:
        file_name += "_%sE%s" % (key, str(config[key]).replace("/", "_"))

    # Check for max file length
    if len(file_name) > 255 - len(msaf.results_ext):
        file_name = file_name[:255 - len(msaf.results_ext)]

    return file_name + msaf.results_ext

def process(in_path, boundaries_id=msaf.DEFAULT_BOUND_ID,
            labels_id=msaf.DEFAULT_LABEL_ID, ds_name="*", annot_beats=False,
            framesync=False, feature="hpcp", hier=False, save=False,
            n_jobs=4, annotator_id=0, config=None):
    """Main process.

    Parameters
    ----------
    in_path : str
        Path to the dataset root folder.
    boundaries_id : str
        Boundaries algorithm identifier (e.g. siplca, cnmf)
    labels_id : str
        Labels algorithm identifier (e.g. siplca, cnmf)
    ds_name : str
        Name of the dataset to be evaluated (e.g. SALAMI). * stands for all.
    annot_beats : boolean
        Whether to use the annotated beats or not.
    framesync: str
        Whether to use framesync features or not (default: False -> beatsync)
    feature: str
        String representing the feature to be used (e.g. hpcp, mfcc, tonnetz)
    hier : bool
        Whether to compute a hierarchical or flat segmentation.
    save: boolean
        Whether to save the results into the SQLite database.
    n_jobs: int
        Number of processes to run in parallel. Only available in collection
        mode.
    annotator_id : int
        Number identifiying the annotator.
    config: dict
        Dictionary containing custom configuration parameters for the
        algorithms.  If None, the default parameters are used.

    Return
    ------
    results : pd.DataFrame
        DataFrame containing the evaluations for each file.
    """

    # Set up configuration based on algorithms parameters
    if config is None:
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

    # Hierarchical segmentation
    config["hier"] = hier

    # Remove actual features
    config.pop("features", None)

    # Get out file in case we want to save results
    out_file = get_results_file_name(boundaries_id, labels_id, config, ds_name,
                                     annotator_id)

    # All evaluations
    results = pd.DataFrame()

    if os.path.isfile(in_path):
        # Single File mode
        evals = [process_track(in_path, boundaries_id, labels_id, config,
                               annotator_id=annotator_id)]
    else:
        # Collection mode
        # If out_file already exists, do not compute new results
        if os.path.exists(out_file):
            logging.info("Results already exists, reading from file %s" %
                         out_file)
            results = pd.read_csv(out_file)
            print_results(results)
            return results

        # Get files
        file_structs = io.get_dataset_files(in_path, ds_name)

        logging.info("Evaluating %d tracks..." % len(file_structs))

        # Evaluate in parallel
        evals = Parallel(n_jobs=n_jobs)(delayed(process_track)(
            file_struct, boundaries_id, labels_id, config,
            annotator_id=annotator_id) for file_struct in file_structs[:])

    # Aggregate evaluations in pandas format
    for e in evals:
        if e != []:
            results = results.append(e, ignore_index=True)
    logging.info("%d tracks analyzed" % len(results))

    # Print results
    print_results(results)

    # Save all results
    if save:
        logging.info("Writing results in %s" % out_file)
        results.to_csv(out_file)

    return results
<commit_msg>Removing the ds_name, explained in #25<commit_after>"""
Evaluates the estimated results of the Segmentation dataset against the
ground truth (human annotated data).
"""
import jams
from joblib import Parallel, delayed
import logging
import mir_eval
import numpy as np
import os
import pandas as pd
import six
import sys

# Local stuff
import msaf
from msaf.exceptions import NoReferencesError
import msaf.input_output as io
from msaf import utils

def print_results(results):
    """Print all the results.

    Parameters
    ----------
    results: pd.DataFrame
        Dataframe with all the results
    """
    res = results.mean()
    logging.info(results["HitRate_3F"])
    logging.info(results["track_id"])
    logging.info("Results:\n%s" % res)

def compute_results(ann_inter, est_inter, ann_labels, est_labels, bins,
                    est_file):
    """Compute the results using all the available evaluations.

    Parameters
    ----------
    ann_inter : np.array
        Annotated intervals in seconds.
    est_inter : np.array
        Estimated intervals in seconds.
    ann_labels : np.array
        Annotated labels.
    est_labels : np.array
        Estimated labels
    bins : int
        Number of bins for the information gain.
    est_file : str
        Path to the output file to store results.

    Return
    ------
    results : dict
        Contains the results of all the evaluations for the given file.
        Keys are the following:
            track_id  : Name of the track
            HitRate_3F  :   F-measure of hit rate at 3 seconds
            HitRate_3P  :   Precision of hit rate at 3 seconds
            HitRate_3R  :   Recall of hit rate at 3 seconds
            HitRate_0.5F  :   F-measure of hit rate at 0.5 seconds
            HitRate_0.5P  :   Precision of hit rate at 0.5 seconds
            HitRate_0.5R  :   Recall of hit rate at 0.5 seconds
            HitRate_t3F  :   F-measure of hit rate at 3 seconds (trimmed)
            HitRate_t3P  :   Precision of hit rate at 3 seconds (trimmed)
            HitRate_t3F  :   Recall of hit rate at 3 seconds (trimmed)
            HitRate_t0.5F  :   F-measure of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5P  :   Precision of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5R  :   Recall of hit rate at 0.5 seconds (trimmed)
            DevA2E  :   Median deviation of annotation to estimation
            DevE2A  :   Median deviation of estimation to annotation
            D   :   Information gain
            PWF : F-measure of pair-wise frame clustering
            PWP : Precision of pair-wise frame clustering
            PWR : Recall of pair-wise frame clustering
            Sf  : F-measure normalized entropy score
            So  : Oversegmentation normalized entropy score
            Su  : Undersegmentation normalized entropy score
    """
    res = {}

    # --Boundaries-- #
    # Hit Rate
    res["HitRate_3P"], res["HitRate_3R"], res["HitRate_3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=False)
    res["HitRate_0.5P"], res["HitRate_0.5R"], res["HitRate_0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=False)
    res["HitRate_t3P"], res["HitRate_t3R"], res["HitRate_t3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=True)
    res["HitRate_t0.5P"], res["HitRate_t0.5R"], res["HitRate_t0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=True)

    # Information gain
    res["D"] = compute_information_gain(ann_inter, est_inter, est_file,
                                        bins=bins)

    # Median Deviations
    res["DevR2E"], res["DevE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=False)
    res["DevtR2E"], res["DevtE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=True)

    # --Labels-- #
    if est_labels is not None and "-1" in est_labels:
        est_labels = None

    if est_labels is not None and len(est_labels) != 0:
        try:
            # Align labels with intervals
            ann_labels = list(ann_labels)
            est_labels = list(est_labels)
            ann_inter, ann_labels = mir_eval.util.adjust_intervals(ann_inter,
                                                                   ann_labels)
            est_inter, est_labels = mir_eval.util.adjust_intervals(
                est_inter, est_labels, t_min=0, t_max=ann_inter.max())

            # Pair-wise frame clustering
            res["PWP"], res["PWR"], res["PWF"] = mir_eval.segment.pairwise(
                ann_inter, ann_labels, est_inter, est_labels)

            # Normalized Conditional Entropies
            res["So"], res["Su"], res["Sf"] = mir_eval.segment.nce(
                ann_inter, ann_labels, est_inter, est_labels)
        except:
            logging.warning("Labeling evaluation failed in file: %s" %
                            est_file)
            return {}

    # Names
    base = os.path.basename(est_file)
    res["track_id"] = base[:-5]

    return res

def compute_gt_results(est_file, ref_file, boundaries_id, labels_id, config,
                       bins=251, annotator_id=0):
    """Computes the results by using the ground truth dataset identified by
    the annotator parameter.

    Return
    ------
    results : dict
        Dictionary of the results (see function compute_results).
    """
    try:
        if config["hier"]:
            ref_times, ref_labels, ref_levels = \
                msaf.io.read_hier_references(
                    ref_file, annotation_id=0,
                    exclude_levels=["segment_salami_function"])
        else:
            jam = jams.load(ref_file, validate=False)
            ann = jam.search(namespace='segment_.*')[annotator_id]
            ref_inter, ref_labels = ann.data.to_interval_values()
    except:
        logging.warning("No references for file: %s" % ref_file)
        return {}

    # Read estimations with correct configuration
    est_inter, est_labels = io.read_estimations(est_file, boundaries_id,
                                                labels_id, **config)
    if len(est_inter) == 0:
        logging.warning("No estimations for file: %s" % est_file)
        return {}

    # Compute the results and return
    logging.info("Evaluating %s" % os.path.basename(est_file))
    if config["hier"]:
        # Hierarchical
        assert len(est_inter) == len(est_labels), "Same number of levels " \
            "are required in the boundaries and labels for the hierarchical " \
            "evaluation."
        est_times = []
        est_labels = []

        # Sort based on how many segments per level
        est_inter = sorted(est_inter, key=lambda level: len(level))

        for inter in est_inter:
            est_times.append(msaf.utils.intervals_to_times(inter))
            # Add fake labels (hierarchical eval does not use labels --yet--)
            est_labels.append(np.ones(len(est_times[-1]) - 1) * -1)

        # Align the times
        utils.align_end_hierarchies(est_times, ref_times, thres=1)

        # To intervals
        est_hier = [utils.times_to_intervals(times) for times in est_times]
        ref_hier = [utils.times_to_intervals(times) for times in ref_times]

        # Compute evaluations
        res = {}
        res["t_recall10"], res["t_precision10"], res["t_measure10"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=10)
        res["t_recall15"], res["t_precision15"], res["t_measure15"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=15)

        res["track_id"] = os.path.basename(est_file)[:-5]
        return res
    else:
        # Flat
        return compute_results(ref_inter, est_inter, ref_labels, est_labels,
                               bins, est_file)

def compute_information_gain(ann_inter, est_inter, est_file, bins):
    """Computes the information gain of the est_file from the annotated
    intervals and the estimated intervals."""
    ann_times = utils.intervals_to_times(ann_inter)
    est_times = utils.intervals_to_times(est_inter)
    try:
        D = mir_eval.beat.information_gain(ann_times, est_times, bins=bins)
    except:
        logging.warning("Couldn't compute the Information Gain for file "
                        "%s" % est_file)
        D = 0
    return D

def process_track(file_struct, boundaries_id, labels_id, config,
                  annotator_id=0):
    """Processes a single track.

    Parameters
    ----------
    file_struct : object (FileStruct) or str
        File struct or full path of the audio file to be evaluated.
    boundaries_id : str
        Identifier of the boundaries algorithm.
    labels_id : str
        Identifier of the labels algorithm.
    config : dict
        Configuration of the algorithms to be evaluated.
    annotator_id : int
        Number identifiying the annotator.

    Returns
    -------
    one_res : dict
        Dictionary of the results (see function compute_results).
    """
    # Convert to file_struct if string is passed
    if isinstance(file_struct, six.string_types):
        file_struct = io.FileStruct(file_struct)

    est_file = file_struct.est_file
    ref_file = file_struct.ref_file

    # Sanity check
    assert os.path.basename(est_file)[:-4] == \
        os.path.basename(ref_file)[:-4], "File names are different %s --- %s" \
        % (os.path.basename(est_file)[:-4], os.path.basename(ref_file)[:-4])

    if not os.path.isfile(ref_file):
        raise NoReferencesError("Reference file %s does not exis. You must "
                                "have annotated references to run "
                                "evaluations." % ref_file)

    # TODO: Better exception handling
    try:
        one_res = compute_gt_results(est_file, ref_file, boundaries_id,
                                     labels_id, config,
                                     annotator_id=annotator_id)
    except:
        logging.warning("Could not compute evaluations for %s. Error: %s" %
                        (est_file, sys.exc_info()[1]))
        one_res = []

    return one_res

def get_results_file_name(boundaries_id, labels_id, config, annotator_id):
    """Based on the config and the dataset, get the file name to store the
    results."""
    utils.ensure_dir(msaf.results_dir)
    file_name = os.path.join(msaf.results_dir, "results")
    file_name += "_boundsE%s_labelsE%s" % (boundaries_id, labels_id)
    file_name += "_annotatorE%d" % (annotator_id)
    sorted_keys = sorted(config.keys(), key=str.lower)
    for key in sorted_keys:
        file_name += "_%sE%s" % (key, str(config[key]).replace("/", "_"))

    # Check for max file length
    if len(file_name) > 255 - len(msaf.results_ext):
        file_name = file_name[:255 - len(msaf.results_ext)]

    return file_name + msaf.results_ext

def process(in_path, boundaries_id=msaf.DEFAULT_BOUND_ID,
            labels_id=msaf.DEFAULT_LABEL_ID, annot_beats=False,
            framesync=False, feature="hpcp", hier=False, save=False,
            n_jobs=4, annotator_id=0, config=None):
    """Main process.

    Parameters
    ----------
    in_path : str
        Path to the dataset root folder.
    boundaries_id : str
        Boundaries algorithm identifier (e.g. siplca, cnmf)
    labels_id : str
        Labels algorithm identifier (e.g. siplca, cnmf)
    annot_beats : boolean
        Whether to use the annotated beats or not.
    framesync: str
        Whether to use framesync features or not (default: False -> beatsync)
    feature: str
        String representing the feature to be used (e.g. hpcp, mfcc, tonnetz)
    hier : bool
        Whether to compute a hierarchical or flat segmentation.
    save: boolean
        Whether to save the results into the SQLite database.
    n_jobs: int
        Number of processes to run in parallel. Only available in collection
        mode.
    annotator_id : int
        Number identifiying the annotator.
    config: dict
        Dictionary containing custom configuration parameters for the
        algorithms.  If None, the default parameters are used.

    Return
    ------
    results : pd.DataFrame
        DataFrame containing the evaluations for each file.
    """

    # Set up configuration based on algorithms parameters
    if config is None:
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

    # Hierarchical segmentation
    config["hier"] = hier

    # Remove actual features
    config.pop("features", None)

    # Get out file in case we want to save results
    out_file = get_results_file_name(boundaries_id, labels_id, config,
                                     annotator_id)

    # All evaluations
    results = pd.DataFrame()

    if os.path.isfile(in_path):
        # Single File mode
        evals = [process_track(in_path, boundaries_id, labels_id, config,
                               annotator_id=annotator_id)]
    else:
        # Collection mode
        # If out_file already exists, do not compute new results
        if os.path.exists(out_file):
            logging.info("Results already exists, reading from file %s" %
                         out_file)
            results = pd.read_csv(out_file)
            print_results(results)
            return results

        # Get files
        file_structs = io.get_dataset_files(in_path)

        logging.info("Evaluating %d tracks..." % len(file_structs))

        # Evaluate in parallel
        evals = Parallel(n_jobs=n_jobs)(delayed(process_track)(
            file_struct, boundaries_id, labels_id, config,
            annotator_id=annotator_id) for file_struct in file_structs[:])

    # Aggregate evaluations in pandas format
    for e in evals:
        if e != []:
            results = results.append(e, ignore_index=True)
    logging.info("%d tracks analyzed" % len(results))

    # Print results
    print_results(results)

    # Save all results
    if save:
        logging.info("Writing results in %s" % out_file)
        results.to_csv(out_file)

    return results
<|endoftext|>


<commit_before>"""
Send an email to settings.FEED_APPROVERS with the feeds that need to
be manually approved.
"""
from __future__ import absolute_import

from django.conf import settings
from django.core.mail import send_mail
from django.core.management.base import NoArgsCommand
from django.template import Context, Template
from ...models import Feed, PENDING_FEED

class Command(NoArgsCommand):

    def handle_noargs(self, **kwargs):
        try:
            verbosity = int(kwargs['verbosity'])
        except (KeyError, TypeError, ValueError):
            verbosity = 1

        feeds = Feed.objects.filter(approval_status=PENDING_FEED)

        if len(feeds) == 0:
            if verbosity >= 1:
                print "There are no pending feeds. Skipping the email."
            return

        email = """The following feeds are pending approval:
{% regroup feeds by feed_type as feed_grouping %}{% for group in feed_grouping %}
{{ group.grouper }} {% for feed in group.list %}
 - {{ feed.title }} ( {{ feed.feed_url }} ) {% endfor %}
{% endfor %}

To approve them, visit: http://djangoproject.com{% url admin:aggregator_feed_changelist %}
"""

        message = Template(email).render(Context({'feeds': feeds}))
        if verbosity >= 2:
            print "Pending approval email:\n"
            print message

        send_mail("django community feeds pending approval", message,
                  'nobody@djangoproject.com', settings.FEED_APPROVERS)

        if verbosity >= 1:
            print "Sent pending approval email to: %s" % (', '.join(settings.FEED_APPROVERS))
<commit_msg>Converting to use groups, rather than hard coded email addresses.<commit_after>"""
Send an email to settings.FEED_APPROVERS with the feeds that need to
be manually approved.
"""
from __future__ import absolute_import

from django.conf import settings
from django.contrib.auth.models import User
from django.core import mail
from django.core.management.base import NoArgsCommand
from django.template import Context, Template
from ...models import Feed, PENDING_FEED

class Command(NoArgsCommand):

    def handle_noargs(self, **kwargs):
        try:
            verbosity = int(kwargs['verbosity'])
        except (KeyError, TypeError, ValueError):
            verbosity = 1

        feeds = Feed.objects.filter(approval_status=PENDING_FEED)
        to_email = [x.email for x in User.objects.filter(groups__name=settings.FEED_APPROVERS_GROUP_NAME)]

        if len(feeds) == 0:
            if verbosity >= 1:
                print "There are no pending feeds. Skipping the email."
            return

        email = """The following feeds are pending approval:
{% regroup feeds by feed_type as feed_grouping %}{% for group in feed_grouping %}
{{ group.grouper }} {% for feed in group.list %}
 - {{ feed.title }} ( {{ feed.feed_url }} ) {% endfor %}
{% endfor %}

To approve them, visit: http://djangoproject.com{% url admin:aggregator_feed_changelist %}
"""

        message = Template(email).render(Context({'feeds': feeds}))
        if verbosity >= 2:
            print "Pending approval email:\n"
            print message

        mail.send_mail("django community feeds pending approval", message,
                       'nobody@djangoproject.com', to_email,
                       fail_silently=False)

        if verbosity >= 1:
            print "Sent pending approval email to: %s" % (', '.join(to_email))
<|endoftext|>