<commit_before>from random import randint
from types import ListType
import warnings

from django import template

register = template.Library()

@register.simple_tag(takes_context=True)
def dfp_header(context):
    warnings.warn("""Please migrate your templates to include {% dfp_footer %} \
near the end of the document body. dfp_header is marked for deprecation.""")
    secure = context['request'].is_secure() and 's' or ''
    result = """

    """ % secure
    return result

@register.simple_tag(takes_context=True)
def dfp_footer(context):
    result = """

"""

    return result

@register.tag
def dfp_tag(parser, token):
    tokens = token.split_contents()[1:]
    if len(tokens) < 5:
        raise template.TemplateSyntaxError(
            'menu tag requires arguments slot_name, width, height, targeting_key, targeting_value1, targeting_value2, ...'
        )
    li = tokens[:4]
    li.append(tokens[4:])
    return DfpTagNode(*li)

class DfpTagNode(template.Node):

    def __init__(self, slot_name, width, height, targeting_key, targeting_values):
        self.slot_name = template.Variable(slot_name)
        self.width = template.Variable(width)
        self.height = template.Variable(height)
        self.targeting_key = template.Variable(targeting_key)
        self.targeting_values = []
        for v in targeting_values:
            self.targeting_values.append(template.Variable(v))

    def render(self, context):
        slot_name = self.slot_name.resolve(context)
        width = self.width.resolve(context)
        height = self.height.resolve(context)
        targeting_key = self.targeting_key.resolve(context)
        targeting_values = []
        for v in self.targeting_values:
            resolved = v.resolve(context)
            if isinstance(resolved, ListType):
                targeting_values.extend(resolved)
            else:
                targeting_values.append(resolved)
        rand_id = randint(0, 2000000000)
        di = {
            'rand_id': rand_id,
            'slot_name': slot_name,
            'width': width,
            'height': height,
            'targeting_key': targeting_key,
            'targeting_values_attr': '|'.join(targeting_values),
            'targeting_values_param': ', '.join(['"'+v+'"' for v in targeting_values])
        }
        return """

""" % di
<commit_msg>Use actual values<commit_after>from random import randint
from types import ListType
import warnings

from django import template

register = template.Library()

@register.simple_tag(takes_context=True)
def dfp_header(context):
    warnings.warn("""Please migrate your templates to include {% dfp_footer %} \
near the end of the document body. dfp_header is marked for deprecation.""")
    secure = context['request'].is_secure() and 's' or ''
    result = """

    """ % secure
    return result

@register.simple_tag(takes_context=True)
def dfp_footer(context):
    result = """

"""

    return result

@register.tag
def dfp_tag(parser, token):
    tokens = token.split_contents()[1:]
    if len(tokens) < 5:
        raise template.TemplateSyntaxError(
            'menu tag requires arguments slot_name, width, height, targeting_key, targeting_value1, targeting_value2, ...'
        )
    li = tokens[:4]
    li.append(tokens[4:])
    return DfpTagNode(*li)

class DfpTagNode(template.Node):

    def __init__(self, slot_name, width, height, targeting_key, targeting_values):
        self.slot_name = template.Variable(slot_name)
        self.width = template.Variable(width)
        self.height = template.Variable(height)
        self.targeting_key = template.Variable(targeting_key)
        self.targeting_values = []
        for v in targeting_values:
            self.targeting_values.append(template.Variable(v))

    def render(self, context):
        slot_name = self.slot_name.resolve(context)
        width = self.width.resolve(context)
        height = self.height.resolve(context)
        targeting_key = self.targeting_key.resolve(context)
        targeting_values = []
        for v in self.targeting_values:
            resolved = v.resolve(context)
            if isinstance(resolved, ListType):
                targeting_values.extend(resolved)
            else:
                targeting_values.append(resolved)
        rand_id = randint(0, 2000000000)
        di = {
            'rand_id': rand_id,
            'slot_name': slot_name,
            'width': width,
            'height': height,
            'targeting_key': targeting_key,
            'targeting_values_attr': '|'.join(targeting_values),
            'targeting_values_param': ', '.join(['"'+v+'"' for v in targeting_values])
        }
        return """

""" % di
<|endoftext|>


<commit_before>#
# Copyright (c) 2004-2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import os
import re
import stat
import string
import xml.dom.minidom
import zipfile
import gzip as gzip_module
import bz2

from conary import rpmhelper
from conary.lib import debhelper
from conary.lib import elf
from conary.lib import javadeps
from conary.lib import util

class Magic(object):
    __slots__ = ['path', 'basedir', 'contents', 'name']
    # The file type is a generic string for a specific file type
    def __init__(self, path, basedir):
	self.path = path
	self.basedir = basedir
        if not hasattr(self, 'contents'):
            self.contents = {}
	self.name = self.__class__.__name__

class ELF(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
        fullpath = basedir+path
	self.contents['stripped'] = elf.stripped(fullpath)
        if self.__class__ is ELF:
            # ar doesn't deal with hasDebug or RPATH
            self.contents['hasDebug'] = elf.hasDebug(fullpath)
            self.contents['RPATH'] = elf.getRPATH(fullpath)
            self.contents['Type'] = elf.getType(fullpath)
	requires, provides = elf.inspect(fullpath)
        # Filter None abi flags
        requires = [ x for x in requires
                     if x[0] != 'abi' or x[2][0] is not None ]
        self.contents['requires'] = requires
        self.contents['provides'] = provides
        for req in requires:
            if req[0] == 'abi':
                self.contents['abi'] = req[1:]
                self.contents['isnset'] = req[2][1]
        for prov in provides:
            if prov[0] == 'soname':
                self.contents['soname'] = prov[1]

class ar(ELF):
    def __init__(self, path, basedir='', buffer=''):
	ELF.__init__(self, path, basedir)
	# no point in looking for __.SYMDEF because GNU ar always keeps
	# symbol table up to date
        # ar archives, like ELF files, are investigated by our elf module.
        # We do still want to be able to distinguish between them via magic,
        # thus the two classes.

class tar(Magic):
    def __init__(self, path, basedir = '', buffer = ''):
        Magic.__init__(self, path, basedir)
        self.contents['GNU'] = (buffer[257:265] == 'ustar  \0')

class gzip(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
	if buffer[3] == '\x08':
	    self.contents['name'] = _string(buffer[10:])
	if buffer[8] == '\x02':
	    self.contents['compression'] = '9'
	else:
	    self.contents['compression'] = '1'

class tar_gz(gzip, tar):
    def __init__(self, path, basedir = '', gzipBuffer = '', tarBuffer = ''):
        gzip.__init__(self, path, basedir = basedir, buffer = gzipBuffer)
        tar.__init__(self, path, basedir = basedir, buffer = tarBuffer)

class bzip(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
	self.contents['compression'] = buffer[3]

class tar_bz2(bzip, tar):
    def __init__(self, path, basedir = '', bzipBuffer = '', tarBuffer = ''):
        bzip.__init__(self, path, basedir = basedir, buffer = bzipBuffer)
        tar.__init__(self, path, basedir = basedir, buffer = tarBuffer)

class changeset(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)

class deb(Magic):
    "Debian package"
    _tagMap = dict([
        (debhelper.NAME, 'name'),
        (debhelper.VERSION, 'version'),
        (debhelper.RELEASE, 'release'),
        (debhelper.SUMMARY, 'summary'),
        (debhelper.DESCRIPTION, 'description'),
    ])
    def __init__(self, path, basedir):
        Magic.__init__(self, path, basedir)
        fullPath = basedir + path
        f = file(fullPath)
        h = debhelper.DebianPackageHeader(f)
        for dtag, cstr in self._tagMap.items():
            self.contents[cstr] = h[dtag]

class jar(Magic):
    def __init__(self, path, basedir='', zipFileObj = None, fileList = []):
	Magic.__init__(self, path, basedir)
        self.contents['files'] = filesMap = {}
        self.contents['provides'] = set()
        self.contents['requires'] = set()

        if zipFileObj is None:
            return

        try:
            for name in fileList:
                contents = zipFileObj.read(name)
                if not _javaMagic(contents):
                    continue
                prov, req = javadeps.getDeps(contents)
                filesMap[name] = (prov, req)
                if prov:
                    self.contents['provides'].add(prov)
                if req:
                    self.contents['requires'].update(req)
        except (IOError, zipfile.BadZipfile):
            # zipfile raises IOError on some malformed zip files
            pass

class WAR(Magic):
    _xmlMetadataFile = "WEB-INF/web.xml"
    def __init__(self, path, basedir='', zipFileObj = None, fileList = []):
        Magic.__init__(self, path, basedir)
        if zipFileObj is None:
            raise ValueError("Expected a Zip file object")
        # Get the contents of the deployment descriptor
        ddcontent = zipFileObj.read(self._xmlMetadataFile)
        try:
            dom = xml.dom.minidom.parseString(ddcontent)
        except Exception, e:
            # Error parsing the XML, move on
            return
        # Grab data from the DOM
        val = dom.getElementsByTagName('display-name')
        if val:
            self.contents['displayName'] = self._getNodeData(val[0])
        val = dom.getElementsByTagName('description')
        if val:
            self.contents['description'] = self._getNodeData(val[0])
        dom.unlink()

    @staticmethod
    def _getNodeData(node):
        node.normalize()
        if not node.hasChildNodes():
            return ''
        return node.childNodes[0].data

class EAR(WAR):
    _xmlMetadataFile = "META-INF/application.xml"

class ZIP(Magic):
    def __init__(self, path, basedir='', zipFileObj = None, fileList = []):
	Magic.__init__(self, path, basedir)

class java(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
        fullpath = basedir+path
        prov, req = javadeps.getDeps(file(fullpath).read())
        if prov:
            self.contents['provides'] = set([prov])
        if req:
            self.contents['requires'] = req
        self.contents['files'] = { path : (prov, req) }

class script(Magic):
    interpreterRe = re.compile(r'^#!\s*([^\s]*)')
    lineRe = re.compile(r'^#!\s*(.*)')
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
        m = self.interpreterRe.match(buffer)
        self.contents['interpreter'] = m.group(1)
        m = self.lineRe.match(buffer)
        self.contents['line'] = m.group(1)

class ltwrapper(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)

class CIL(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)

class RPM(Magic):
    _tagMap = [
        ("name",    rpmhelper.NAME, str),
        ("version", rpmhelper.VERSION, str),
        ("release", rpmhelper.RELEASE, str),
        ("epoch",   rpmhelper.EPOCH, int),
        ("arch",    rpmhelper.ARCH, str),
        ("summary", rpmhelper.SUMMARY, str),
        ("description", rpmhelper.DESCRIPTION, str),
        ("license", rpmhelper.LICENSE, str),
    ]
    def __init__(self, path, basedir=''):
	Magic.__init__(self, path, basedir)
        try:
            f = file(path)
        except:
            return None
        # Convert list of objects to simple types
        hdr = rpmhelper.readHeader(f)
        for key, tagName, valType in self._tagMap:
            val = hdr.get(tagName, None)
            if isinstance(val, list):
                if not val:
                    val = None
                else:
                    val = val[0]
            if val is not None:
                if valType == int:
                    val = int(val)
                elif valType == str:
                    val = str(val)
            self.contents[key] = val
        self.contents['isSource'] = hdr.isSource

def _javaMagic(b):
    if len(b) > 4 and b[0:4] == "\xCA\xFE\xBA\xBE":
        return True
    return False

def _tarMagic(b):
    return len(b) > 262 and b[257:262] == 'ustar'

def magic(path, basedir=''):
    """
    Returns a magic class with information about the file mentioned
    """
    if basedir and not basedir.endswith('/'):
	basedir += '/'

    n = basedir+path
    if not util.exists(n) or not util.isregular(n):
	return None

    oldmode = None
    mode = os.lstat(n)[stat.ST_MODE]
    if (mode & 0400) != 0400:
        oldmode = mode
        os.chmod(n, mode | 0400)

    f = file(n)
    if oldmode is not None:
        os.chmod(n, oldmode)

    b = f.read(4096)
    f.close()

    if len(b) > 4 and b[0] == '\x7f' and b[1:4] == "ELF":
	return ELF(path, basedir, b)
    elif len(b) > 14 and b[0:14] == '!<arch>\ndebian':
	return deb(path, basedir)
    elif len(b) > 7 and b[0:7] == "!<arch>":
	return ar(path, basedir, b)
    elif len(b) > 2 and b[0] == '\x1f' and b[1] == '\x8b':
        try:
            uncompressedBuffer = gzip_module.GzipFile(n).read(4096)
            if _tarMagic(uncompressedBuffer):
                return tar_gz(path, basedir, b, uncompressedBuffer)
        except IOError:
            # gzip raises IOError instead of any module specific errors
            pass
        return gzip(path, basedir, b)
    elif len(b) > 3 and b[0:3] == "BZh":
        try:
            uncompressedBuffer = bz2.BZ2File(n).read(4096)
            if _tarMagic(uncompressedBuffer):
                return tar_bz2(path, basedir, b, uncompressedBuffer)
        except IOError:
            # bz2 raises IOError instead of any module specific errors
            pass
        return bzip(path, basedir, b)
    elif len(b) > 4 and b[0:4] == "\xEA\x3F\x81\xBB":
	return changeset(path, basedir, b)
    elif len(b) > 4 and b[0:4] == "PK\x03\x04":
        # Zip file. Peek inside the file to extract the file list
        try:
            zf = zipfile.ZipFile(n)
            namelist = set(i.filename for i in zf.infolist()
                         if not i.filename.endswith('/') and i.file_size > 0)
        except (IOError, zipfile.BadZipfile):
            # zipfile raises IOError on some malformed zip files
            # We are producing a dummy jar or ZIP with no contents
            if path.endswith('.jar'):
                return jar(path, basedir)
            return ZIP(path, basedir)
        except RuntimeError:
            # not a proper zip archive -- likely a .car archive CNY-2871
            namelist = None
        if namelist is not None:
            if 'META-INF/application.xml' in namelist:
                return EAR(path, basedir, zipFileObj = zf, fileList = namelist)
            elif 'WEB-INF/web.xml' in namelist:
                return WAR(path, basedir, zipFileObj = zf, fileList = namelist)
            elif 'META-INF/MANIFEST.MF' in namelist:
                return jar(path, basedir, zipFileObj = zf, fileList = namelist)
            #elif path.endswith('.par'):
            #    perl archive
            else:
                return ZIP(path, basedir, zipFileObj = zf, fileList = namelist)
    elif _javaMagic(b):
        return java(path, basedir, b)
    elif len(b) > 4 and b[0:2] == "#!":
        if b.find(
            '# This wrapper script should never be moved out of the build directory.\n'
            '# If it is, it will not operate correctly.') > 0:
            return ltwrapper(path, basedir, b)
        return script(path, basedir, _line(b))
    elif (len(b) > 130
          and b[0:2] == 'MZ'
          and b[78:117] == "This program cannot be run in DOS mode."
          and b[128:130] == "PE"):
        # FIXME - this is not sufficient to detect a CIL file this
        # will match all PE executables.  See ECMA-335, partition ii,
        # section 25
        return CIL(path, basedir, b)
    elif (len(b) > 4 and b[:4] == "\xed\xab\xee\xdb"):
        return RPM(path, basedir)
    elif _tarMagic(b):
        return tar(path, basedir, b)

    return None

class magicCache(dict):
    def __init__(self, basedir=''):
	self.basedir = basedir
    def __getitem__(self, name):
	if name not in self:
	    self[name] = magic(name, self.basedir)
	return dict.__getitem__(self, name)

# internal helpers

def _string(buffer):
    return buffer[:string.find(buffer, '\0')]

def _line(buffer):
    return buffer[:string.find(buffer, '\n')]
<commit_msg>No metadata for control files that are not .tar.gz<commit_after>#
# Copyright (c) 2004-2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import os
import re
import stat
import string
import xml.dom.minidom
import zipfile
import gzip as gzip_module
import bz2

from conary import rpmhelper
from conary.lib import debhelper
from conary.lib import elf
from conary.lib import javadeps
from conary.lib import util

class Magic(object):
    __slots__ = ['path', 'basedir', 'contents', 'name']
    # The file type is a generic string for a specific file type
    def __init__(self, path, basedir):
	self.path = path
	self.basedir = basedir
        if not hasattr(self, 'contents'):
            self.contents = {}
	self.name = self.__class__.__name__

class ELF(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
        fullpath = basedir+path
	self.contents['stripped'] = elf.stripped(fullpath)
        if self.__class__ is ELF:
            # ar doesn't deal with hasDebug or RPATH
            self.contents['hasDebug'] = elf.hasDebug(fullpath)
            self.contents['RPATH'] = elf.getRPATH(fullpath)
            self.contents['Type'] = elf.getType(fullpath)
	requires, provides = elf.inspect(fullpath)
        # Filter None abi flags
        requires = [ x for x in requires
                     if x[0] != 'abi' or x[2][0] is not None ]
        self.contents['requires'] = requires
        self.contents['provides'] = provides
        for req in requires:
            if req[0] == 'abi':
                self.contents['abi'] = req[1:]
                self.contents['isnset'] = req[2][1]
        for prov in provides:
            if prov[0] == 'soname':
                self.contents['soname'] = prov[1]

class ar(ELF):
    def __init__(self, path, basedir='', buffer=''):
	ELF.__init__(self, path, basedir)
	# no point in looking for __.SYMDEF because GNU ar always keeps
	# symbol table up to date
        # ar archives, like ELF files, are investigated by our elf module.
        # We do still want to be able to distinguish between them via magic,
        # thus the two classes.

class tar(Magic):
    def __init__(self, path, basedir = '', buffer = ''):
        Magic.__init__(self, path, basedir)
        self.contents['GNU'] = (buffer[257:265] == 'ustar  \0')

class gzip(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
	if buffer[3] == '\x08':
	    self.contents['name'] = _string(buffer[10:])
	if buffer[8] == '\x02':
	    self.contents['compression'] = '9'
	else:
	    self.contents['compression'] = '1'

class tar_gz(gzip, tar):
    def __init__(self, path, basedir = '', gzipBuffer = '', tarBuffer = ''):
        gzip.__init__(self, path, basedir = basedir, buffer = gzipBuffer)
        tar.__init__(self, path, basedir = basedir, buffer = tarBuffer)

class bzip(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
	self.contents['compression'] = buffer[3]

class tar_bz2(bzip, tar):
    def __init__(self, path, basedir = '', bzipBuffer = '', tarBuffer = ''):
        bzip.__init__(self, path, basedir = basedir, buffer = bzipBuffer)
        tar.__init__(self, path, basedir = basedir, buffer = tarBuffer)

class changeset(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)

class deb(Magic):
    "Debian package"
    _tagMap = dict([
        (debhelper.NAME, 'name'),
        (debhelper.VERSION, 'version'),
        (debhelper.RELEASE, 'release'),
        (debhelper.SUMMARY, 'summary'),
        (debhelper.DESCRIPTION, 'description'),
    ])
    def __init__(self, path, basedir):
        Magic.__init__(self, path, basedir)
        fullPath = basedir + path
        f = file(fullPath)
        try:
            h = debhelper.DebianPackageHeader(f)
        except debhelper.Error:
            # Probably the control file was missing; ignore the error, we
            # don't know how to extract metadata
            return

        for dtag, cstr in self._tagMap.items():
            self.contents[cstr] = h[dtag]

class jar(Magic):
    def __init__(self, path, basedir='', zipFileObj = None, fileList = []):
	Magic.__init__(self, path, basedir)
        self.contents['files'] = filesMap = {}
        self.contents['provides'] = set()
        self.contents['requires'] = set()

        if zipFileObj is None:
            return

        try:
            for name in fileList:
                contents = zipFileObj.read(name)
                if not _javaMagic(contents):
                    continue
                prov, req = javadeps.getDeps(contents)
                filesMap[name] = (prov, req)
                if prov:
                    self.contents['provides'].add(prov)
                if req:
                    self.contents['requires'].update(req)
        except (IOError, zipfile.BadZipfile):
            # zipfile raises IOError on some malformed zip files
            pass

class WAR(Magic):
    _xmlMetadataFile = "WEB-INF/web.xml"
    def __init__(self, path, basedir='', zipFileObj = None, fileList = []):
        Magic.__init__(self, path, basedir)
        if zipFileObj is None:
            raise ValueError("Expected a Zip file object")
        # Get the contents of the deployment descriptor
        ddcontent = zipFileObj.read(self._xmlMetadataFile)
        try:
            dom = xml.dom.minidom.parseString(ddcontent)
        except Exception, e:
            # Error parsing the XML, move on
            return
        # Grab data from the DOM
        val = dom.getElementsByTagName('display-name')
        if val:
            self.contents['displayName'] = self._getNodeData(val[0])
        val = dom.getElementsByTagName('description')
        if val:
            self.contents['description'] = self._getNodeData(val[0])
        dom.unlink()

    @staticmethod
    def _getNodeData(node):
        node.normalize()
        if not node.hasChildNodes():
            return ''
        return node.childNodes[0].data

class EAR(WAR):
    _xmlMetadataFile = "META-INF/application.xml"

class ZIP(Magic):
    def __init__(self, path, basedir='', zipFileObj = None, fileList = []):
	Magic.__init__(self, path, basedir)

class java(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
        fullpath = basedir+path
        prov, req = javadeps.getDeps(file(fullpath).read())
        if prov:
            self.contents['provides'] = set([prov])
        if req:
            self.contents['requires'] = req
        self.contents['files'] = { path : (prov, req) }

class script(Magic):
    interpreterRe = re.compile(r'^#!\s*([^\s]*)')
    lineRe = re.compile(r'^#!\s*(.*)')
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)
        m = self.interpreterRe.match(buffer)
        self.contents['interpreter'] = m.group(1)
        m = self.lineRe.match(buffer)
        self.contents['line'] = m.group(1)

class ltwrapper(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)

class CIL(Magic):
    def __init__(self, path, basedir='', buffer=''):
	Magic.__init__(self, path, basedir)

class RPM(Magic):
    _tagMap = [
        ("name",    rpmhelper.NAME, str),
        ("version", rpmhelper.VERSION, str),
        ("release", rpmhelper.RELEASE, str),
        ("epoch",   rpmhelper.EPOCH, int),
        ("arch",    rpmhelper.ARCH, str),
        ("summary", rpmhelper.SUMMARY, str),
        ("description", rpmhelper.DESCRIPTION, str),
        ("license", rpmhelper.LICENSE, str),
    ]
    def __init__(self, path, basedir=''):
	Magic.__init__(self, path, basedir)
        try:
            f = file(path)
        except:
            return None
        # Convert list of objects to simple types
        hdr = rpmhelper.readHeader(f)
        for key, tagName, valType in self._tagMap:
            val = hdr.get(tagName, None)
            if isinstance(val, list):
                if not val:
                    val = None
                else:
                    val = val[0]
            if val is not None:
                if valType == int:
                    val = int(val)
                elif valType == str:
                    val = str(val)
            self.contents[key] = val
        self.contents['isSource'] = hdr.isSource

def _javaMagic(b):
    if len(b) > 4 and b[0:4] == "\xCA\xFE\xBA\xBE":
        return True
    return False

def _tarMagic(b):
    return len(b) > 262 and b[257:262] == 'ustar'

def magic(path, basedir=''):
    """
    Returns a magic class with information about the file mentioned
    """
    if basedir and not basedir.endswith('/'):
	basedir += '/'

    n = basedir+path
    if not util.exists(n) or not util.isregular(n):
	return None

    oldmode = None
    mode = os.lstat(n)[stat.ST_MODE]
    if (mode & 0400) != 0400:
        oldmode = mode
        os.chmod(n, mode | 0400)

    f = file(n)
    if oldmode is not None:
        os.chmod(n, oldmode)

    b = f.read(4096)
    f.close()

    if len(b) > 4 and b[0] == '\x7f' and b[1:4] == "ELF":
	return ELF(path, basedir, b)
    elif len(b) > 14 and b[0:14] == '!<arch>\ndebian':
	return deb(path, basedir)
    elif len(b) > 7 and b[0:7] == "!<arch>":
	return ar(path, basedir, b)
    elif len(b) > 2 and b[0] == '\x1f' and b[1] == '\x8b':
        try:
            uncompressedBuffer = gzip_module.GzipFile(n).read(4096)
            if _tarMagic(uncompressedBuffer):
                return tar_gz(path, basedir, b, uncompressedBuffer)
        except IOError:
            # gzip raises IOError instead of any module specific errors
            pass
        return gzip(path, basedir, b)
    elif len(b) > 3 and b[0:3] == "BZh":
        try:
            uncompressedBuffer = bz2.BZ2File(n).read(4096)
            if _tarMagic(uncompressedBuffer):
                return tar_bz2(path, basedir, b, uncompressedBuffer)
        except IOError:
            # bz2 raises IOError instead of any module specific errors
            pass
        return bzip(path, basedir, b)
    elif len(b) > 4 and b[0:4] == "\xEA\x3F\x81\xBB":
	return changeset(path, basedir, b)
    elif len(b) > 4 and b[0:4] == "PK\x03\x04":
        # Zip file. Peek inside the file to extract the file list
        try:
            zf = zipfile.ZipFile(n)
            namelist = set(i.filename for i in zf.infolist()
                         if not i.filename.endswith('/') and i.file_size > 0)
        except (IOError, zipfile.BadZipfile):
            # zipfile raises IOError on some malformed zip files
            # We are producing a dummy jar or ZIP with no contents
            if path.endswith('.jar'):
                return jar(path, basedir)
            return ZIP(path, basedir)
        except RuntimeError:
            # not a proper zip archive -- likely a .car archive CNY-2871
            namelist = None
        if namelist is not None:
            if 'META-INF/application.xml' in namelist:
                return EAR(path, basedir, zipFileObj = zf, fileList = namelist)
            elif 'WEB-INF/web.xml' in namelist:
                return WAR(path, basedir, zipFileObj = zf, fileList = namelist)
            elif 'META-INF/MANIFEST.MF' in namelist:
                return jar(path, basedir, zipFileObj = zf, fileList = namelist)
            #elif path.endswith('.par'):
            #    perl archive
            else:
                return ZIP(path, basedir, zipFileObj = zf, fileList = namelist)
    elif _javaMagic(b):
        return java(path, basedir, b)
    elif len(b) > 4 and b[0:2] == "#!":
        if b.find(
            '# This wrapper script should never be moved out of the build directory.\n'
            '# If it is, it will not operate correctly.') > 0:
            return ltwrapper(path, basedir, b)
        return script(path, basedir, _line(b))
    elif (len(b) > 130
          and b[0:2] == 'MZ'
          and b[78:117] == "This program cannot be run in DOS mode."
          and b[128:130] == "PE"):
        # FIXME - this is not sufficient to detect a CIL file this
        # will match all PE executables.  See ECMA-335, partition ii,
        # section 25
        return CIL(path, basedir, b)
    elif (len(b) > 4 and b[:4] == "\xed\xab\xee\xdb"):
        return RPM(path, basedir)
    elif _tarMagic(b):
        return tar(path, basedir, b)

    return None

class magicCache(dict):
    def __init__(self, basedir=''):
	self.basedir = basedir
    def __getitem__(self, name):
	if name not in self:
	    self[name] = magic(name, self.basedir)
	return dict.__getitem__(self, name)

# internal helpers

def _string(buffer):
    return buffer[:string.find(buffer, '\0')]

def _line(buffer):
    return buffer[:string.find(buffer, '\n')]
<|endoftext|>


<commit_before>#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2014-2016  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016       Marek Marczykowski <marmarek@invisiblethingslab.com>)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
#

''' A disposable vm implementation '''

import asyncio

import qubes.vm.qubesvm
import qubes.vm.appvm
import qubes.config

class DispVM(qubes.vm.qubesvm.QubesVM):
    '''Disposable VM'''

    template = qubes.VMProperty('template',
                                load_stage=4,
                                vmclass=qubes.vm.appvm.AppVM,
                                doc='AppVM, on which this DispVM is based.')

    dispid = qubes.property('dispid', type=int, write_once=True,
        clone=False,
        doc='''Internal, persistent identifier of particular DispVM.''')

    auto_cleanup = qubes.property('auto_cleanup', type=bool, default=False,
        doc='automatically remove this VM upon shutdown')

    include_in_backups = qubes.property('include_in_backups', type=bool,
        default=(lambda self: not self.auto_cleanup),
        doc='If this domain is to be included in default backup.')

    default_dispvm = qubes.VMProperty('default_dispvm',
        load_stage=4,
        allow_none=True,
        default=(lambda self: self.template),
        doc='Default VM to be used as Disposable VM for service calls.')

    def __init__(self, app, xml, *args, **kwargs):
        self.volume_config = {
            'root': {
                'name': 'root',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'private': {
                'name': 'private',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'volatile': {
                'name': 'volatile',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': True,
                'size': qubes.config.defaults['root_img_size'] +
                        qubes.config.defaults['private_img_size'],
            },
            'kernel': {
                'name': 'kernel',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': False,
            }
        }

        template = kwargs.get('template', None)

        if xml is None:
            assert template is not None

            if not getattr(template, 'template_for_dispvms', False):
                raise qubes.exc.QubesValueError(
                    'template for DispVM ({}) needs to be an AppVM with '
                    'template_for_dispvms=True'.format(template.name))

            if 'dispid' not in kwargs:
                kwargs['dispid'] = app.domains.get_new_unused_dispid()
            if 'name' not in kwargs:
                kwargs['name'] = 'disp' + str(kwargs['dispid'])

        if template is not None:
            # template is only passed if the AppVM is created, in other cases we
            # don't need to patch the volume_config because the config is
            # coming from XML, already as we need it
            for name, config in template.volume_config.items():
                # in case the template vm has more volumes add them to own
                # config
                if name not in self.volume_config:
                    self.volume_config[name] = config.copy()
                    if 'vid' in self.volume_config[name]:
                        del self.volume_config[name]['vid']

        super(DispVM, self).__init__(app, xml, *args, **kwargs)

        if xml is None:
            # by default inherit properties from the DispVM template
            proplist = [prop.__name__ for prop in template.property_list()
                if prop.clone and prop.__name__ not in ['template']]
            self_props = [prop.__name__ for prop in self.property_list()]
            self.clone_properties(template, set(proplist).intersection(
                self_props))

            self.firewall.clone(template.firewall)
            self.features.update(template.features)
            self.tags.update(template.tags)

    @qubes.events.handler('domain-load')
    def on_domain_loaded(self, event):
        ''' When domain is loaded assert that this vm has a template.
        '''  # pylint: disable=unused-argument
        assert self.template

    @qubes.events.handler('property-pre-set:template',
        'property-pre-del:template')
    def on_property_pre_set_template(self, event, name, newvalue=None,
            oldvalue=None):
        ''' Disposable VM cannot have template changed '''
        # pylint: disable=unused-argument
        raise qubes.exc.QubesValueError(self,
            'Cannot change template of Disposable VM')

    @qubes.events.handler('domain-shutdown')
    @asyncio.coroutine
    def on_domain_shutdown(self, _event, **_kwargs):
        yield from self._auto_cleanup()

    @asyncio.coroutine
    def _auto_cleanup(self):
        '''Do auto cleanup if enabled'''
        if self.auto_cleanup and self in self.app.domains:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @classmethod
    @asyncio.coroutine
    def from_appvm(cls, appvm, **kwargs):
        '''Create a new instance from given AppVM

        :param qubes.vm.appvm.AppVM appvm: template from which the VM should \
            be created
        :returns: new disposable vm

        *kwargs* are passed to the newly created VM

        >>> import qubes.vm.dispvm.DispVM
        >>> dispvm = qubes.vm.dispvm.DispVM.from_appvm(appvm).start()
        >>> dispvm.run_service('qubes.VMShell', input='firefox')
        >>> dispvm.cleanup()

        This method modifies :file:`qubes.xml` file.
        The qube returned is not started.
        '''
        if not appvm.template_for_dispvms:
            raise qubes.exc.QubesException(
                'Refusing to create DispVM out of this AppVM, because '
                'template_for_dispvms=False')
        app = appvm.app
        dispvm = app.add_new_vm(
            cls,
            template=appvm,
            auto_cleanup=True,
            **kwargs)
        yield from dispvm.create_on_disk()
        app.save()
        return dispvm

    @asyncio.coroutine
    def cleanup(self):
        '''Clean up after the DispVM

        This stops the disposable qube and removes it from the store.
        This method modifies :file:`qubes.xml` file.
        '''
        try:
            # pylint: disable=not-an-iterable
            yield from self.kill()
        except qubes.exc.QubesVMNotStartedError:
            pass
        # if auto_cleanup is set, this will be done automatically
        if not self.auto_cleanup:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @asyncio.coroutine
    def start(self, **kwargs):
        # pylint: disable=arguments-differ

        try:
            # sanity check, if template_for_dispvm got changed in the meantime
            if not self.template.template_for_dispvms:
                raise qubes.exc.QubesException(
                    'template for DispVM ({}) needs to have '
                    'template_for_dispvms=True'.format(self.template.name))

            yield from super(DispVM, self).start(**kwargs)
        except:
            # Cleanup also on failed startup
            yield from self._auto_cleanup()
            raise

    def create_qdb_entries(self):
        super().create_qdb_entries()
        self.untrusted_qdb.write('/qubes-vm-persistence', 'none')
<commit_msg>DispVM: Do not overwrite properties set explicitly<commit_after>#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2014-2016  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016       Marek Marczykowski <marmarek@invisiblethingslab.com>)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
#

''' A disposable vm implementation '''

import asyncio

import qubes.vm.qubesvm
import qubes.vm.appvm
import qubes.config

class DispVM(qubes.vm.qubesvm.QubesVM):
    '''Disposable VM'''

    template = qubes.VMProperty('template',
                                load_stage=4,
                                vmclass=qubes.vm.appvm.AppVM,
                                doc='AppVM, on which this DispVM is based.')

    dispid = qubes.property('dispid', type=int, write_once=True,
        clone=False,
        doc='''Internal, persistent identifier of particular DispVM.''')

    auto_cleanup = qubes.property('auto_cleanup', type=bool, default=False,
        doc='automatically remove this VM upon shutdown')

    include_in_backups = qubes.property('include_in_backups', type=bool,
        default=(lambda self: not self.auto_cleanup),
        doc='If this domain is to be included in default backup.')

    default_dispvm = qubes.VMProperty('default_dispvm',
        load_stage=4,
        allow_none=True,
        default=(lambda self: self.template),
        doc='Default VM to be used as Disposable VM for service calls.')

    def __init__(self, app, xml, *args, **kwargs):
        self.volume_config = {
            'root': {
                'name': 'root',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'private': {
                'name': 'private',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'volatile': {
                'name': 'volatile',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': True,
                'size': qubes.config.defaults['root_img_size'] +
                        qubes.config.defaults['private_img_size'],
            },
            'kernel': {
                'name': 'kernel',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': False,
            }
        }

        template = kwargs.get('template', None)

        if xml is None:
            assert template is not None

            if not getattr(template, 'template_for_dispvms', False):
                raise qubes.exc.QubesValueError(
                    'template for DispVM ({}) needs to be an AppVM with '
                    'template_for_dispvms=True'.format(template.name))

            if 'dispid' not in kwargs:
                kwargs['dispid'] = app.domains.get_new_unused_dispid()
            if 'name' not in kwargs:
                kwargs['name'] = 'disp' + str(kwargs['dispid'])

        if template is not None:
            # template is only passed if the AppVM is created, in other cases we
            # don't need to patch the volume_config because the config is
            # coming from XML, already as we need it
            for name, config in template.volume_config.items():
                # in case the template vm has more volumes add them to own
                # config
                if name not in self.volume_config:
                    self.volume_config[name] = config.copy()
                    if 'vid' in self.volume_config[name]:
                        del self.volume_config[name]['vid']

        super(DispVM, self).__init__(app, xml, *args, **kwargs)

        if xml is None:
            # by default inherit properties from the DispVM template
            proplist = [prop.__name__ for prop in template.property_list()
                if prop.clone and prop.__name__ not in ['template']]
            # Do not overwrite properties that have already been set to a
            # non-default value.
            self_props = [prop.__name__ for prop in self.property_list()
                if self.property_is_default(prop)]
            self.clone_properties(template, set(proplist).intersection(
                self_props))

            self.firewall.clone(template.firewall)
            self.features.update(template.features)
            self.tags.update(template.tags)

    @qubes.events.handler('domain-load')
    def on_domain_loaded(self, event):
        ''' When domain is loaded assert that this vm has a template.
        '''  # pylint: disable=unused-argument
        assert self.template

    @qubes.events.handler('property-pre-set:template',
        'property-pre-del:template')
    def on_property_pre_set_template(self, event, name, newvalue=None,
            oldvalue=None):
        ''' Disposable VM cannot have template changed '''
        # pylint: disable=unused-argument
        raise qubes.exc.QubesValueError(self,
            'Cannot change template of Disposable VM')

    @qubes.events.handler('domain-shutdown')
    @asyncio.coroutine
    def on_domain_shutdown(self, _event, **_kwargs):
        yield from self._auto_cleanup()

    @asyncio.coroutine
    def _auto_cleanup(self):
        '''Do auto cleanup if enabled'''
        if self.auto_cleanup and self in self.app.domains:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @classmethod
    @asyncio.coroutine
    def from_appvm(cls, appvm, **kwargs):
        '''Create a new instance from given AppVM

        :param qubes.vm.appvm.AppVM appvm: template from which the VM should \
            be created
        :returns: new disposable vm

        *kwargs* are passed to the newly created VM

        >>> import qubes.vm.dispvm.DispVM
        >>> dispvm = qubes.vm.dispvm.DispVM.from_appvm(appvm).start()
        >>> dispvm.run_service('qubes.VMShell', input='firefox')
        >>> dispvm.cleanup()

        This method modifies :file:`qubes.xml` file.
        The qube returned is not started.
        '''
        if not appvm.template_for_dispvms:
            raise qubes.exc.QubesException(
                'Refusing to create DispVM out of this AppVM, because '
                'template_for_dispvms=False')
        app = appvm.app
        dispvm = app.add_new_vm(
            cls,
            template=appvm,
            auto_cleanup=True,
            **kwargs)
        yield from dispvm.create_on_disk()
        app.save()
        return dispvm

    @asyncio.coroutine
    def cleanup(self):
        '''Clean up after the DispVM

        This stops the disposable qube and removes it from the store.
        This method modifies :file:`qubes.xml` file.
        '''
        try:
            # pylint: disable=not-an-iterable
            yield from self.kill()
        except qubes.exc.QubesVMNotStartedError:
            pass
        # if auto_cleanup is set, this will be done automatically
        if not self.auto_cleanup:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @asyncio.coroutine
    def start(self, **kwargs):
        # pylint: disable=arguments-differ

        try:
            # sanity check, if template_for_dispvm got changed in the meantime
            if not self.template.template_for_dispvms:
                raise qubes.exc.QubesException(
                    'template for DispVM ({}) needs to have '
                    'template_for_dispvms=True'.format(self.template.name))

            yield from super(DispVM, self).start(**kwargs)
        except:
            # Cleanup also on failed startup
            yield from self._auto_cleanup()
            raise

    def create_qdb_entries(self):
        super().create_qdb_entries()
        self.untrusted_qdb.write('/qubes-vm-persistence', 'none')
<|endoftext|>


<commit_before>"""
>>> d.markers
[
u'lg',
u'exylat',
u'ge',
u'bzns',
u'lex',
u'gn',
u'sem',
u'src',
u'crossref',
u'ps',
u'ety',
u'pc',
u'intstr',
u'gram',
u'exyeng',
u'bzn',
u'eth',
u'ct']

>>> for k, v in sorted(list(d.values('ps').items()), key=lambda i: -i[1]):
...   print k, v
...
n 975
vt 564
adv 297
vi 211
adj 140
vlab 79
pro 51
vtrimp 25
ptcl 25
interj 21
postp 12
gm 9
num 8
conn 6
onom 5
quant 1
"""
import re
from collections import defaultdict

from path import path
from clld.db.models import common
from clld.db.meta import DBSession

from dictionaria.lib.sfm import Dictionary, Entry
from dictionaria import models

POS_MAP = {
    'n': 'noun',
    'vt': 'transitive verb',
    'adv': 'adverb',
    'vi': 'intransitive verb',
    'adj': 'adjective',
    'vlab': 'verb',
    'pro': 'pronoun',
    'num': 'numeral',
    'vtrimp': 'verb',
    'ptcl': 'particle',
    'interj': 'particle',
    'postp': 'adposition',
    'gm': 'other',
    'conn': 'other',
    'onom': 'other',
    'quant': 'quantifier',
}

MARKER_MAP = {
    'lexdev': 'devanagri',
    'intstr': 'internal structure',
    'gn': 'Nepali gloss',
    'eth': 'ethnographic notes',
    'bzn': 'botanical or zoological name',
    'sem': 'semantic categories',
}

class YakkhaEntry(Entry):
    def get_meanings(self):
        for m in self.get('ge', '').split(';'):
            m = m.strip().replace('.', ' ').replace('_', ' ')
            if m:
                yield m

    def get_example(self):
        if self.get('exylat'):
            return self.get('exylat'), self.get('exyeng'), self.get('exydev')

def load(id_, data, files_dir):
    d = Dictionary(
        path(__file__).dirname().joinpath('Yakkha_WB2013_for-archive.db'),
        validate=False,
        entry_impl=YakkhaEntry,
        entry_sep='\\lex ')
    d.entries = filter(lambda r: r.get('lex'), d.entries)
    lang = data['Language'][id_]
    vocab = data['Dictionary'][id_]

    sep = re.compile('\.|,|;')
    cats = defaultdict(lambda: 0)
    #homonyms = defaultdict(list)
    for entry in d.entries:
        sem = entry.get('sem')
        if sem:
            if not sem.startswith('e.g.') and sep.search(sem):
                cat, note = [s.strip() for s in sep.split(sem, 1)]
            else:
                cat = sem.strip()
            cats[cat] += 1
    #just_one = 0
    #for cat, count in sorted(list(cats.items()), key=lambda i: i[1], reverse=True):
    #    if count > 1:
    #        print cat, count
    #    else:
    #        just_one += 1
    #print just_one, 'semantic categories(?) with just one entry'
        #homonyms[entry.get('lex')].append((entry.get('ps'), entry.get('ge')))
    #for lex, ps in homonyms.items():
    #    if len(ps) > 1:
    #        print '%s\t%s' % (lex, ','.join(p for p, g in ps))

    rel = []

    for i, row in enumerate(d.entries):
        if not row.get('ge'):
            print row.get('lex')
            continue
        w = data.add(
            models.Word, row.get('lex'),
            id='%s-%s' % (id_, i),
            name=row.get('lex'),
            description=row.get('ge'),
            dictionary=vocab)
        w.language = lang

        if row.get('crossref'):
            rel.append((row.get('lex'), row.get('crossref')))

        DBSession.flush()

        for index, pair in enumerate(row):
            key, value = pair
            if key in MARKER_MAP:
                DBSession.add(common.Unit_data(
                    object_pk=w.pk,
                    key=MARKER_MAP[key],
                    value=value,
                    ord=index))

        ex = row.get_example()
        if ex:
            s = common.Sentence(
                id='%s-%s' % (id_, i + 1),
                name=ex[0],
                language=lang,
                original_script=ex[2],
                description=ex[1])
            DBSession.add(s)
            DBSession.add(models.WordSentence(word=w, sentence=s))

        meaning_prefix = ''
        for j, name in enumerate(row.getall('ps')):
            name = POS_MAP[name]
            if name == 'verb' or ' verb' in name:
                meaning_prefix = 'to '
            elif name == 'noun':
                meaning_prefix = 'the '
            if j > 0:
                # only one part-of-speech value per entry!
                raise ValueError
            DBSession.add(common.UnitValue(
                id='pos-%s-%s' % (id_, i + 1),
                unit=w,
                unitparameter=data['UnitParameter']['pos'],
                unitdomainelement=data['UnitDomainElement'][name],
                contribution=vocab,
            ))

        for j, name in enumerate(row.get_meanings()):
            name = name.strip()
            key = '%s%s' % (meaning_prefix, name.lower())
            if key in data['Meaning']:
                meaning = data['Meaning'][key]
                vsid = '%s-%s' % (key, id_),
                if vsid in data['ValueSet']:
                    vs = data['ValueSet'][vsid]
                else:
                    vs = data.add(
                        common.ValueSet, vsid,
                        id='%s-%s' % (id_, meaning.id),
                        language=lang,
                        contribution=vocab,
                        parameter=meaning)

                DBSession.add(models.Counterpart(
                    id='%s-%s-%s' % (id_, i, j),
                    name=row.get('lx'),
                    valueset=vs,
                    word=w))

    for s, t in rel:
        if s in data['Word'] and t in data['Word']:
            DBSession.add(models.SeeAlso(
                source_pk=data['Word'][s].pk,
                target_pk=data['Word'][t].pk))
<commit_msg>recognize igt in yakkha dict<commit_after>"""
>>> d.markers
[
u'lg',
u'exylat',
u'ge',
u'bzns',
u'lex',
u'gn',
u'sem',
u'src',
u'crossref',
u'ps',
u'ety',
u'pc',
u'intstr',
u'gram',
u'exyeng',
u'bzn',
u'eth',
u'ct']

>>> for k, v in sorted(list(d.values('ps').items()), key=lambda i: -i[1]):
...   print k, v
...
n 975
vt 564
adv 297
vi 211
adj 140
vlab 79
pro 51
vtrimp 25
ptcl 25
interj 21
postp 12
gm 9
num 8
conn 6
onom 5
quant 1
"""
import re
from collections import defaultdict

from path import path
from clld.db.models import common
from clld.db.meta import DBSession

from dictionaria.lib.sfm import Dictionary, Entry
from dictionaria import models

POS_MAP = {
    'n': 'noun',
    'vt': 'transitive verb',
    'adv': 'adverb',
    'vi': 'intransitive verb',
    'adj': 'adjective',
    'vlab': 'verb',
    'pro': 'pronoun',
    'num': 'numeral',
    'vtrimp': 'verb',
    'ptcl': 'particle',
    'interj': 'particle',
    'postp': 'adposition',
    'gm': 'other',
    'conn': 'other',
    'onom': 'other',
    'quant': 'quantifier',
}

MARKER_MAP = {
    'lexdev': 'devanagri',
    'intstr': 'internal structure',
    'gn': 'Nepali gloss',
    'eth': 'ethnographic notes',
    'bzn': 'botanical or zoological name',
    'sem': 'semantic categories',
}

class YakkhaEntry(Entry):
    def igt(self, field):
        res = self.get(field)
        if res:
            return '\t'.join(res.split())

    def get_meanings(self):
        for m in self.get('ge', '').split(';'):
            m = m.strip().replace('.', ' ').replace('_', ' ')
            if m:
                yield m

    def get_example(self):
        """
        \exylat nam ayana.
        \exymph nam a-ya=na
        \exygl sun descend[3SG]-PST=NMLZ
        \exyeng The sun set.
        """
        if self.get('exylat'):
            return self.get('exylat'), self.get('exyeng'), self.get('exydev'), self.igt('exymph'), self.igt('exygl')

def load(id_, data, files_dir):
    d = Dictionary(
        path(__file__).dirname().joinpath('Yakkha_WB2013_for-archive_A-glossed.db'),
        validate=False,
        entry_impl=YakkhaEntry,
        entry_sep='\\lex ')
    d.entries = filter(lambda r: r.get('lex'), d.entries)
    lang = data['Language'][id_]
    vocab = data['Dictionary'][id_]

    sep = re.compile('\.|,|;')
    cats = defaultdict(lambda: 0)
    #homonyms = defaultdict(list)
    for entry in d.entries:
        sem = entry.get('sem')
        if sem:
            if not sem.startswith('e.g.') and sep.search(sem):
                cat, note = [s.strip() for s in sep.split(sem, 1)]
            else:
                cat = sem.strip()
            cats[cat] += 1
    #just_one = 0
    #for cat, count in sorted(list(cats.items()), key=lambda i: i[1], reverse=True):
    #    if count > 1:
    #        print cat, count
    #    else:
    #        just_one += 1
    #print just_one, 'semantic categories(?) with just one entry'
        #homonyms[entry.get('lex')].append((entry.get('ps'), entry.get('ge')))
    #for lex, ps in homonyms.items():
    #    if len(ps) > 1:
    #        print '%s\t%s' % (lex, ','.join(p for p, g in ps))

    rel = []

    for i, row in enumerate(d.entries):
        if not row.get('ge'):
            print row.get('lex')
            continue
        w = data.add(
            models.Word, row.get('lex'),
            id='%s-%s' % (id_, i),
            name=row.get('lex'),
            description=row.get('ge'),
            dictionary=vocab)
        w.language = lang

        if row.get('crossref'):
            rel.append((row.get('lex'), row.get('crossref')))

        DBSession.flush()

        for index, pair in enumerate(row):
            key, value = pair
            if key in MARKER_MAP:
                DBSession.add(common.Unit_data(
                    object_pk=w.pk,
                    key=MARKER_MAP[key],
                    value=value,
                    ord=index))

        ex = row.get_example()
        if ex:
            s = common.Sentence(
                id='%s-%s' % (id_, i + 1),
                name=ex[0],
                language=lang,
                original_script=ex[2],
                description=ex[1],
                analyzed=ex[3],
                gloss=ex[4])
            DBSession.add(s)
            DBSession.add(models.WordSentence(word=w, sentence=s))

        meaning_prefix = ''
        for j, name in enumerate(row.getall('ps')):
            name = POS_MAP[name]
            if name == 'verb' or ' verb' in name:
                meaning_prefix = 'to '
            elif name == 'noun':
                meaning_prefix = 'the '
            if j > 0:
                # only one part-of-speech value per entry!
                raise ValueError
            DBSession.add(common.UnitValue(
                id='pos-%s-%s' % (id_, i + 1),
                unit=w,
                unitparameter=data['UnitParameter']['pos'],
                unitdomainelement=data['UnitDomainElement'][name],
                contribution=vocab,
            ))

        for j, name in enumerate(row.get_meanings()):
            name = name.strip()
            key = '%s%s' % (meaning_prefix, name.lower())
            if key in data['Meaning']:
                meaning = data['Meaning'][key]
                vsid = '%s-%s' % (key, id_),
                if vsid in data['ValueSet']:
                    vs = data['ValueSet'][vsid]
                else:
                    vs = data.add(
                        common.ValueSet, vsid,
                        id='%s-%s' % (id_, meaning.id),
                        language=lang,
                        contribution=vocab,
                        parameter=meaning)

                DBSession.add(models.Counterpart(
                    id='%s-%s-%s' % (id_, i, j),
                    name=row.get('lx'),
                    valueset=vs,
                    word=w))

    for s, t in rel:
        if s in data['Word'] and t in data['Word']:
            DBSession.add(models.SeeAlso(
                source_pk=data['Word'][s].pk,
                target_pk=data['Word'][t].pk))
<|endoftext|>


<commit_before>from django.template.context import RequestContext
from django.shortcuts import render_to_response
from django.http import HttpResponse
from piston.emitters import Emitter
from api.views import display_spatial_query, display_aoi_uploader
from util.ncconv.experimental import ocg_converter

import logging
from util.ncconv.experimental.ocg_stat import OcgStat
from django.conf import settings
import time
logger = logging.getLogger(__name__)

class OpenClimateEmitter(Emitter):
    """
    Superclass for all OpenClimateGIS emitters.
    """

    def render(self,request):
        return(self._render_(request))

    def _render_(self,request):
        raise(NotImplementedError)

class IdentityEmitter(OpenClimateEmitter):
    """
    The standard Django Piston emitter does unnecessary computations when an
    emitter is searching for the raw data from its associated handler.
    """

    def construct(self):
        return self.data

class HelloWorldEmitter(OpenClimateEmitter):

    def _render_(self,request):
        names = [n['name'] for n in self.construct()]
        msg = 'Hello, World!! The climate model names are:

{0}'.format('
'.join(names))
        return HttpResponse(msg)

class HTMLEmitter(IdentityEmitter):
    """Emits an HTML representation
    """
    def _render_(self,request):

        logger.info("starting HTMLEmitter.render()...")

        c = RequestContext(request)

        template_name = request.url_args.get('template_name')
        is_collection = request.url_args.get('is_collection')

        ## return data from the construct method of the resource's handler
        try:
            data = self.construct()
            logger.debug("len(data) = {0}".format(len(data)))
        except:
            data = []
            logger.debug("data is None!")

        ## if we need the query form generate and pass accordingly
        if template_name == 'query.html':
            response = display_spatial_query(request)
        elif template_name == 'aoi_upload.html':
            response = display_aoi_uploader(request)
        else:
            response = render_to_response(
                template_name=template_name,
                dictionary={'data':data, 'is_collection':is_collection},
                context_instance=c,
            )

        logger.info("...ending HTMLEmitter.render()")

        return(response)
Emitter.register('html', HTMLEmitter, 'text/html; charset=utf-8')

class SubOcgDataEmitter(IdentityEmitter):
    __converter__ = None
    __file_ext__ = ''

    def _render_(self,request):
        logger.info("starting {0}.render()...".format(self.__converter__.__name__))
#        import ipdb;ipdb.set_trace()
        self.sub = self.construct()
        self.db = self.get_db()
        self.request = request
        if request.ocg.query.use_stat:
            st = OcgStat(self.db,
                         self.sub,
                         request.ocg.query.grouping,
                         procs=settings.MAXPROCESSES)
            st.calculate_load(self.request.ocg.query.functions)
        self.cfvar = request.ocg.simulation_output.variable.code
        self.converter = self.get_converter()
        logger.info("...ending {0}.render()...".format(self.__converter__.__name__))
        return(self.get_response())

    def get_db(self):
        return(self.sub.to_db(procs=settings.MAXPROCESSES))

    def get_converter(self):
        return(self.__converter__(self.db,
                                  self.cfvar+self.__file_ext__,
                                  use_stat=self.request.ocg.query.use_stat))

    def get_response(self):
        return(self.converter.response())

class ZippedSubOcgDataEmitter(SubOcgDataEmitter):

    def _render_(self,request):
        base_response = super(ZippedSubOcgDataEmitter,self).render(request)
        response = HttpResponse()
        response['Content-Disposition'] = 'attachment; filename={0}.zip'.\
            format(request.ocg.simulation_output.variable.code)
        response['Content-length'] = str(len(base_response))
        response['Content-Type'] = 'application/zip'
        response.write(base_response)
        return(response)

class MetacontentEmitter(SubOcgDataEmitter):

    def _render_(self,request):
        request.ocg.end_time = time.time()
        converter = ocg_converter.MetacontentConverter(request)
        response = converter.response()
        return(response)

class SqliteEmitter(ZippedSubOcgDataEmitter):
    __converter__ = ocg_converter.SqliteConverter
    __file_ext__ = ''

    def get_db(self):
        return(self.sub.as_sqlite(to_disk=True))

class ShapefileEmitter(ZippedSubOcgDataEmitter):
    """
    Emits zipped shapefile (.shz)
    """
    __converter__ = ocg_converter.ShpConverter
    __file_ext__ = '.shp'

class LinkedShapefileEmitter(ZippedSubOcgDataEmitter):
    __converter__ = ocg_converter.LinkedShpConverter
    __file_ext__ = '.lshz'

class KmlEmitter(SubOcgDataEmitter):
    """
    Emits raw KML (.kml)
    """

    __converter__ = ocg_converter.KmlConverter
    __file_ext__ = '.kml'

    def _response_(self):
        return(self.converter())

    def get_response(self):
        return(self.converter.response(self.request))

class KmzEmitter(KmlEmitter):
    """
    Subclass of KmlEmitter. Emits KML in a zipped format (.kmz)
    """

    __converter__ = ocg_converter.KmzConverter
    __file_ext__ = '.kmz'

#    def _render_(self,request):
#        logger.info("starting KmzEmitter.render()...")
#        kml = super(KmzEmitter,self).render(request)
#        iobuffer = io.BytesIO()
#        zf = zipfile.ZipFile(
#            iobuffer,
#            mode='w',
#            compression=zipfile.ZIP_DEFLATED,
#        )
#        try:
#            zf.writestr('doc.kml',kml)
#        finally:
#            zf.close()
#        iobuffer.flush()
#        zip_stream = iobuffer.getvalue()
#        iobuffer.close()
#        logger.info("...ending KmzEmitter.render()")
#        return(zip_stream)

class GeoJsonEmitter(SubOcgDataEmitter):
    """
    JSON format for geospatial data (.json)
    """
    __converter__ = ocg_converter.GeojsonConverter
    __file_ext__ = '.json'

class CsvEmitter(SubOcgDataEmitter):
    """
    Tabular CSV format. (.csv)
    """
    __converter__ = ocg_converter.CsvConverter
    __file_ext__ = '.csv'

class LinkedCsvEmitter(ZippedSubOcgDataEmitter):
    __converter__ = ocg_converter.LinkedCsvConverter
    __file_ext__ = ''

class NcEmitter(SubOcgDataEmitter):
    __converter__ = ocg_converter.NcConverter
    __file_ext__ = '.nc'

    def get_response(self):
        from util.ncconv.experimental.ocg_dataset.dataset import OcgDataset
        ds = OcgDataset(self.request.ocg.dataset_uri,**self.request.ocg.ocg_opts)
        return(self.converter.response(self.sub,ds))

Emitter.register('shz',ShapefileEmitter,'application/zip; charset=utf-8')
Emitter.register('lshz',LinkedShapefileEmitter,'application/zip; charset=utf-8')
#Emitter.unregister('json')
Emitter.register('kml',KmlEmitter,'application/vnd.google-earth.kml+xml')
Emitter.register('kmz',KmzEmitter,'application/vnd.google-earth.kmz')
Emitter.register('geojson',GeoJsonEmitter,'text/plain; charset=utf-8')
Emitter.register('csv',CsvEmitter,'text/csv; charset=utf-8')
Emitter.register('kcsv',LinkedCsvEmitter,'application/zip; charset=utf-8')
Emitter.register('sqlite',SqliteEmitter,'application/zip; charset=utf-8')
Emitter.register('nc',NcEmitter,'application/x-netcdf; charset=utf-8')
Emitter.register('meta',MetacontentEmitter,'text/plain; charset=utf-8')<commit_msg>fixed recursion error in zipped emitter.<commit_after>from django.template.context import RequestContext
from django.shortcuts import render_to_response
from django.http import HttpResponse
from piston.emitters import Emitter
from api.views import display_spatial_query, display_aoi_uploader
from util.ncconv.experimental import ocg_converter
from util.ncconv.experimental.ocg_stat import OcgStat
from django.conf import settings
import time

import logging
logger = logging.getLogger(__name__)

class OpenClimateEmitter(Emitter):
    """
    Superclass for all OpenClimateGIS emitters.
    """

    def render(self,request):
        return(self._render_(request))

    def _render_(self,request):
        raise(NotImplementedError)

class IdentityEmitter(OpenClimateEmitter):
    """
    The standard Django Piston emitter does unnecessary computations when an
    emitter is searching for the raw data from its associated handler.
    """

    def construct(self):
        return self.data

class HelloWorldEmitter(OpenClimateEmitter):

    def _render_(self,request):
        names = [n['name'] for n in self.construct()]
        msg = 'Hello, World!! The climate model names are:

{0}'.format('
'.join(names))
        return HttpResponse(msg)

class HTMLEmitter(IdentityEmitter):
    """Emits an HTML representation
    """
    def _render_(self,request):

        logger.info("starting HTMLEmitter.render()...")

        c = RequestContext(request)

        template_name = request.url_args.get('template_name')
        is_collection = request.url_args.get('is_collection')

        ## return data from the construct method of the resource's handler
        try:
            data = self.construct()
            logger.debug("len(data) = {0}".format(len(data)))
        except:
            data = []
            logger.debug("data is None!")

        ## if we need the query form generate and pass accordingly
        if template_name == 'query.html':
            response = display_spatial_query(request)
        elif template_name == 'aoi_upload.html':
            response = display_aoi_uploader(request)
        else:
            response = render_to_response(
                template_name=template_name,
                dictionary={'data':data, 'is_collection':is_collection},
                context_instance=c,
            )

        logger.info("...ending HTMLEmitter.render()")

        return(response)
Emitter.register('html', HTMLEmitter, 'text/html; charset=utf-8')

class SubOcgDataEmitter(IdentityEmitter):
    __converter__ = None
    __file_ext__ = ''

    def _render_(self,request):
        logger.info("starting {0}.render()...".format(self.__converter__.__name__))
#        import ipdb;ipdb.set_trace()
        self.sub = self.construct()
        self.db = self.get_db()
        self.request = request
        if request.ocg.query.use_stat:
            st = OcgStat(self.db,
                         self.sub,
                         request.ocg.query.grouping,
                         procs=settings.MAXPROCESSES)
            st.calculate_load(self.request.ocg.query.functions)
        self.cfvar = request.ocg.simulation_output.variable.code
        self.converter = self.get_converter()
        logger.info("...ending {0}.render()...".format(self.__converter__.__name__))
        return(self.get_response())

    def get_db(self):
        return(self.sub.to_db(procs=settings.MAXPROCESSES))

    def get_converter(self):
        return(self.__converter__(self.db,
                                  self.cfvar+self.__file_ext__,
                                  use_stat=self.request.ocg.query.use_stat))

    def get_response(self):
        return(self.converter.response())

class ZippedSubOcgDataEmitter(SubOcgDataEmitter):

    def _render_(self,request):
        base_response = super(ZippedSubOcgDataEmitter,self)._render_(request)
        response = HttpResponse()
        response['Content-Disposition'] = 'attachment; filename={0}.zip'.\
            format(request.ocg.simulation_output.variable.code)
        response['Content-length'] = str(len(base_response))
        response['Content-Type'] = 'application/zip'
        response.write(base_response)
        return(response)

class MetacontentEmitter(SubOcgDataEmitter):

    def _render_(self,request):
        request.ocg.end_time = time.time()
        converter = ocg_converter.MetacontentConverter(request)
        response = converter.response()
        return(response)

class SqliteEmitter(ZippedSubOcgDataEmitter):
    __converter__ = ocg_converter.SqliteConverter
    __file_ext__ = ''

    def get_db(self):
        return(self.sub.as_sqlite(to_disk=True))

class ShapefileEmitter(ZippedSubOcgDataEmitter):
    """
    Emits zipped shapefile (.shz)
    """
    __converter__ = ocg_converter.ShpConverter
    __file_ext__ = '.shp'

class LinkedShapefileEmitter(ZippedSubOcgDataEmitter):
    __converter__ = ocg_converter.LinkedShpConverter
    __file_ext__ = '.lshz'

class KmlEmitter(SubOcgDataEmitter):
    """
    Emits raw KML (.kml)
    """

    __converter__ = ocg_converter.KmlConverter
    __file_ext__ = '.kml'

    def _response_(self):
        return(self.converter())

    def get_response(self):
        return(self.converter.response(self.request))

class KmzEmitter(KmlEmitter):
    """
    Subclass of KmlEmitter. Emits KML in a zipped format (.kmz)
    """

    __converter__ = ocg_converter.KmzConverter
    __file_ext__ = '.kmz'

#    def _render_(self,request):
#        logger.info("starting KmzEmitter.render()...")
#        kml = super(KmzEmitter,self).render(request)
#        iobuffer = io.BytesIO()
#        zf = zipfile.ZipFile(
#            iobuffer,
#            mode='w',
#            compression=zipfile.ZIP_DEFLATED,
#        )
#        try:
#            zf.writestr('doc.kml',kml)
#        finally:
#            zf.close()
#        iobuffer.flush()
#        zip_stream = iobuffer.getvalue()
#        iobuffer.close()
#        logger.info("...ending KmzEmitter.render()")
#        return(zip_stream)

class GeoJsonEmitter(SubOcgDataEmitter):
    """
    JSON format for geospatial data (.json)
    """
    __converter__ = ocg_converter.GeojsonConverter
    __file_ext__ = '.json'

class CsvEmitter(SubOcgDataEmitter):
    """
    Tabular CSV format. (.csv)
    """
    __converter__ = ocg_converter.CsvConverter
    __file_ext__ = '.csv'

class LinkedCsvEmitter(ZippedSubOcgDataEmitter):
    __converter__ = ocg_converter.LinkedCsvConverter
    __file_ext__ = ''

class NcEmitter(SubOcgDataEmitter):
    __converter__ = ocg_converter.NcConverter
    __file_ext__ = '.nc'

    def get_response(self):
        from util.ncconv.experimental.ocg_dataset.dataset import OcgDataset
        ds = OcgDataset(self.request.ocg.dataset_uri,**self.request.ocg.ocg_opts)
        return(self.converter.response(self.sub,ds))

Emitter.register('shz',ShapefileEmitter,'application/zip; charset=utf-8')
Emitter.register('lshz',LinkedShapefileEmitter,'application/zip; charset=utf-8')
#Emitter.unregister('json')
Emitter.register('kml',KmlEmitter,'application/vnd.google-earth.kml+xml')
Emitter.register('kmz',KmzEmitter,'application/vnd.google-earth.kmz')
Emitter.register('geojson',GeoJsonEmitter,'text/plain; charset=utf-8')
Emitter.register('csv',CsvEmitter,'text/csv; charset=utf-8')
Emitter.register('kcsv',LinkedCsvEmitter,'application/zip; charset=utf-8')
Emitter.register('sqlite',SqliteEmitter,'application/zip; charset=utf-8')
Emitter.register('nc',NcEmitter,'application/x-netcdf; charset=utf-8')
Emitter.register('meta',MetacontentEmitter,'text/plain; charset=utf-8')<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import logging
import time

log = logging.getLogger(__name__)

def catch(api, eid, sid, pid):
    # Try to catch pokemon, but don't get stuck.
    attempts = 1
    while attempts < 3:
        log.info('Starting attempt %s to catch pid: %s!', attempts, pid)
        try:
            req = api.create_request()
            catch_result = req.catch_pokemon(
                encounter_id=eid,
                pokeball=1,
                normalized_reticle_size=1.950,
                spawn_point_id=sid,
                hit_pokemon=1,
                spin_modifier=1.0,
                normalized_hit_position=1.0)
            catch_result = req.check_challenge()
            catch_result = req.get_inventory()
            catch_result = req.call()

            if (catch_result is not None and 'CATCH_POKEMON' in catch_result['responses']):
                #log.info('DEBUG: %s', catch_result['responses']['CATCH_POKEMON'])
                catch_status = catch_result['responses']['CATCH_POKEMON']['status'];
                # Success!
                if catch_status == 1:
                    cpid = catch_result['responses']['CATCH_POKEMON']['captured_pokemon_id']
                    log.info('Catch attempt %s was successful for pid: %s! The cpid is %s.', attempts, pid, str(cpid))

                    # Check inventory for new pokemon id and movesets
                    iitems = catch_result['responses']['GET_INVENTORY']['inventory_delta']['inventory_items']
                    for item in iitems:
                        iidata = item['inventory_item_data']
                        if str(cpid) in str(item):
                            npid = item['inventory_item_data']['pokemon_data']['pokemon_id']
                            m1 = item['inventory_item_data']['pokemon_data']['move_1']
                            m2 = item['inventory_item_data']['pokemon_data']['move_2']

                    rv = [{'catch_status':'success', 'pid':npid, 'm1':m1, 'm2':m2}]

                    released = release(api, pid, cpid)

                    break

                # Broke free!
                if catch_status == 2:
                    log.info('Catch attempt %s failed for pid: %s. It broke free!', attempts, pid)

                # Ran away!
                if catch_status == 3:
                    log.info('Catch attempt %s failed for pid: %s. It ran away!', attempts, pid)
                    rv = [{'catch_status':'ran'}]
                    break

                # Dodged!
                if catch_status == 4:
                    log.info('Catch attempt %s failed for pid: %s. It dodged the ball!', attempts, pid)

            else:
                log.error('Catch attempt %s failed for pid: %s. The api response was empty!', attempts, pid)

        except Exception as e:
            log.error('Catch attempt %s failed for pid: %s. The api response returned an error!', attempts, pid)
            rv = [{'catch_status':'error', 'error':str(e)}]

        attempts += 1
        time.sleep(10)

    if attempts >= 3:
        log.error('Failed to catch pid: %s after %s attempts. Giving up.', pid, attempts)
        rv = [{'catch_status':'fail'}]

    return dict(data=rv)

def release(api, pid, cpid):
    try:
        log.info('Releasing pid: %s', pid)
        req = api.create_request()
        release_result = req.release_pokemon(pokemon_id=cpid)
        release_result = req.check_challenge()
        release_result = req.get_inventory()
        release_result = req.call()

        if (release_result is not None and 'RELEASE_POKEMON' in release_result['responses']):
            log.info('DEBUG: %s', release_result['responses']['RELEASE_POKEMON'])
            release_result = release_result['responses']['RELEASE_POKEMON']['result'];
            log.info('DEBUG: %s', release_result)
    except Exception as e:
        log.error(e)
        return False
    return True
<commit_msg>Ditto finder logic. (DO NOT PULL - Not finished, just testing)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import logging
import time

log = logging.getLogger(__name__)

def catch(api, eid, sid, pid):
    # Try to catch pokemon, but don't get stuck.
    attempts = 1
    while attempts < 3:
        log.info('Starting attempt %s to catch pid: %s!', attempts, pid)
        try:
            req = api.create_request()
            catch_result = req.catch_pokemon(
                encounter_id=eid,
                pokeball=1,
                normalized_reticle_size=1.950,
                spawn_point_id=sid,
                hit_pokemon=1,
                spin_modifier=1.0,
                normalized_hit_position=1.0)
            catch_result = req.check_challenge()
            catch_result = req.get_inventory()
            catch_result = req.call()

            if (catch_result is not None and 'CATCH_POKEMON' in catch_result['responses']):
                #log.info('DEBUG: %s', catch_result['responses']['CATCH_POKEMON'])
                catch_status = catch_result['responses']['CATCH_POKEMON']['status'];
                # Success!
                if catch_status == 1:
                    cpid = catch_result['responses']['CATCH_POKEMON']['captured_pokemon_id']
                    log.info('Catch attempt %s was successful for pid: %s! The cpid is %s.', attempts, pid, str(cpid))

                    # Check inventory for new pokemon id and movesets
                    iitems = catch_result['responses']['GET_INVENTORY']['inventory_delta']['inventory_items']
                    for item in iitems:
                        iidata = item['inventory_item_data']
                        if str(cpid) in str(item):
                            npid = item['inventory_item_data']['pokemon_data']['pokemon_id']
                            m1 = item['inventory_item_data']['pokemon_data']['move_1']
                            m2 = item['inventory_item_data']['pokemon_data']['move_2']

                    rv = [{'catch_status':'success', 'pid':npid, 'm1':m1, 'm2':m2}]

                    time.sleep(10)
                    released = release(api, pid, cpid)

                    break

                # Broke free!
                if catch_status == 2:
                    log.info('Catch attempt %s failed for pid: %s. It broke free!', attempts, pid)

                # Ran away!
                if catch_status == 3:
                    log.info('Catch attempt %s failed for pid: %s. It ran away!', attempts, pid)
                    rv = [{'catch_status':'ran'}]
                    break

                # Dodged!
                if catch_status == 4:
                    log.info('Catch attempt %s failed for pid: %s. It dodged the ball!', attempts, pid)

            else:
                log.error('Catch attempt %s failed for pid: %s. The api response was empty!', attempts, pid)

        except Exception as e:
            log.error('Catch attempt %s failed for pid: %s. The api response returned an error!', attempts, pid)
            rv = [{'catch_status':'error', 'error':str(e)}]

        attempts += 1
        time.sleep(10)

    if attempts >= 3:
        log.error('Failed to catch pid: %s after %s attempts. Giving up.', pid, attempts)
        rv = [{'catch_status':'fail'}]

    return dict(data=rv)

def release(api, pid, cpid):
    try:
        log.info('Releasing pid: %s', pid)
        req = api.create_request()
        release_result = req.release_pokemon(pokemon_id=cpid)
        release_result = req.check_challenge()
        release_result = req.get_inventory()
        release_result = req.call()

        if (release_result is not None and 'RELEASE_POKEMON' in release_result['responses']):
            log.info('DEBUG: %s', release_result['responses']['RELEASE_POKEMON'])
            release_result = release_result['responses']['RELEASE_POKEMON']['result'];
            log.info('DEBUG: %s', release_result)
    except Exception as e:
        log.error(e)
        return False
    return True
<|endoftext|>