<commit_before>import struct

from io import BytesIO
from struct import pack, unpack
from thriftpy2.protocol.exc import TProtocolException
from thriftpy2.thrift import TType

VERSION_MASK = -65536
VERSION_1 = -2147418112
TYPE_MASK = 0x000000FF

def pack_i8(byte):
    return struct.pack("!b", byte)

def pack_i16(i16):
    return struct.pack("!h", i16)

def pack_i32(i32):
    return struct.pack("!i", i32)

def pack_i64(i64):
    return struct.pack("!q", i64)

def pack_double(dub):
    return struct.pack("!d", dub)

def pack_string(string):
    return struct.pack("!i%ds" % len(string), len(string), string)

def unpack_i8(buffer):
    return struct.unpack("!b", buffer)[0]

def unpack_i16(buffer):
    return struct.unpack("!h", buffer)[0]

def unpack_i32(buffer):
    return struct.unpack("!i", buffer)[0]

def unpack_i64(buffer):
    return struct.unpack("!q", buffer)[0]

def unpack_double(buffer):
    return struct.unpack("!d", buffer)[0]

def write_message_begin(writer, name, ttype, seqid, strict=True):
    if strict:
        writer.write(pack_i32(VERSION_1 | ttype))
        writer.write(pack_string(name.encode("utf-8")))
    else:
        writer.write(pack_string(name.encode("utf-8")))
        writer.write(pack_i8(ttype))

    writer.write(pack_i32(seqid))

def write_field_begin(writer, ttype, fid):
    writer.write(pack_i8(ttype) + pack_i16(fid))

def write_field_stop(writer):
    writer.write(pack_i8(TType.STOP))

def write_list_begin(writer, etype, size):
    writer.write(pack_i8(etype) + pack_i32(size))

def write_map_begin(writer, ktype, vtype, size):
    writer.write(pack_i8(ktype) + pack_i8(vtype) + pack_i32(size))

def write_val(writer, ttype, val, spec=None):
    if ttype == TType.BOOL:
        if val:
            writer.write(pack_i8(1))
        else:
            writer.write(pack_i8(0))

    elif ttype == TType.BYTE:
        writer.write(pack_i8(val))

    elif ttype == TType.I16:
        writer.write(pack_i16(val))

    elif ttype == TType.I32:
        writer.write(pack_i32(val))

    elif ttype == TType.I64:
        writer.write(pack_i64(val))

    elif ttype == TType.DOUBLE:
        writer.write(pack_double(val))

    elif ttype == TType.STRING:
        if not isinstance(val, bytes):
            val = val.encode("utf-8")
        writer.write(pack_string(val))

    elif ttype == TType.SET or ttype == TType.LIST:
        if isinstance(spec, tuple):
            e_type, t_spec = spec[0], spec[1]
        else:
            e_type, t_spec = spec, None

        val_len = len(val)
        write_list_begin(writer, e_type, val_len)
        for e_val in val:
            write_val(writer, e_type, e_val, t_spec)

    elif ttype == TType.MAP:
        if isinstance(spec[0], int):
            k_type = spec[0]
            k_spec = None
        else:
            k_type, k_spec = spec[0]

        if isinstance(spec[1], int):
            v_type = spec[1]
            v_spec = None
        else:
            v_type, v_spec = spec[1]

        write_map_begin(writer, k_type, v_type, len(val))
        for k in iter(val):
            write_val(writer, k_type, k, k_spec)
            write_val(writer, v_type, val[k], v_spec)

    elif ttype == TType.STRUCT:
        for fid in iter(val.thrift_spec):
            f_spec = val.thrift_spec[fid]
            if len(f_spec) == 3:
                f_type, f_name, f_req = f_spec
                f_container_spec = None
            else:
                f_type, f_name, f_container_spec, f_req = f_spec

            v = getattr(val, f_name)
            if v is None:
                continue

            write_field_begin(writer, f_type, fid)
            write_val(writer, f_type, v, f_container_spec)
        write_field_stop(writer)

async def read_message_begin(reader, strict=True):
    data = await reader.readexactly(4)
    sz = unpack_i32(data)
    if sz < 0:
        version = sz & VERSION_MASK
        if version != VERSION_1:
            raise TProtocolException(
                type=TProtocolException.BAD_VERSION,
                message="Bad version in read_message_begin: %d" % (sz),
            )

        data = await reader.readexactly(4)
        name_sz = unpack_i32(data)
        data = await reader.readexactly(name_sz)
        name = data.decode("utf-8")
        type_ = sz & TYPE_MASK
    else:
        if strict:
            raise TProtocolException(
                type=TProtocolException.BAD_VERSION,
                message="No protocol version header",
            )

        data = await reader.readexactly(sz)
        name = data.decode("utf-8")
        data = await reader.readexactly(1)
        type_ = unpack_i8(data)

    data = await reader.readexactly(4)
    seqid = unpack_i32(data)

    return name, type_, seqid

async def read_field_begin(reader):
    data = await reader.readexactly(1)
    f_type = unpack_i8(data)
    if f_type == TType.STOP:
        return f_type, 0

    data = await reader.readexactly(2)
    return f_type, unpack_i16(data)

async def read_list_begin(reader):
    data = await reader.readexactly(1)
    e_type = unpack_i8(data)
    data = await reader.readexactly(4)
    sz = unpack_i32(data)
    return e_type, sz

async def read_map_begin(reader):
    k = await reader.readexactly(1)
    v = await reader.readexactly(1)
    k_type, v_type = unpack_i8(k), unpack_i8(v)
    data = await reader.readexactly(4)
    sz = unpack_i32(data)
    return k_type, v_type, sz

async def read_val(reader, ttype, spec=None, decode_response=True):
    if ttype == TType.BOOL:
        data = await reader.readexactly(1)
        return bool(unpack_i8(data))

    elif ttype == TType.BYTE:
        data = await reader.readexactly(1)
        return unpack_i8(data)

    elif ttype == TType.I16:
        data = await reader.readexactly(2)
        return unpack_i16(data)

    elif ttype == TType.I32:
        data = await reader.readexactly(4)
        return unpack_i32(data)

    elif ttype == TType.I64:
        data = await reader.readexactly(8)
        return unpack_i64(data)

    elif ttype == TType.DOUBLE:
        data = await reader.readexactly(8)
        return unpack_double(data)

    elif ttype == TType.STRING:
        data = await reader.readexactly(4)
        sz = unpack_i32(data)
        byte_payload = await reader.readexactly(sz)

        # Since we cannot tell if we're getting STRING or BINARY
        # if not asked not to decode, try both
        if decode_response:
            try:
                return byte_payload.decode("utf-8")
            except UnicodeDecodeError:
                pass
        return byte_payload

    elif ttype == TType.SET or ttype == TType.LIST:
        if isinstance(spec, tuple):
            v_type, v_spec = spec[0], spec[1]
        else:
            v_type, v_spec = spec, None

        result = []
        r_type, sz = await read_list_begin(reader)
        # the v_type is useless here since we already get it from spec
        if r_type != v_type:
            for _ in range(sz):
                await skip(reader, r_type)
            return []

        for i in range(sz):
            data = await read_val(reader, v_type, v_spec, decode_response)
            result.append(data)
        return result

    elif ttype == TType.MAP:
        if isinstance(spec[0], int):
            k_type = spec[0]
            k_spec = None
        else:
            k_type, k_spec = spec[0]

        if isinstance(spec[1], int):
            v_type = spec[1]
            v_spec = None
        else:
            v_type, v_spec = spec[1]

        result = {}
        sk_type, sv_type, sz = await read_map_begin(reader)
        if sk_type != k_type or sv_type != v_type:
            for _ in range(sz):
                await skip(reader, sk_type)
                await skip(reader, sv_type)
            return {}

        for i in range(sz):
            k_val = await read_val(reader, k_type, k_spec, decode_response)
            v_val = await read_val(reader, v_type, v_spec, decode_response)
            result[k_val] = v_val

        return result

    elif ttype == TType.STRUCT:
        obj = spec()
        await read_struct(reader, obj, decode_response)
        return obj

async def read_struct(reader, obj, decode_response=True):
    while True:
        f_type, fid = await read_field_begin(reader)
        if f_type == TType.STOP:
            break

        if fid not in obj.thrift_spec:
            await skip(reader, f_type)
            continue

        if len(obj.thrift_spec[fid]) == 3:
            sf_type, f_name, f_req = obj.thrift_spec[fid]
            f_container_spec = None
        else:
            sf_type, f_name, f_container_spec, f_req = obj.thrift_spec[fid]

        # it really should equal here. but since we already wasted
        # space storing the duplicate info, let's check it.
        if f_type != sf_type:
            await skip(reader, f_type)
            continue

        data = await read_val(reader, f_type, f_container_spec, decode_response)
        setattr(obj, f_name, data)

async def skip(reader, ftype):
    if ftype == TType.BOOL or ftype == TType.BYTE:
        await reader.readexactly(1)

    elif ftype == TType.I16:
        await reader.readexactly(2)

    elif ftype == TType.I32:
        await reader.readexactly(4)

    elif ftype == TType.I64:
        await reader.readexactly(8)

    elif ftype == TType.DOUBLE:
        await reader.readexactly(8)

    elif ftype == TType.STRING:
        await reader.readexactly(unpack_i32(reader.readexactly(4)))

    elif ftype == TType.SET or ftype == TType.LIST:
        v_type, sz = await read_list_begin(reader)
        for i in range(sz):
            await skip(reader, v_type)

    elif ftype == TType.MAP:
        k_type, v_type, sz = read_map_begin(reader)
        for i in range(sz):
            await skip(reader, k_type)
            await skip(reader, v_type)

    elif ftype == TType.STRUCT:
        while True:
            f_type, fid = await read_field_begin(reader)
            if f_type == TType.STOP:
                break
            await skip(reader, f_type)

class TFramedTransport:
    """Implement the Twisted framed transport protocol.

    Since most async servers use this including the python2 twisted
    bindings, this is likely of interest.
    """
    def __init__(self, base):
        self.__base = base
        self.__read_buffer = BytesIO()
        self.__write_buffer = BytesIO()

    def read(self, n=-1):
        if len(self.__read_buffer.getvalue()) == 0:
            self.readFrame()
        return self.__read_buffer.read(n)

    async def readFrame(self):
        buff = await self.__base.readexactly(4)
        sz, = unpack('!i', buff)
        self.__read_buffer = BytesIO(await self.__base.readexactly(sz))

    async def readexactly(self, n):
        now, end = self.__read_buffer.tell(), self.__read_buffer.seek(0, 2)
        remaining = end - now
        self.__read_buffer.seek(now)

        if 0 < remaining < n:
            raise IOError("Tried to read invalid amount from framed transport")
        elif remaining == 0:
            await self.readFrame()

        return self.__read_buffer.read(n)

    def write(self, val):
        self.__write_buffer.write(val)

    async def drain(self):
        wout = self.__write_buffer.getvalue()
        wsz = len(wout)
        self.__write_buffer = BytesIO()
        buf = pack("!i", wsz) + wout
        self.__base.write(buf)
        await self.__base.drain()

    def at_eof(self):
        return self.__base.at_eof()

    def close(self):
        return self.__base.close()

class TProtocol:
    """
    Base class for thrift protocols, subclass should implement some of the protocol methods,
    currently we only have :class:`TBinaryProtocol` implemented for you.
    """

    def __init__(
            self, trans, strict_read=True, strict_write=True, decode_response=True
    ):
        self.trans = trans
        self.strict_read = strict_read
        self.strict_write = strict_write
        self.decode_response = decode_response

    def skip(self, ttype):
        pass

    async def read_message_begin(self):
        pass

    async def read_message_end(self):
        pass

    def write_message_begin(self, name, ttype, seqid):
        pass

    def write_message_end(self):
        pass

    async def read_struct(self, obj):
        pass

    def write_struct(self, obj):
        pass

class TBinaryProtocol(TProtocol):
    """Binary implementation of the Thrift protocol driver."""

    def skip(self, ttype):
        skip(self.trans, ttype)

    async def read_message_begin(self):
        api, ttype, seqid = await read_message_begin(
            self.trans, strict=self.strict_read
        )
        return api, ttype, seqid

    def write_message_begin(self, name, ttype, seqid):
        write_message_begin(self.trans, name, ttype, seqid, strict=self.strict_write)

    async def read_struct(self, obj):
        data = await read_struct(self.trans, obj, self.decode_response)
        return data

    def write_struct(self, obj):
        write_val(self.trans, TType.STRUCT, obj)
<commit_msg>Rename readFrame -> read_frame<commit_after>import struct

from io import BytesIO
from struct import pack, unpack
from thriftpy2.protocol.exc import TProtocolException
from thriftpy2.thrift import TType

VERSION_MASK = -65536
VERSION_1 = -2147418112
TYPE_MASK = 0x000000FF

def pack_i8(byte):
    return struct.pack("!b", byte)

def pack_i16(i16):
    return struct.pack("!h", i16)

def pack_i32(i32):
    return struct.pack("!i", i32)

def pack_i64(i64):
    return struct.pack("!q", i64)

def pack_double(dub):
    return struct.pack("!d", dub)

def pack_string(string):
    return struct.pack("!i%ds" % len(string), len(string), string)

def unpack_i8(buffer):
    return struct.unpack("!b", buffer)[0]

def unpack_i16(buffer):
    return struct.unpack("!h", buffer)[0]

def unpack_i32(buffer):
    return struct.unpack("!i", buffer)[0]

def unpack_i64(buffer):
    return struct.unpack("!q", buffer)[0]

def unpack_double(buffer):
    return struct.unpack("!d", buffer)[0]

def write_message_begin(writer, name, ttype, seqid, strict=True):
    if strict:
        writer.write(pack_i32(VERSION_1 | ttype))
        writer.write(pack_string(name.encode("utf-8")))
    else:
        writer.write(pack_string(name.encode("utf-8")))
        writer.write(pack_i8(ttype))

    writer.write(pack_i32(seqid))

def write_field_begin(writer, ttype, fid):
    writer.write(pack_i8(ttype) + pack_i16(fid))

def write_field_stop(writer):
    writer.write(pack_i8(TType.STOP))

def write_list_begin(writer, etype, size):
    writer.write(pack_i8(etype) + pack_i32(size))

def write_map_begin(writer, ktype, vtype, size):
    writer.write(pack_i8(ktype) + pack_i8(vtype) + pack_i32(size))

def write_val(writer, ttype, val, spec=None):
    if ttype == TType.BOOL:
        if val:
            writer.write(pack_i8(1))
        else:
            writer.write(pack_i8(0))

    elif ttype == TType.BYTE:
        writer.write(pack_i8(val))

    elif ttype == TType.I16:
        writer.write(pack_i16(val))

    elif ttype == TType.I32:
        writer.write(pack_i32(val))

    elif ttype == TType.I64:
        writer.write(pack_i64(val))

    elif ttype == TType.DOUBLE:
        writer.write(pack_double(val))

    elif ttype == TType.STRING:
        if not isinstance(val, bytes):
            val = val.encode("utf-8")
        writer.write(pack_string(val))

    elif ttype == TType.SET or ttype == TType.LIST:
        if isinstance(spec, tuple):
            e_type, t_spec = spec[0], spec[1]
        else:
            e_type, t_spec = spec, None

        val_len = len(val)
        write_list_begin(writer, e_type, val_len)
        for e_val in val:
            write_val(writer, e_type, e_val, t_spec)

    elif ttype == TType.MAP:
        if isinstance(spec[0], int):
            k_type = spec[0]
            k_spec = None
        else:
            k_type, k_spec = spec[0]

        if isinstance(spec[1], int):
            v_type = spec[1]
            v_spec = None
        else:
            v_type, v_spec = spec[1]

        write_map_begin(writer, k_type, v_type, len(val))
        for k in iter(val):
            write_val(writer, k_type, k, k_spec)
            write_val(writer, v_type, val[k], v_spec)

    elif ttype == TType.STRUCT:
        for fid in iter(val.thrift_spec):
            f_spec = val.thrift_spec[fid]
            if len(f_spec) == 3:
                f_type, f_name, f_req = f_spec
                f_container_spec = None
            else:
                f_type, f_name, f_container_spec, f_req = f_spec

            v = getattr(val, f_name)
            if v is None:
                continue

            write_field_begin(writer, f_type, fid)
            write_val(writer, f_type, v, f_container_spec)
        write_field_stop(writer)

async def read_message_begin(reader, strict=True):
    data = await reader.readexactly(4)
    sz = unpack_i32(data)
    if sz < 0:
        version = sz & VERSION_MASK
        if version != VERSION_1:
            raise TProtocolException(
                type=TProtocolException.BAD_VERSION,
                message="Bad version in read_message_begin: %d" % (sz),
            )

        data = await reader.readexactly(4)
        name_sz = unpack_i32(data)
        data = await reader.readexactly(name_sz)
        name = data.decode("utf-8")
        type_ = sz & TYPE_MASK
    else:
        if strict:
            raise TProtocolException(
                type=TProtocolException.BAD_VERSION,
                message="No protocol version header",
            )

        data = await reader.readexactly(sz)
        name = data.decode("utf-8")
        data = await reader.readexactly(1)
        type_ = unpack_i8(data)

    data = await reader.readexactly(4)
    seqid = unpack_i32(data)

    return name, type_, seqid

async def read_field_begin(reader):
    data = await reader.readexactly(1)
    f_type = unpack_i8(data)
    if f_type == TType.STOP:
        return f_type, 0

    data = await reader.readexactly(2)
    return f_type, unpack_i16(data)

async def read_list_begin(reader):
    data = await reader.readexactly(1)
    e_type = unpack_i8(data)
    data = await reader.readexactly(4)
    sz = unpack_i32(data)
    return e_type, sz

async def read_map_begin(reader):
    k = await reader.readexactly(1)
    v = await reader.readexactly(1)
    k_type, v_type = unpack_i8(k), unpack_i8(v)
    data = await reader.readexactly(4)
    sz = unpack_i32(data)
    return k_type, v_type, sz

async def read_val(reader, ttype, spec=None, decode_response=True):
    if ttype == TType.BOOL:
        data = await reader.readexactly(1)
        return bool(unpack_i8(data))

    elif ttype == TType.BYTE:
        data = await reader.readexactly(1)
        return unpack_i8(data)

    elif ttype == TType.I16:
        data = await reader.readexactly(2)
        return unpack_i16(data)

    elif ttype == TType.I32:
        data = await reader.readexactly(4)
        return unpack_i32(data)

    elif ttype == TType.I64:
        data = await reader.readexactly(8)
        return unpack_i64(data)

    elif ttype == TType.DOUBLE:
        data = await reader.readexactly(8)
        return unpack_double(data)

    elif ttype == TType.STRING:
        data = await reader.readexactly(4)
        sz = unpack_i32(data)
        byte_payload = await reader.readexactly(sz)

        # Since we cannot tell if we're getting STRING or BINARY
        # if not asked not to decode, try both
        if decode_response:
            try:
                return byte_payload.decode("utf-8")
            except UnicodeDecodeError:
                pass
        return byte_payload

    elif ttype == TType.SET or ttype == TType.LIST:
        if isinstance(spec, tuple):
            v_type, v_spec = spec[0], spec[1]
        else:
            v_type, v_spec = spec, None

        result = []
        r_type, sz = await read_list_begin(reader)
        # the v_type is useless here since we already get it from spec
        if r_type != v_type:
            for _ in range(sz):
                await skip(reader, r_type)
            return []

        for i in range(sz):
            data = await read_val(reader, v_type, v_spec, decode_response)
            result.append(data)
        return result

    elif ttype == TType.MAP:
        if isinstance(spec[0], int):
            k_type = spec[0]
            k_spec = None
        else:
            k_type, k_spec = spec[0]

        if isinstance(spec[1], int):
            v_type = spec[1]
            v_spec = None
        else:
            v_type, v_spec = spec[1]

        result = {}
        sk_type, sv_type, sz = await read_map_begin(reader)
        if sk_type != k_type or sv_type != v_type:
            for _ in range(sz):
                await skip(reader, sk_type)
                await skip(reader, sv_type)
            return {}

        for i in range(sz):
            k_val = await read_val(reader, k_type, k_spec, decode_response)
            v_val = await read_val(reader, v_type, v_spec, decode_response)
            result[k_val] = v_val

        return result

    elif ttype == TType.STRUCT:
        obj = spec()
        await read_struct(reader, obj, decode_response)
        return obj

async def read_struct(reader, obj, decode_response=True):
    while True:
        f_type, fid = await read_field_begin(reader)
        if f_type == TType.STOP:
            break

        if fid not in obj.thrift_spec:
            await skip(reader, f_type)
            continue

        if len(obj.thrift_spec[fid]) == 3:
            sf_type, f_name, f_req = obj.thrift_spec[fid]
            f_container_spec = None
        else:
            sf_type, f_name, f_container_spec, f_req = obj.thrift_spec[fid]

        # it really should equal here. but since we already wasted
        # space storing the duplicate info, let's check it.
        if f_type != sf_type:
            await skip(reader, f_type)
            continue

        data = await read_val(reader, f_type, f_container_spec, decode_response)
        setattr(obj, f_name, data)

async def skip(reader, ftype):
    if ftype == TType.BOOL or ftype == TType.BYTE:
        await reader.readexactly(1)

    elif ftype == TType.I16:
        await reader.readexactly(2)

    elif ftype == TType.I32:
        await reader.readexactly(4)

    elif ftype == TType.I64:
        await reader.readexactly(8)

    elif ftype == TType.DOUBLE:
        await reader.readexactly(8)

    elif ftype == TType.STRING:
        await reader.readexactly(unpack_i32(reader.readexactly(4)))

    elif ftype == TType.SET or ftype == TType.LIST:
        v_type, sz = await read_list_begin(reader)
        for i in range(sz):
            await skip(reader, v_type)

    elif ftype == TType.MAP:
        k_type, v_type, sz = read_map_begin(reader)
        for i in range(sz):
            await skip(reader, k_type)
            await skip(reader, v_type)

    elif ftype == TType.STRUCT:
        while True:
            f_type, fid = await read_field_begin(reader)
            if f_type == TType.STOP:
                break
            await skip(reader, f_type)

class TFramedTransport:
    """Implement the Twisted framed transport protocol.

    Since most async servers use this including the python2 twisted
    bindings, this is likely of interest.
    """
    def __init__(self, base):
        self.__base = base
        self.__read_buffer = BytesIO()
        self.__write_buffer = BytesIO()

    def read(self, n=-1):
        if len(self.__read_buffer.getvalue()) == 0:
            self.read_frame()
        return self.__read_buffer.read(n)

    async def read_frame(self):
        buff = await self.__base.readexactly(4)
        sz, = unpack('!i', buff)
        self.__read_buffer = BytesIO(await self.__base.readexactly(sz))

    async def readexactly(self, n):
        now, end = self.__read_buffer.tell(), self.__read_buffer.seek(0, 2)
        remaining = end - now
        self.__read_buffer.seek(now)

        if 0 < remaining < n:
            raise IOError("Tried to read invalid amount from framed transport")
        elif remaining == 0:
            await self.read_frame()

        return self.__read_buffer.read(n)

    def write(self, val):
        self.__write_buffer.write(val)

    async def drain(self):
        wout = self.__write_buffer.getvalue()
        wsz = len(wout)
        self.__write_buffer = BytesIO()
        buf = pack("!i", wsz) + wout
        self.__base.write(buf)
        await self.__base.drain()

    def at_eof(self):
        return self.__base.at_eof()

    def close(self):
        return self.__base.close()

class TProtocol:
    """
    Base class for thrift protocols, subclass should implement some of the protocol methods,
    currently we only have :class:`TBinaryProtocol` implemented for you.
    """

    def __init__(
            self, trans, strict_read=True, strict_write=True, decode_response=True
    ):
        self.trans = trans
        self.strict_read = strict_read
        self.strict_write = strict_write
        self.decode_response = decode_response

    def skip(self, ttype):
        pass

    async def read_message_begin(self):
        pass

    async def read_message_end(self):
        pass

    def write_message_begin(self, name, ttype, seqid):
        pass

    def write_message_end(self):
        pass

    async def read_struct(self, obj):
        pass

    def write_struct(self, obj):
        pass

class TBinaryProtocol(TProtocol):
    """Binary implementation of the Thrift protocol driver."""

    def skip(self, ttype):
        skip(self.trans, ttype)

    async def read_message_begin(self):
        api, ttype, seqid = await read_message_begin(
            self.trans, strict=self.strict_read
        )
        return api, ttype, seqid

    def write_message_begin(self, name, ttype, seqid):
        write_message_begin(self.trans, name, ttype, seqid, strict=self.strict_write)

    async def read_struct(self, obj):
        data = await read_struct(self.trans, obj, self.decode_response)
        return data

    def write_struct(self, obj):
        write_val(self.trans, TType.STRUCT, obj)
<|endoftext|>


<commit_before>from graph_tool.all import *
from words import data as words
from bigrams import data as bigrams
from trigrams import data as trigrams
from math import log10, log

def clear_entries(entries, containers, treshold=0):
    non_needed_entries = set(entries.keys())
    non_needed_containers = set(containers.keys())
    for entry in entries:
        if entries[entry] <= treshold:
            continue
        exists = False
        for container in containers:
            if entry in container:
                if container in non_needed_containers:
                    non_needed_containers.remove(container)
                non_needed_entries.remove(entry)
                break
    for container in non_needed_containers:
        del containers[container]
    for entry in non_needed_entries:
        del entries[entry]

def get_vertex(g, name, word, vertices):
    # If we already have this vertex, just use it from cache
    if name in vertices:
        return vertices[name]
    # Otherwise we have to create new one
    a = g.add_vertex()
    word[a] = name
    # Add the vertex to cache
    vertices[name] = a
    return a

def build_graph(g, entries, containers, weight, word, color, vertices,
                treshold=0, last_word=-1, width_scale = 3.0,
                entry_color='red', container_color='red'):
    width_scale *= len(set(containers.values()))
    min_tfidf = min(containers.values())
    def add_to_container(entry, container):
        a = get_vertex(g, entry, word, vertices)
        color[a] = entry_color
        b = get_vertex(g, container, word, vertices)
        color[b] = container_color
        e = g.add_edge(a, b)
        # Set weight for new edge (tf-idf)
        # Just empirical formula
        weight[e] = log(log(containers[container]/min_tfidf)+1) * width_scale + 1
    for entry in entries:
        if entries[entry] < treshold:
            continue
        for container in containers:
            if entry in container:
                add_to_container(entry, container)
    return

def init_graph():
    # Create directed graph
    g = Graph(directed=True)
    # Create 'weight' property for edge: will contain tf-idf
    weight = g.new_edge_property('float')
    # Create 'word' property for vertex: will contain string with current word
    word = g.new_vertex_property('string')
    color = g.new_vertex_property('string')
    return g, weight, word, color, {}

if __name__ == '__main__':
    g, weight, word, color, vertices = init_graph()
    # Dictionary with existent vertices (cache)
    clear_entries(words, bigrams, 0.005)
    clear_entries(bigrams, trigrams)
    clear_entries(words, bigrams)
    build_graph(g, entries=words, containers=bigrams, color=color,
                weight=weight, word=word, vertices=vertices,
                entry_color='red', container_color='blue')
    build_graph(g, entries=bigrams, containers=trigrams, color=color,
                weight=weight, word=word, vertices=vertices,
                entry_color='blue', container_color='purple')
    # Draw the graph;
    # Weight is responsible for edges widths
    # Word contains labels for vertices
    graph_draw(g, vertex_font_size=12, edge_pen_width=weight, vertex_text=word,
               vertex_fill_color=color, vertex_text_position=0,
               output='words.png', output_size=(2000, 2000))
    # Alternative
    # graph_draw(g, edge_pen_width=weight, vertex_text=word, node_first=True,
    #            vertex_text_position=0, vertex_size=20, vertex_shape='double_square')
    # Or even
    # state=minimize_nested_blockmodel_dl(g)
    # draw_hierarchy(state, vertex_text=word, vertex_text_position=0)

<commit_msg>Improved graph drawer: added command-line parameters<commit_after>from graph_tool.all import *
from words import data as words
from bigrams import data as bigrams
from trigrams import data as trigrams
from math import log10, log
from sys import argv

def clear_entries(entries, containers, treshold=0):
    non_needed_entries = set(entries.keys())
    non_needed_containers = set(containers.keys())
    for entry in entries:
        if entries[entry] <= treshold:
            continue
        exists = False
        for container in containers:
            if entry in container:
                if container in non_needed_containers:
                    non_needed_containers.remove(container)
                non_needed_entries.remove(entry)
                break
    for container in non_needed_containers:
        del containers[container]
    for entry in non_needed_entries:
        del entries[entry]

def get_vertex(g, name, word, vertices):
    # If we already have this vertex, just use it from cache
    if name in vertices:
        return vertices[name]
    # Otherwise we have to create new one
    a = g.add_vertex()
    word[a] = name
    # Add the vertex to cache
    vertices[name] = a
    return a

def build_graph(g, entries, containers,
                weight, word, color, vertices,
                treshold=0, last_word=-1, width_scale = 3.0,
                entry_color='red', container_color='red'):
    min_tfidf = min(containers.values())
    def add_to_container(entry, container):
        a = get_vertex(g, entry, word, vertices)
        color[a] = entry_color
        b = get_vertex(g, container, word, vertices)
        color[b] = container_color
        e = g.add_edge(a, b)
        # Set weight for new edge (tf-idf)
        # Just empirical formula
        raw_weight = containers[container]/min_tfidf
        weight[e] = log(log(raw_weight)+1) * width_scale + 1
    for entry in entries:
        if entries[entry] < treshold:
            continue
        for container in containers:
            if entry in container:
                add_to_container(entry, container)
    return

def init_graph():
    # Create directed graph
    g = Graph(directed=True)
    # Create 'weight' property for edge:
    # will contain tf-idf
    weight = g.new_edge_property('float')
    # Create 'word' property for vertex:
    # will contain string with current word
    word = g.new_vertex_property('string')
    color = g.new_vertex_property('string')
    return g, weight, word, color, {}

if __name__ == '__main__':
    img_name='output.png'
    if len(argv) > 2:
        if argv[1] in ['-w', '--word']:
            words = dict(words.items()[:int(argv[2])])
            clear_entries(words, bigrams)
        elif argv[1] in ['-b', '--bigram']:
            bigrams = dict(bigrams.items()[:int(argv[2])])
            clear_entries(bigrams, trigrams)
        elif argv[1] in ['-t', '--triad']:
            trigrams = dict(trigrams.items()[:int(argv[2])])
            clear_entries(bigrams, trigrams)
    if len(argv) > 3:
        img_name = argv[3]
    g, weight, word, color, vertices = init_graph()
    # Dictionary with existent vertices (cache)
    clear_entries(words, bigrams)
    clear_entries(bigrams, trigrams)
    build_graph(g, entries=words, containers=bigrams,
                color=color, weight=weight, word=word,
                vertices=vertices,
                entry_color='red', container_color='blue')
    build_graph(g, entries=bigrams, containers=trigrams,
                color=color, weight=weight, word=word,
                vertices=vertices,
                entry_color='blue', container_color='purple')
    # Draw the graph;
    # Weight is responsible for edges widths
    # Word contains labels for vertices
    # graph_draw(g, vertex_font_size=10, edge_pen_width=weight,
    #            vertex_text=word, vertex_fill_color=color,
    #            vertex_text_position=0, output=img_name,
    #            output_size=(300, 500))
    # Alternative
    # graph_draw(g, edge_pen_width=weight, vertex_text=word,
    #            node_first=True, vertex_text_position=0,
    #            vertex_size=20, vertex_shape='double_square')
    # Or even
    state=minimize_nested_blockmodel_dl(g)
    draw_hierarchy(state, vertex_text=word,
            vertex_text_position=1, edge_pen_width=weight,
            vertex_fill_color=color,
            output=img_name, output_size=(800, 600))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# #############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import time
from openerp.report import report_sxw
from openerp import pooler
from datetime import date, datetime, timedelta
import logging
_logger = logging.getLogger(__name__)

class doctor_account_voucher(report_sxw.rml_parse):
    def __init__(self, cr, uid, name, context):
        super(doctor_account_voucher, self).__init__(cr, uid, name, context=context)
        self.localcontext.update({
            'time': time,
            'base': self.base,
            'select_valor_procedimiento': self.select_valor_procedimiento,
            'select_cantidad_procedimiento': self.select_cantidad_procedimiento,
            'select_nombre_procedimiento': self.select_nombre_procedimiento,
        })

    def base(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        doctor_paciente = self.pool.get('doctor.patient')
        cuenta = self.pool.get('account.invoice')
        cuenta_line = self.pool.get('account.invoice.line')
        paciente_id = doctor_paciente.search(self.cr, self.uid, [('patient', '=', partner_id)], context=context)
        cuenta_ids = []
        cuenta_id = cuenta.search(self.cr, self.uid, [('partner_id', '=', partner_id), ('date_invoice', '=', '2016-04-12'), ('amount_patient', '=', pago_paciente)], context=context)

        for i in cuenta.browse(self.cr, self.uid, cuenta_id, context=context):
            cuenta_ids.append(i.id)

        return cuenta_line.search(self.cr, self.uid, [('invoice_id', 'in', cuenta_ids)], context=context)

    def select_valor_procedimiento(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        cuenta = self.pool.get('account.invoice.line')
        dato = 0
        for i in cuenta.browse(self.cr, self.uid, self.base(partner_id, fecha, pago_paciente), context=context):
            dato += i.price_subtotal
        return dato

    def select_cantidad_procedimiento(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        cuenta = self.pool.get('account.invoice.line')
        dato = 0
        for i in cuenta.browse(self.cr, self.uid, self.base(partner_id, fecha, pago_paciente), context=context):
            dato += i.quantity
        return dato

    def select_nombre_procedimiento(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        cuenta = self.pool.get('account.invoice.line')
        dato = ""
        for i in cuenta.browse(self.cr, self.uid, self.base(partner_id, fecha, pago_paciente), context=context):
            dato += i.name + ", "

        dato = dato[:dato.rfind(',')]

        return dato

report_sxw.report_sxw('report.doctor_account_voucher', 'account.voucher',
                      'addons/l10n_co_doctor/report/doctor_account_voucher.rml',
                      parser=doctor_account_voucher)
        <commit_msg>agregando cambios de los reportes<commit_after># -*- coding: utf-8 -*-
# #############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import time
from openerp.report import report_sxw
from openerp import pooler
from datetime import date, datetime, timedelta
import logging
_logger = logging.getLogger(__name__)

class doctor_account_voucher(report_sxw.rml_parse):
    def __init__(self, cr, uid, name, context):
        super(doctor_account_voucher, self).__init__(cr, uid, name, context=context)
        self.localcontext.update({
            'time': time,
            'base': self.base,
            'select_valor_procedimiento': self.select_valor_procedimiento,
            'select_cantidad_procedimiento': self.select_cantidad_procedimiento,
            'select_nombre_procedimiento': self.select_nombre_procedimiento,
        })

    def base(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        doctor_paciente = self.pool.get('doctor.patient')
        cuenta = self.pool.get('account.invoice')
        cuenta_line = self.pool.get('account.invoice.line')
        paciente_id = doctor_paciente.search(self.cr, self.uid, [('patient', '=', partner_id)], context=context)
        cuenta_ids = []

        cuenta_id = cuenta.search(self.cr, self.uid, [('patient_id', '=', paciente_id), ('date_invoice', '=', fecha), ('amount_patient', '=', pago_paciente)], context=context)

        for i in cuenta.browse(self.cr, self.uid, cuenta_id, context=context):
            cuenta_ids.append(i.id)

        return cuenta_line.search(self.cr, self.uid, [('invoice_id', 'in', cuenta_ids)], context=context)

    def select_valor_procedimiento(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        cuenta = self.pool.get('account.invoice.line')
        dato = 0
        for i in cuenta.browse(self.cr, self.uid, self.base(partner_id, fecha, pago_paciente), context=context):
            dato += i.price_subtotal
        return dato

    def select_cantidad_procedimiento(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        cuenta = self.pool.get('account.invoice.line')
        dato = 0
        for i in cuenta.browse(self.cr, self.uid, self.base(partner_id, fecha, pago_paciente), context=context):
            dato += i.quantity
        return dato

    def select_nombre_procedimiento(self, partner_id, fecha, pago_paciente):
        context = {}
        context.update({'lang' : self.pool.get('res.users').browse(self.cr, self.uid, self.uid, context=context).lang})
        cuenta = self.pool.get('account.invoice.line')
        dato = ""
        for i in cuenta.browse(self.cr, self.uid, self.base(partner_id, fecha, pago_paciente), context=context):
            dato += i.name + ", "

        dato = dato[:dato.rfind(',')]

        return dato

report_sxw.report_sxw('report.doctor_account_voucher', 'account.voucher',
                      'addons/l10n_co_doctor/report/doctor_account_voucher.rml',
                      parser=doctor_account_voucher)
        <|endoftext|>


<commit_before>import numpy as np
import pygame
from pygame import gfxdraw
import sys

from src.play.model.Board import BLACK, WHITE
from src.play.model.Move import Move

brown = (165, 42, 42)
black = (0, 0, 0)
white = (255, 255, 255)
yellow = (255, 255, 0)
orange = (255, 165, 0)
blue = (0, 0, 255)

window_size = (500, 600)
board_top_left_coord = (50, 100)  # coordinate of the boards top left point
board_size = 400
stone_radius = 20

class PygameView:

    def __init__(self, controller):
        controller.view = self
        self.controller = controller
        self.game = controller.game
        self.running = False
        self.cell_size = board_size / (self.game.size - 1)
        self.buttons = []
        self.labels = []

    def open(self):
        pygame.init()
        self.running = True
        self.screen = pygame.display.set_mode(window_size)
        pygame.display.set_caption('Go')
        self.buttons.append(Button(
            210, 530, 80, 40, 'Pass', self.screen, self.send_pass_move))
        self.labels.append(Label(
            100, 30, 300, 40, self.get_turn_label_text, self.screen))
        self.render()

        while self.running:
            event = pygame.event.poll()
            if event.type == pygame.MOUSEBUTTONUP:
                x, y = event.pos
                col = int(round((x - board_top_left_coord[0]) /
                                self.cell_size))
                row = int(round((y - board_top_left_coord[1]) /
                                self.cell_size))
                if 0 <= col < self.game.size and 0 <= row < self.game.size:
                    self.controller.receive_move_from_gui(Move(col, row))
                for btn in self.buttons:
                    btn.check_mouse_released()
            if event.type == pygame.QUIT:
                self.running = False
            for btn in self.buttons:
                btn.is_mouse_over_btn()
            self.render()

        pygame.quit()
        sys.exit(0)

    def game_ended(self):
        self.labels[0] = (Label(100, 30, 300, 40, lambda: "Game has ended", self.screen))

    def send_pass_move(self):
        self.controller.receive_move_from_gui(Move(is_pass=True))

    def get_turn_label_text(self):
        return 'It\'s ' + self.controller.current_player.name + '\'s turn'

    def render(self):
        # board
        self.screen.fill(brown)
        for i in range(0, self.game.size):
            # horizontals
            pygame.draw.line(
                self.screen, black,
                [board_top_left_coord[0],
                 board_top_left_coord[1] + i * self.cell_size],
                [board_top_left_coord[0] + board_size,
                 board_top_left_coord[1] + i * self.cell_size], 1)
            # verticals
            pygame.draw.line(
                self.screen, black,
                [board_top_left_coord[0] + i * self.cell_size,
                 board_top_left_coord[1]],
                [board_top_left_coord[0] + i * self.cell_size,
                 board_top_left_coord[1] + board_size], 1)
        # stones
        black_rows, black_cols = np.where(self.game.board == BLACK)
        white_rows, white_cols = np.where(self.game.board == WHITE)
        for i in range(0, len(black_rows)):
            indices = black_cols[i], black_rows[i]
            self.draw_stone(indices, black)
        for i in range(0, len(white_rows)):
            indices = white_cols[i], white_rows[i]
            self.draw_stone(indices, white)
        for btn in self.buttons:
            btn.draw()
        for label in self.labels:
            label.draw()
        pygame.display.flip()  # update the screen

    def draw_stone(self, indices, col):
        x = int(board_top_left_coord[0] + self.cell_size * indices[0])
        y = int(board_top_left_coord[1] + indices[1] * self.cell_size)
        # antialiasing via stackoverflow.com/a/26774279/2474159
        gfxdraw.aacircle(self.screen, x, y, stone_radius, col)
        gfxdraw.filled_circle(self.screen, x, y, stone_radius, col)

class Button:
    """Adapted from
    gamedev.net/forums/topic/686666-pygame-buttons/?do=findComment&comment=5333411
    """
    def __init__(self, x, y, w, h, text, screen, on_click):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.rect = (self.x, self.y, self.w, self.h)
        pygame.font.init()
        self.font = pygame.font.Font(None, 25)  # compute font size from h?
        self.text = text
        self.mouse_is_over_btn = False
        self.screen = screen
        self.on_click = on_click

    def draw(self):
        if self.mouse_is_over_btn:
            bg = yellow
        else:
            bg = orange
        surf = self.font.render(self.text, True, black, bg)
        xo = self.x + (self.w - surf.get_width()) / 2
        yo = self.y + (self.h - surf.get_height()) / 2
        self.screen.fill(bg, self.rect)
        pygame.draw.rect(self.screen, black, self.rect, 1)
        self.screen.blit(surf, (xo, yo))

    def is_mouse_over_btn(self):
        pos = pygame.mouse.get_pos()
        self.mouse_is_over_btn = (
            self.x <= pos[0] < self.x + self.w and
            self.y <= pos[1] < self.y + self.h)

    def check_mouse_released(self):
        if self.mouse_is_over_btn:
            self.on_click()

class Label:  # make Button extend Label?

    def __init__(self, x, y, w, h, get_text, screen):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.rect = (self.x, self.y, self.w, self.h)
        pygame.font.init()
        self.font = pygame.font.Font(None, 30)
        self.get_text = get_text  # method, get's called in draw()
        self.screen = screen

    def draw(self):
        surf = self.font.render(self.get_text(), True, yellow, brown)
        xo = self.x + (self.w - surf.get_width()) / 2
        yo = self.y + (self.h - surf.get_height()) / 2
        self.screen.blit(surf, (xo, yo))
<commit_msg>add color to turn label<commit_after>import numpy as np
import pygame
from pygame import gfxdraw
import sys

from src.play.model.Board import BLACK, WHITE
from src.play.model.Move import Move

brown = (165, 42, 42)
black = (0, 0, 0)
white = (255, 255, 255)
yellow = (255, 255, 0)
orange = (255, 165, 0)
blue = (0, 0, 255)

window_size = (500, 600)
board_top_left_coord = (50, 100)  # coordinate of the boards top left point
board_size = 400
stone_radius = 20

class PygameView:

    def __init__(self, controller):
        controller.view = self
        self.controller = controller
        self.game = controller.game
        self.running = False
        self.cell_size = board_size / (self.game.size - 1)
        self.buttons = []
        self.labels = []

    def open(self):
        pygame.init()
        self.running = True
        self.screen = pygame.display.set_mode(window_size)
        pygame.display.set_caption('Go')
        self.buttons.append(Button(
            210, 530, 80, 40, 'Pass', self.screen, self.send_pass_move))
        self.labels.append(Label(
            100, 30, 300, 40, self.get_turn_label_text, self.screen))
        self.render()

        while self.running:
            event = pygame.event.poll()
            if event.type == pygame.MOUSEBUTTONUP:
                x, y = event.pos
                col = int(round((x - board_top_left_coord[0]) /
                                self.cell_size))
                row = int(round((y - board_top_left_coord[1]) /
                                self.cell_size))
                if 0 <= col < self.game.size and 0 <= row < self.game.size:
                    self.controller.receive_move_from_gui(Move(col, row))
                for btn in self.buttons:
                    btn.check_mouse_released()
            if event.type == pygame.QUIT:
                self.running = False
            for btn in self.buttons:
                btn.is_mouse_over_btn()
            self.render()

        pygame.quit()
        sys.exit(0)

    def game_ended(self):
        self.labels[0] = (Label(100, 30, 300, 40, lambda: "Game has ended", self.screen))

    def send_pass_move(self):
        self.controller.receive_move_from_gui(Move(is_pass=True))

    def get_turn_label_text(self):
        return self.controller.current_player.name + '\'s (' \
               + self.controller.current_player.color + ') turn'

    def render(self):
        # board
        self.screen.fill(brown)
        for i in range(0, self.game.size):
            # horizontals
            pygame.draw.line(
                self.screen, black,
                [board_top_left_coord[0],
                 board_top_left_coord[1] + i * self.cell_size],
                [board_top_left_coord[0] + board_size,
                 board_top_left_coord[1] + i * self.cell_size], 1)
            # verticals
            pygame.draw.line(
                self.screen, black,
                [board_top_left_coord[0] + i * self.cell_size,
                 board_top_left_coord[1]],
                [board_top_left_coord[0] + i * self.cell_size,
                 board_top_left_coord[1] + board_size], 1)
        # stones
        black_rows, black_cols = np.where(self.game.board == BLACK)
        white_rows, white_cols = np.where(self.game.board == WHITE)
        for i in range(0, len(black_rows)):
            indices = black_cols[i], black_rows[i]
            self.draw_stone(indices, black)
        for i in range(0, len(white_rows)):
            indices = white_cols[i], white_rows[i]
            self.draw_stone(indices, white)
        for btn in self.buttons:
            btn.draw()
        for label in self.labels:
            label.draw()
        pygame.display.flip()  # update the screen

    def draw_stone(self, indices, col):
        x = int(board_top_left_coord[0] + self.cell_size * indices[0])
        y = int(board_top_left_coord[1] + indices[1] * self.cell_size)
        # antialiasing via stackoverflow.com/a/26774279/2474159
        gfxdraw.aacircle(self.screen, x, y, stone_radius, col)
        gfxdraw.filled_circle(self.screen, x, y, stone_radius, col)

class Button:
    """Adapted from
    gamedev.net/forums/topic/686666-pygame-buttons/?do=findComment&comment=5333411
    """
    def __init__(self, x, y, w, h, text, screen, on_click):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.rect = (self.x, self.y, self.w, self.h)
        pygame.font.init()
        self.font = pygame.font.Font(None, 25)  # compute font size from h?
        self.text = text
        self.mouse_is_over_btn = False
        self.screen = screen
        self.on_click = on_click

    def draw(self):
        if self.mouse_is_over_btn:
            bg = yellow
        else:
            bg = orange
        surf = self.font.render(self.text, True, black, bg)
        xo = self.x + (self.w - surf.get_width()) / 2
        yo = self.y + (self.h - surf.get_height()) / 2
        self.screen.fill(bg, self.rect)
        pygame.draw.rect(self.screen, black, self.rect, 1)
        self.screen.blit(surf, (xo, yo))

    def is_mouse_over_btn(self):
        pos = pygame.mouse.get_pos()
        self.mouse_is_over_btn = (
            self.x <= pos[0] < self.x + self.w and
            self.y <= pos[1] < self.y + self.h)

    def check_mouse_released(self):
        if self.mouse_is_over_btn:
            self.on_click()

class Label:  # make Button extend Label?

    def __init__(self, x, y, w, h, get_text, screen):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.rect = (self.x, self.y, self.w, self.h)
        pygame.font.init()
        self.font = pygame.font.Font(None, 30)
        self.get_text = get_text  # method, get's called in draw()
        self.screen = screen

    def draw(self):
        surf = self.font.render(self.get_text(), True, yellow, brown)
        xo = self.x + (self.w - surf.get_width()) / 2
        yo = self.y + (self.h - surf.get_height()) / 2
        self.screen.blit(surf, (xo, yo))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

HEADERS = {
    "User-Agent": "Tinder/6.1.0 (iPhone; iOS 10.0.1; Scale/2.00)",
    "Content-Type": "application/json",
    "os_version": "100000000001",
    "platform": "ios",
    "app-version": "1768",
    "x-client-version": "61014"
}

API_URL = "https://api.gotinder.com/"
<commit_msg>update tinder headers<commit_after># -*- coding: utf-8 -*-

HEADERS = {
    "User-Agent": "Tinder/6.2.0 (iPhone; iOS 10.0.2; Scale/2.00)",
    "Content-Type": "application/json",
    "os_version": "100000000002",
    "platform": "ios",
    "app-version": "1768",
    "x-client-version": "62022"
}

API_URL = "https://api.gotinder.com/"
<|endoftext|>


<commit_before>from celery import uuid as celery_uuid
from django import forms
import itertools
import logging

from disease.files_utils import get_genome_filepath
from twentythree.tasks import fetch_genome_and_push_forward
from twentythree.models import CeleryTask23

log = logging.getLogger('genoome.twentythree.forms')

class ChooseProfileForm(forms.Form):
    profile = forms.ChoiceField()

    def __init__(self, *args, **kwargs):
        self.profiles = kwargs.pop('profiles')
        self.genotyped_profiles = [p['id'] for p in self.profiles if p['genotyped']]
        self.user = kwargs.pop('user')
        super().__init__(*args, **kwargs)
        cnt = itertools.count()
        self.fields['profile'].choices = [ (next(cnt), p_id) for p_id in sorted(self.genotyped_profiles) ]

    def clean_profile(self):
        profile_id = self.genotyped_profiles[int(self.cleaned_data['profile'])]
        return profile_id

    def save(self):
        if len(self.genotyped_profiles) == 1:
            profile_id = self.genotyped_profiles[0]
        else:
            profile_id = self.cleaned_data['profile']
        celery_task_id = celery_uuid()

        ctask = CeleryTask23.objects.create(user=user, chosen_profile=profile_id, fetch_task_id=celery_task_id)
        fetch_genome_and_push_forward.apply_async(args=(ctask.pk), task_id=ctask.fetch_task_id)
        return ctask
<commit_msg>23andme wip<commit_after>from celery import uuid as celery_uuid
from django import forms
import itertools
import logging

from disease.files_utils import get_genome_filepath
from twentythree.tasks import fetch_genome_and_push_forward
from twentythree.models import CeleryTask23

log = logging.getLogger('genoome.twentythree.forms')

class ChooseProfileForm(forms.Form):
    profile = forms.ChoiceField()

    def __init__(self, *args, **kwargs):
        self.profiles = kwargs.pop('profiles')
        self.genotyped_profiles = [p['id'] for p in self.profiles if p['genotyped']]
        self.user = kwargs.pop('user')
        super().__init__(*args, **kwargs)
        cnt = itertools.count()
        self.fields['profile'].choices = [ (next(cnt), p_id) for p_id in sorted(self.genotyped_profiles) ]

    def clean_profile(self):
        profile_id = self.genotyped_profiles[int(self.cleaned_data['profile'])]
        return profile_id

    def save(self):
        if len(self.genotyped_profiles) == 1:
            profile_id = self.genotyped_profiles[0]
        else:
            profile_id = self.cleaned_data['profile']
        celery_task_id = celery_uuid()

        ctask = CeleryTask23.objects.create(user=self.user, chosen_profile=profile_id, fetch_task_id=celery_task_id)
        fetch_genome_and_push_forward.apply_async(args=(ctask.pk), task_id=ctask.fetch_task_id)
        return ctask
<|endoftext|>


<commit_before>"""
Format is a class which is abstract enough for Images to talk with. It has:

*grid and datasource
*metadata in the form of a "header"
*data get/set (though no specific means to find that data)
*canonical fields, which is a sort of intersection of all specific
formats' metadata, but in a known NI language

BinaryFormat(Format) is an object with binary header data and a "brick" of
binary data. The binary.py module has a lot of struct packing/unpacking
methods to make life nicer. BinaryFormats have:
*filenames and modes--in fact, BinaryFormats are always constructed with a
filename, this is not an option anymore (makes sense to me).
*header formats (in package struct language)
*means to read/write contiguous/deterministic-style headers

"""
__all__ = ['Format']

import numpy

from neuroimaging import import_from
from neuroimaging.data_io import DataSource
from neuroimaging.utils.path import path
from neuroimaging.utils.odict import odict
from neuroimaging.core.reference.grid import SamplingGrid
from neuroimaging.core.image.base_image import BaseImage

class Format(BaseImage):

    """ Valid filename extensions for the file format. """
    extensions = []

    def __init__(self, datasource=DataSource(), grid=None, **keywords):
        BaseImage.__init__(self, NotImplemented, grid, NotImplemented)
        # Formats should concern themselves with datasources and grids
        self.datasource = datasource
        # Has metadata
        self.header = odict()
        # Possibly has extended data
        self.ext_header = odict()
        # Has general information (or should!)
        self.canonical_fields = odict((
            ('datasize', 0),
            ('ndim', 0),
            ('xdim', 0),
            ('ydim', 0),
            ('zdim', 0),
            ('tdim', 0),
            ('intent', ''),
            ('scaling', 0),
        ))

    def dump_header(self):
        return "\n".join(["%s\t%s"%(field,`self.header[field]`) \
                          for field in self.header.keys()])

    def inform_canonical(self, fieldsDict=None):
        raise NotImplementedError

    def __str__(self):
        return self.dump_header()

    def __getitem__(self, slice_):
        """Data access"""
        raise NotImplementedError

    def __setitem__(self, slice_, data):
        """Data access"""
        raise NotImplementedError

    @classmethod
    def valid(self, filename, verbose=False, mode='r'):
        # verbose not implemented
        try:
            ext = path(filename).splitext()[1]
            if ext not in self.extensions:
                return False
        except:
            return False
        return True

    #############################################
    ## The following header manipulations might
    ## not be implemented for a given format.
    ## They require specific knowledge of the
    ## header field names
    #############################################
    def add_header_field(self, name, type, value):
        """
        Add a field to the header. Type should be
        a format string interpretable by struct.pack.
        """
        raise NotImplementedError

    def remove_header_field(self, name):
        """
        Remove a field from the header. Will Definitely Not
        remove any protected fields.
        """
        raise NotImplementedError

    def set_header_field(self, name, value):
        """
        Set a field's value
        """
        raise NotImplementedError

    def get_header_field(self, name):
        """
        Get a field's value
        """
        raise NotImplementedError

format_modules = (
  "neuroimaging.data_io.formats.analyze",
  "neuroimaging.data_io.formats.nifti1",
  "neuroimaging.data_io.formats.ecat7",
  #"neuroimaging.data_io.formats.afni",
  #"neuroimaging.data_io.formats.minc",
)

default_formats = [("neuroimaging.data_io.formats.nifti1", "Nifti1"),
                   ("neuroimaging.data_io.formats.analyze", "Analyze"),
                   ("neuroimaging.data_io.formats.ecat7", "Ecat7"),
                  ]

def getformats(filename):
    "Return the appropriate image format for the given file type."
    all_formats = []
    valid_formats = []
    for modname, formatname in default_formats:
        all_formats.append(formatname)
        format = import_from(modname, formatname)
        if format.valid(filename):
            valid_formats.append(format)

    if valid_formats:
        return valid_formats

    # if we made it this far, a format was not found

    extension = path(filename).splitext()[1]
    raise NotImplementedError,\
      "file extension %(ext)s not recognized, %(exts)s files can be created "\
      "at this time."% {'ext':extension, 'exts':all_formats}

def hasformat(filename):
    """
    Determine if there is an image format format registered for the given
    file type.
    """
    try:
        getformats(filename)
        return True
    except NotImplementedError:
        return False

<commit_msg>small bug fix<commit_after>"""
Format is a class which is abstract enough for Images to talk with. It has:

*grid and datasource
*metadata in the form of a "header"
*data get/set (though no specific means to find that data)
*canonical fields, which is a sort of intersection of all specific
formats' metadata, but in a known NI language

BinaryFormat(Format) is an object with binary header data and a "brick" of
binary data. The binary.py module has a lot of struct packing/unpacking
methods to make life nicer. BinaryFormats have:
*filenames and modes--in fact, BinaryFormats are always constructed with a
filename, this is not an option anymore (makes sense to me).
*header formats (in package struct language)
*means to read/write contiguous/deterministic-style headers

"""
__all__ = ['Format']

import numpy

from neuroimaging import import_from
from neuroimaging.data_io import DataSource
from neuroimaging.utils.path import path
from neuroimaging.utils.odict import odict
from neuroimaging.core.reference.grid import SamplingGrid
from neuroimaging.core.image.base_image import BaseImage

class Format(BaseImage):

    """ Valid filename extensions for the file format. """
    extensions = []

    def __init__(self, datasource=DataSource(), grid=None, **keywords):
        BaseImage.__init__(self, NotImplemented, grid, NotImplemented)
        # Formats should concern themselves with datasources and grids
        self.datasource = datasource
        # Has metadata
        self.header = odict()
        # Possibly has extended data
        self.ext_header = odict()
        # Has general information (or should!)
        self.canonical_fields = odict((
            ('datasize', 0),
            ('ndim', 0),
            ('xdim', 0),
            ('ydim', 0),
            ('zdim', 0),
            ('tdim', 0),
            ('intent', ''),
            ('scaling', 0),
        ))

    def dump_header(self):
        return "\n".join(["%s\t%s"%(field,`self.header[field]`) \
                          for field in self.header.keys()])

    def inform_canonical(self, fieldsDict=None):
        raise NotImplementedError

    def __str__(self):
        return self.dump_header()

    def __getitem__(self, slice_):
        """Data access"""
        raise NotImplementedError

    def __setitem__(self, slice_, data):
        """Data access"""
        raise NotImplementedError

    @classmethod
    def valid(self, filename, verbose=False, mode='r'):
        # verbose not implemented
        try:
            ext = path(filename).splitext()[1]
            if ext not in self.extensions:
                return False
        except:
            return False
        return True

    #############################################
    ## The following header manipulations might
    ## not be implemented for a given format.
    ## They require specific knowledge of the
    ## header field names
    #############################################
    def add_header_field(self, name, type, value):
        """
        Add a field to the header. Type should be
        a format string interpretable by struct.pack.
        """
        raise NotImplementedError

    def remove_header_field(self, name):
        """
        Remove a field from the header. Will Definitely Not
        remove any protected fields.
        """
        raise NotImplementedError

    def set_header_field(self, name, value):
        """
        Set a field's value
        """
        raise NotImplementedError

    def get_header_field(self, name):
        """
        Get a field's value
        """
        raise NotImplementedError

format_modules = (
  "neuroimaging.data_io.formats.analyze",
  "neuroimaging.data_io.formats.nifti1",
  "neuroimaging.data_io.formats.ecat7",
  #"neuroimaging.data_io.formats.afni",
  #"neuroimaging.data_io.formats.minc",
)

default_formats = [("neuroimaging.data_io.formats.nifti1", "Nifti1"),
                   ("neuroimaging.data_io.formats.analyze", "Analyze"),
                   ("neuroimaging.data_io.formats.ecat7", "ECAT7"),
                  ]

def getformats(filename):
    "Return the appropriate image format for the given file type."
    all_formats = []
    valid_formats = []
    for modname, formatname in default_formats:
        all_formats.append(formatname)
        format = import_from(modname, formatname)
        if format.valid(filename):
            valid_formats.append(format)

    if valid_formats:
        return valid_formats

    # if we made it this far, a format was not found

    extension = path(filename).splitext()[1]
    raise NotImplementedError,\
      "file extension %(ext)s not recognized, %(exts)s files can be created "\
      "at this time."% {'ext':extension, 'exts':all_formats}

def hasformat(filename):
    """
    Determine if there is an image format format registered for the given
    file type.
    """
    try:
        getformats(filename)
        return True
    except NotImplementedError:
        return False

<|endoftext|>


<commit_before>"""

Reference Resolver
------------------

NOTE: All of these approaches use web-scraping directly from publisher sites.

Goal:
    Take something like this:
        Senís, Elena, et al. "CRISPR/Cas9‐mediated genome engineering:
        An adeno‐associated viral (AAV) vector toolbox."
        Biotechnology journal 9.11 (2014): 1402-1412.

    and retrieve the entry data and/or that paper's references from the
    corresponding site where it is hosted.

    Do the same for a DOI or link.

------------------

Basically follow the method outlined here:
    http://labs.crossref.org/resolving-citations-we-dont-need-no-stinkin-parser/

Another super helpful link about DOIs and publishers:
    https://webhome.weizmann.ac.il/home/comartin/doi.html

Steps:
    1) Search for the original reference on CrossRef
    2) Extract DOI from CrossRef
    3) Match DOI with provider
    4) Call appropriate scraper with DOI and get info
    5) Return info

"""
# Standard imports
import json
import json.decoder
import sys

if sys.version_info.major == 2:
    from urllib import quote as urllib_quote
else:
    from urllib.parse import quote as urllib_quote

# Third party imports
import requests

# Local imports
from pypub.paper_info import PaperInfo
from pypub.publishers import pub_resolve
from pypub.utils import find_nth
from . import ref_retrieval
from .rr_errors import *

from scopy import Scopus

from database import db_logging as db

# -----------------------------------------------------

def paper_info_from_citation(citation):
    """
    Gets the paper and references information from
    a plaintext citation.

    Uses a search to CrossRef.org to retrive paper DOI.

    Parameters
    ----------
    citation : str
        Full journal article citation.
        Example: Senís, Elena, et al. "CRISPR/Cas9‐mediated genome
                engineering: An adeno‐associated viral (AAV) vector
                toolbox. Biotechnology journal 9.11 (2014): 1402-1412.

    Returns
    -------
    paper_info : PaperInfo
        Class containing relevant paper meta-information and
        references list.
        Information about the paper itself is in 'entry' value, and is a dict
        (with str and dict values). References list is in 'references'
        value and is a list of dicts (each with str and dict values).
        Must call .__dict__ to be JSON-serializable

    """
    # Encode raw citation
    citation = urllib_quote(citation)

    # Search for citation on CrossRef.org to try to get a DOI link
    api_search_url = 'http://search.labs.crossref.org/dois?q=' + citation
    response = requests.get(api_search_url).json()
    resp = response[0]
    doi = resp.get('doi')
    print(doi)

    if doi is None:
        raise LookupError('No DOI could be found for the given citation')

    # If crossref returns a http://dx.doi.org/ link, retrieve the doi from it
    # and save the URL to pass to doi_to_info
    url = None
    if doi[0:18] == 'http://dx.doi.org/':
        url = doi
        doi = doi[18:]

    # Check if this DOI has been searched and saved before.
    # If it has, return saved information
    saved_info = db.get_saved_info(doi)
    if saved_info is not None:
        saved_info.make_interface_object()
        return saved_info

    paper_info = doi_to_webscraped_info(doi=doi, url=url)
    db.log_info(paper_info)

    return paper_info

# This is commented out because retrieve_all_info subsumes it.
'''
def paper_info_from_doi(doi, skip_saved=False):
    """
    Gets the paper and references information from an article DOI.

    Parameters
    ----------
    doi : str
        DOI = digital object identifier.
        Unique ID assigned to a journal article.
        Example: 10.1002/biot.201400046

    Returns
    -------
    paper_info : PaperInfo
        See resolve_citation for description.

    """
    if not skip_saved:
        # Check for the DOI and corresponding paper in user's database.
        # If it has already been saved, return saved values.
        saved_info = db.get_saved_info(doi)
        if saved_info is not None:
            saved_info.make_interface_object()
            return saved_info

    paper_info = doi_to_webscraped_info(doi=doi)
    db.log_info(paper_info)

    return paper_info
'''

def paper_info_from_link(link):
    pub_dict = resolve_link(link)

def resolve_link(link):
    """
    Gets the paper and references information from a link (URL)
    to a specific journal article page.

    Parameters
    ----------
    link : str
        URL to journal article page on publisher's website.
        Example: http://onlinelibrary.wiley.com/doi/10.1002/biot.201400046/references

    Returns
    -------
    pub_dict : dict
        See resolve_citation for description.

    """

    # First format the link correctly and determine the publisher
    # ---------------------
    # Make sure 'http://' and not 'www.' is at the beginning
    link = link.replace('www.', '')
    if link[0:4] != 'http':
        link = 'http://' + link

    # Find the third '/', which should be after the base URL
    # i.e. giving something like http://onlinelibrary.wiley.com/
    end_of_base_url = find_nth(link, '/', 3)
    base_url = link[:end_of_base_url+1]

    return pub_resolve.get_publisher_site_info(base_url=base_url)

def link_to_doi(link):
    return_values = resolve_link(link)
    return return_values

def doi_and_title_from_citation(citation):
    """
    Gets the DOI from
    a plaintext citation.

    Uses a search to CrossRef.org to retrive paper DOI.

    Parameters
    ----------
    citation : str
        Full journal article citation.
        Example: Senís, Elena, et al. "CRISPR/Cas9‐mediated genome
                engineering: An adeno‐associated viral (AAV) vector
                toolbox. Biotechnology journal 9.11 (2014): 1402-1412.

    Returns
    -------
    doi : str
    """
    # Encode raw citation
    citation = urllib_quote(citation)

    # Search for citation on CrossRef.org to try to get a DOI link
    api_search_url = 'http://search.labs.crossref.org/dois?q=' + citation
    try:
        response = requests.get(api_search_url).json()
    except json.decoder.JSONDecodeError:
        return None

    resp = response[0]
    doi = resp.get('doi')
    title = resp.get('title')

    if doi is None:
        return None

    # If crossref returns a http://dx.doi.org/ link, retrieve the doi from it
    # and save the URL to pass to doi_to_info
    if 'http://dx.doi.org/' in doi:
        doi = doi.replace('http://dx.doi.org/', '')
        doi = doi.strip()

    return doi, title

def doi_to_webscraped_info(doi=None, url=None):
    """
    Gets entry and references information for an article DOI.

    Uses saved dicts matching DOI prefixes to publishers and web scrapers
    to retrieve information. Will fail if DOI prefix hasn't been saved
    with a publisher link or if a scraper for a specific publisher
    site hasn't been built.

    Parameters
    ----------
    doi : str
        Unique ID assigned to a journal article.
    url : str
        The CrossRef URL to the article page.
        I.e. http://dx.doi.org/10.######

    Returns
    -------
    paper_info : PaperInfo
        Class containing parameters including the following:

        entry_dict : dict
            Contains information about the paper referenced by the DOI.
            Includes title, authors, affiliations, publish date, journal
            title, volume, and pages, and keywords. Some values are other
            dicts (for example, the author info with affiliation values).
            Formatted to be JSON serializable.

        refs_dicts : list of dicts
            Each list item is a dict corresponding to an individual reference
            from the article's reference list. Includes title, authors,
            publishing date, journal title, volume, and pages (if listed),
            and any external URL links available (i.e. to where it is hosted
            on other sites, or pdf links).

        full_url : str
            URL to the journal article page on publisher's website.

    """
    # Resolve DOI or URL through PyPub pub_resolve methods
    publisher_base_url, full_url = pub_resolve.get_publisher_urls(doi=doi, url=url)
    pub_dict = pub_resolve.get_publisher_site_info(publisher_base_url)

    # Create a PaperInfo object to hold all information and call appropriate scraper
    paper_info = PaperInfo(doi=doi, scraper_obj=pub_dict.get('object'), url=full_url)
    paper_info.populate_info()

    return paper_info

def retrieve_all_info(input, input_type, skip_saved=False):
    """
    Checks Scopus first, and then web-scrapes.

    Potential input types:
    'doi', 'pubmed_id', 'eid', 'url'

    Returns
    -------
    paper_info : PaperInfo object

    """
    paper_info = None

    scopus_api = Scopus()

    if input_type == 'doi':
        # Check for the DOI and corresponding paper in user's database.
        # If it has already been saved, return saved values.
        if not skip_saved:
            saved_info = db.get_saved_info(input)
            if saved_info:
                saved_info.make_interface_object()
                return saved_info
            else:
                saved_info.references = ref_retrieval.retrieve_references(saved_info.doi)
                # These lines probably don't belong in this function. This is for the case in which
                # a paper's information has already been saved in the database, but has no corresponding
                # references. This saves the references after having retrieved them.
                if saved_info.references is not None and len(saved_info.references) > 0:
                    db.add_references(refs=saved_info.references, main_paper_doi=saved_info.doi.lower(),
                                     main_paper_title=getattr(saved_info.entry, 'title', None))
                else:
                    return saved_info

        try:
            # This next line fetches information from Scopus
            paper_info = scopus_api.get_all_data.get_from_doi(doi=input)
        except LookupError or ConnectionRefusedError or ConnectionError:
            # If Scopus could not connect or info wasn't found there,
            # this line attempts to web-scrape from a publisher site.
            paper_info = doi_to_webscraped_info(doi=input)
            db.log_info(paper_info)

        if paper_info.pdf_link is None:
            if paper_info.publisher_interface is None:
                paper_info.make_interface_object()
            try:
                paper_info.pdf_link = paper_info.publisher_interface.get_pdf_link(input)
            except Exception:
                pass

    elif input_type == 'pubmed_id':
        paper_info = scopus_api.get_all_data.get_from_pubmed(pubmed_id=input)

    return paper_info

def retrieve_only_references(input, input_type, skip_saved=False):
    """
    Checks Scopus first, and then web-scrapes.

    Potential input types:
    'doi', 'pubmed_id', 'eid', 'url'

    Returns
    -------
    references

    """
    scopus_api = Scopus()

    if input_type == 'doi':
        # Check for the DOI and corresponding paper in user's database.
        # If it has already been saved, return saved values.
        if not skip_saved:
            references = db.get_references_from_db(input)
            if references:
                return references
            else:
                references = ref_retrieval.retrieve_references(input)
                # These lines probably don't belong in this function. This is for the case in which
                # a paper's information has already been saved in the database, but has no corresponding
                # references. This saves the references after having retrieved them.
                if references:
                    db.add_references(refs=references, main_paper_doi=input.lower())
                else:
                    return references

        else:
            references = ref_retrieval.retrieve_references(input)
            return references

    elif input_type == 'pubmed_id':
        paper_info = scopus_api.get_all_data.get_from_pubmed(pubmed_id=input)
        return paper_info
    else:
        raise UnsupportedTypeError('Only DOI and PMID lookups are supported at this time.')

'''
DOI_SEARCH = 'http://doi.crossref.org/search/doi'

crossref_email = 'jim.hokanson@gmail.com'

def getMeta(doi, format='unixsd'):

    """

    Given a doi, return article meta data

    #Output formats:
    #-------------------------
    #1) xsd_xml - default - 'unixsd' - recommended for future use
    #http://help.crossref.org/#unixsd
    #DOC: query_output3.0
    #http://www.crossref.org/schema/documentation/crossref_query_output3.0/query_output3.0.html
    #2) 'unixref'
    #http://help.crossref.org/#unixref-query-result-format

    #TODO: Document why unixref might be preferable
    #TODO: Finish parsing of xsd

    """

    #pid    - email
    #format - unixsd
    #doi    - doi
    payload = {'pid':crossref_email, 'format':format, 'doi':doi}
    r = requests.get(DOI_SEARCH, params=payload)

    #XSD code generation
    #http://www.rexx.com/~dkuhlman/generateDS.html
    #generateds_gui.py

'''<commit_msg>updates<commit_after>"""

Reference Resolver
------------------

NOTE: All of these approaches use web-scraping directly from publisher sites.

Goal:
    Take something like this:
        Senís, Elena, et al. "CRISPR/Cas9‐mediated genome engineering:
        An adeno‐associated viral (AAV) vector toolbox."
        Biotechnology journal 9.11 (2014): 1402-1412.

    and retrieve the entry data and/or that paper's references from the
    corresponding site where it is hosted.

    Do the same for a DOI or link.

------------------

Basically follow the method outlined here:
    http://labs.crossref.org/resolving-citations-we-dont-need-no-stinkin-parser/

Another super helpful link about DOIs and publishers:
    https://webhome.weizmann.ac.il/home/comartin/doi.html

Steps:
    1) Search for the original reference on CrossRef
    2) Extract DOI from CrossRef
    3) Match DOI with provider
    4) Call appropriate scraper with DOI and get info
    5) Return info

"""
# Standard imports
import json
import json.decoder
import sys

if sys.version_info.major == 2:
    from urllib import quote as urllib_quote
else:
    from urllib.parse import quote as urllib_quote

# Third party imports
import requests

# Local imports
from pypub.paper_info import PaperInfo
from pypub.publishers import pub_resolve
from pypub.utils import find_nth
from . import ref_retrieval
from .rr_errors import *

from scopy import Scopus

from database import db_logging as db

# -----------------------------------------------------

def paper_info_from_citation(citation):
    """
    Gets the paper and references information from
    a plaintext citation.

    Uses a search to CrossRef.org to retrive paper DOI.

    Parameters
    ----------
    citation : str
        Full journal article citation.
        Example: Senís, Elena, et al. "CRISPR/Cas9‐mediated genome
                engineering: An adeno‐associated viral (AAV) vector
                toolbox. Biotechnology journal 9.11 (2014): 1402-1412.

    Returns
    -------
    paper_info : PaperInfo
        Class containing relevant paper meta-information and
        references list.
        Information about the paper itself is in 'entry' value, and is a dict
        (with str and dict values). References list is in 'references'
        value and is a list of dicts (each with str and dict values).
        Must call .__dict__ to be JSON-serializable

    """
    # Encode raw citation
    citation = urllib_quote(citation)

    # Search for citation on CrossRef.org to try to get a DOI link
    api_search_url = 'http://search.labs.crossref.org/dois?q=' + citation
    response = requests.get(api_search_url).json()
    resp = response[0]
    doi = resp.get('doi')
    print(doi)

    if doi is None:
        raise LookupError('No DOI could be found for the given citation')

    # If crossref returns a http://dx.doi.org/ link, retrieve the doi from it
    # and save the URL to pass to doi_to_info
    url = None
    if doi[0:18] == 'http://dx.doi.org/':
        url = doi
        doi = doi[18:]

    # Check if this DOI has been searched and saved before.
    # If it has, return saved information
    saved_info = db.get_saved_info(doi)
    if saved_info is not None:
        saved_info.make_interface_object()
        return saved_info

    paper_info = doi_to_webscraped_info(doi=doi, url=url)
    db.log_info(paper_info)

    return paper_info

# This is commented out because retrieve_all_info subsumes it.
'''
def paper_info_from_doi(doi, skip_saved=False):
    """
    Gets the paper and references information from an article DOI.

    Parameters
    ----------
    doi : str
        DOI = digital object identifier.
        Unique ID assigned to a journal article.
        Example: 10.1002/biot.201400046

    Returns
    -------
    paper_info : PaperInfo
        See resolve_citation for description.

    """
    if not skip_saved:
        # Check for the DOI and corresponding paper in user's database.
        # If it has already been saved, return saved values.
        saved_info = db.get_saved_info(doi)
        if saved_info is not None:
            saved_info.make_interface_object()
            return saved_info

    paper_info = doi_to_webscraped_info(doi=doi)
    db.log_info(paper_info)

    return paper_info
'''

def paper_info_from_link(link):
    pub_dict = resolve_link(link)

def resolve_link(link):
    """
    Gets the paper and references information from a link (URL)
    to a specific journal article page.

    Parameters
    ----------
    link : str
        URL to journal article page on publisher's website.
        Example: http://onlinelibrary.wiley.com/doi/10.1002/biot.201400046/references

    Returns
    -------
    pub_dict : dict
        See resolve_citation for description.

    """

    # First format the link correctly and determine the publisher
    # ---------------------
    # Make sure 'http://' and not 'www.' is at the beginning
    link = link.replace('www.', '')
    if link[0:4] != 'http':
        link = 'http://' + link

    # Find the third '/', which should be after the base URL
    # i.e. giving something like http://onlinelibrary.wiley.com/
    end_of_base_url = find_nth(link, '/', 3)
    base_url = link[:end_of_base_url+1]

    return pub_resolve.get_publisher_site_info(base_url=base_url)

def link_to_doi(link):
    return_values = resolve_link(link)
    return return_values

def doi_and_title_from_citation(citation):
    """
    Gets the DOI from
    a plaintext citation.

    Uses a search to CrossRef.org to retrive paper DOI.

    Parameters
    ----------
    citation : str
        Full journal article citation.
        Example: Senís, Elena, et al. "CRISPR/Cas9‐mediated genome
                engineering: An adeno‐associated viral (AAV) vector
                toolbox. Biotechnology journal 9.11 (2014): 1402-1412.

    Returns
    -------
    doi : str
    """
    # Encode raw citation
    citation = urllib_quote(citation)

    # Search for citation on CrossRef.org to try to get a DOI link
    api_search_url = 'http://search.labs.crossref.org/dois?q=' + citation
    try:
        response = requests.get(api_search_url).json()
    except json.decoder.JSONDecodeError:
        return None

    resp = response[0]
    doi = resp.get('doi')
    title = resp.get('title')

    if doi is None:
        return None

    # If crossref returns a http://dx.doi.org/ link, retrieve the doi from it
    # and save the URL to pass to doi_to_info
    if 'http://dx.doi.org/' in doi:
        doi = doi.replace('http://dx.doi.org/', '')
        doi = doi.strip()

    return doi, title

def doi_to_webscraped_info(doi=None, url=None):
    """
    Gets entry and references information for an article DOI.

    Uses saved dicts matching DOI prefixes to publishers and web scrapers
    to retrieve information. Will fail if DOI prefix hasn't been saved
    with a publisher link or if a scraper for a specific publisher
    site hasn't been built.

    Parameters
    ----------
    doi : str
        Unique ID assigned to a journal article.
    url : str
        The CrossRef URL to the article page.
        I.e. http://dx.doi.org/10.######

    Returns
    -------
    paper_info : PaperInfo
        Class containing parameters including the following:

        entry_dict : dict
            Contains information about the paper referenced by the DOI.
            Includes title, authors, affiliations, publish date, journal
            title, volume, and pages, and keywords. Some values are other
            dicts (for example, the author info with affiliation values).
            Formatted to be JSON serializable.

        refs_dicts : list of dicts
            Each list item is a dict corresponding to an individual reference
            from the article's reference list. Includes title, authors,
            publishing date, journal title, volume, and pages (if listed),
            and any external URL links available (i.e. to where it is hosted
            on other sites, or pdf links).

        full_url : str
            URL to the journal article page on publisher's website.

    """
    # Resolve DOI or URL through PyPub pub_resolve methods
    publisher_base_url, full_url = pub_resolve.get_publisher_urls(doi=doi, url=url)
    pub_dict = pub_resolve.get_publisher_site_info(publisher_base_url)

    # Create a PaperInfo object to hold all information and call appropriate scraper
    paper_info = PaperInfo(doi=doi, scraper_obj=pub_dict.get('object'), url=full_url)
    paper_info.populate_info()

    return paper_info

def retrieve_all_info(input, input_type, skip_saved=False):
    """
    Checks Scopus first, and then web-scrapes.

    Potential input types:
    'doi', 'pubmed_id', 'eid', 'url'

    Returns
    -------
    paper_info : PaperInfo object

    """
    paper_info = None

    scopus_api = Scopus()

    if input_type == 'doi':
        # Check for the DOI and corresponding paper in user's database.
        # If it has already been saved, return saved values.
        if not skip_saved:
            saved_info = db.get_saved_info(input)
            if saved_info:
                if getattr(saved_info, 'references', None) is not None:
                    saved_info.make_interface_object()
                    return saved_info
                else:
                    saved_info.references = ref_retrieval.retrieve_references(saved_info.doi)
                    # These lines probably don't belong in this function. This is for the case in which
                    # a paper's information has already been saved in the database, but has no corresponding
                    # references. This saves the references after having retrieved them.
                    if saved_info.references is not None and len(saved_info.references) > 0:
                        db.add_references(refs=saved_info.references, main_paper_doi=saved_info.doi.lower(),
                                         main_paper_title=getattr(saved_info.entry, 'title', None))
                    else:
                        return saved_info

        try:
            # This next line fetches information from Scopus
            paper_info = scopus_api.get_all_data.get_from_doi(doi=input)
        except LookupError or ConnectionRefusedError or ConnectionError:
            # If Scopus could not connect or info wasn't found there,
            # this line attempts to web-scrape from a publisher site.
            paper_info = doi_to_webscraped_info(doi=input)
            db.log_info(paper_info)

        if paper_info.pdf_link is None:
            if paper_info.publisher_interface is None:
                paper_info.make_interface_object()
            try:
                paper_info.pdf_link = paper_info.publisher_interface.get_pdf_link(input)
            except Exception:
                pass

    elif input_type == 'pubmed_id':
        paper_info = scopus_api.get_all_data.get_from_pubmed(pubmed_id=input)

    return paper_info

def retrieve_only_references(input, input_type, skip_saved=False):
    """
    Checks Scopus first, and then web-scrapes.

    Potential input types:
    'doi', 'pubmed_id', 'eid', 'url'

    Returns
    -------
    references

    """
    scopus_api = Scopus()

    if input_type == 'doi':
        # Check for the DOI and corresponding paper in user's database.
        # If it has already been saved, return saved values.
        if not skip_saved:
            references, main_paper_exists = db.get_references_from_db(input)
            if references:
                return references
            else:
                references = ref_retrieval.retrieve_references(input)
                # These lines probably don't belong in this function. This is for the case in which
                # a paper's information has already been saved in the database, but has no corresponding
                # references. This saves the references after having retrieved them.
                if references and main_paper_exists:
                    db.add_references(refs=references, main_paper_doi=input.lower())
                else:
                    return references

        else:
            references = ref_retrieval.retrieve_references(input)
            return references

    elif input_type == 'pubmed_id':
        paper_info = scopus_api.get_all_data.get_from_pubmed(pubmed_id=input)
        return paper_info
    else:
        raise UnsupportedTypeError('Only DOI and PMID lookups are supported at this time.')

'''
DOI_SEARCH = 'http://doi.crossref.org/search/doi'

crossref_email = 'jim.hokanson@gmail.com'

def getMeta(doi, format='unixsd'):

    """

    Given a doi, return article meta data

    #Output formats:
    #-------------------------
    #1) xsd_xml - default - 'unixsd' - recommended for future use
    #http://help.crossref.org/#unixsd
    #DOC: query_output3.0
    #http://www.crossref.org/schema/documentation/crossref_query_output3.0/query_output3.0.html
    #2) 'unixref'
    #http://help.crossref.org/#unixref-query-result-format

    #TODO: Document why unixref might be preferable
    #TODO: Finish parsing of xsd

    """

    #pid    - email
    #format - unixsd
    #doi    - doi
    payload = {'pid':crossref_email, 'format':format, 'doi':doi}
    r = requests.get(DOI_SEARCH, params=payload)

    #XSD code generation
    #http://www.rexx.com/~dkuhlman/generateDS.html
    #generateds_gui.py

'''<|endoftext|>